From 3e864eca04bb11e1e0c2129e8226b12569c798f3 Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Thu, 8 Oct 2026 18:35:47 +0800 Subject: [PATCH 01/20] refactor(api): switch application services and execution entry points --- api/.importlinter | 4 +- api/celery_healthcheck.py | 36 +- api/commands/data_migration.py | 19 +- api/controllers/console/agent/composer.py | 14 +- api/controllers/console/agent/roster.py | 32 +- api/controllers/console/app/completion.py | 17 +- api/controllers/console/app/error.py | 4 +- api/controllers/console/app/model_config.py | 7 +- api/controllers/console/app/workflow.py | 1566 +++--------- .../console/app/workflow_admission.py | 58 + .../app/workflow_node_output_inspector.py | 3 +- .../datasource_content_preview.py | 5 +- .../rag_pipeline/rag_pipeline_workflow.py | 305 ++- api/controllers/console/explore/message.py | 12 +- api/controllers/console/explore/trial.py | 147 +- .../console/snippets/snippet_workflow.py | 214 +- api/controllers/console/workspace/snippets.py | 39 +- api/controllers/inner_api/agent/tools.py | 9 +- api/controllers/inner_api/plugin/plugin.py | 11 +- .../inner_api/runtime_credentials.py | 2 +- api/controllers/mcp/mcp.py | 17 +- api/controllers/openapi/app_run.py | 37 +- api/controllers/service_api/app/completion.py | 21 +- api/controllers/service_api/app/workflow.py | 14 +- .../rag_pipeline/rag_pipeline_workflow.py | 20 +- api/controllers/trigger/__init__.py | 3 + api/controllers/trigger/errors.py | 25 + api/controllers/trigger/trigger.py | 4 +- api/controllers/trigger/webhook.py | 21 +- api/controllers/web/completion.py | 15 +- api/controllers/web/message.py | 22 +- api/controllers/web/workflow.py | 5 +- api/core/app/apps/agent_app/session_store.py | 3 +- .../app/layers/pause_state_persist_layer.py | 4 +- .../based_generate_task_pipeline.py | 18 +- api/core/app/workflow/__init__.py | 3 - api/core/app/workflow/file_runtime.py | 17 +- api/core/app/workflow/layers/__init__.py | 9 +- api/core/mcp/server/streamable_http.py | 13 +- api/core/plugin/backwards_invocation/node.py | 2 - api/core/tools/__base/tool.py | 6 +- .../builtin_tool/providers/audio/tools/asr.py | 3 - .../builtin_tool/providers/audio/tools/tts.py | 3 - .../providers/code/tools/simple_code.py | 3 - .../providers/time/tools/current_time.py | 2 - .../time/tools/localtime_to_timestamp.py | 2 - .../time/tools/timestamp_to_localtime.py | 2 - .../time/tools/timezone_conversion.py | 2 - .../providers/time/tools/weekday.py | 3 - .../providers/webscraper/tools/webscraper.py | 3 - api/core/tools/custom_tool/tool.py | 2 - api/core/tools/mcp_tool/tool.py | 5 +- api/core/tools/plugin_tool/tool.py | 3 - api/core/workflow/nodes/agent/__init__.py | 3 +- .../nodes/agent/message_transformer.py | 40 +- api/core/workflow/nodes/agent_v2/__init__.py | 3 +- .../workflow/nodes/agent_v2/ask_human_hitl.py | 2 +- .../workflow/nodes/agent_v2/session_store.py | 5 +- .../workflow/nodes/human_input/__init__.py | 52 +- .../nodes/trigger_webhook/__init__.py | 3 - api/events/app_event.py | 3 - api/events/event_handlers/__init__.py | 12 - .../create_installed_app_when_app_created.py | 2 +- .../create_site_record_when_app_created.py | 2 +- ...rameters_cache_when_sync_draft_workflow.py | 5 +- .../sync_webhook_when_app_created.py | 26 +- api/extensions/application_services/agent.py | 17 +- api/extensions/application_services/app.py | 13 +- .../application_services/installed_app.py | 9 +- .../application_services/knowledge.py | 38 +- .../application_services/snippets.py | 65 + .../application_services/trial_app.py | 11 +- .../application_services/workflow.py | 142 ++ api/extensions/ext_application_services.py | 62 +- api/extensions/workflow_warm_shutdown.py | 2 +- api/fields/agent_fields.py | 2 +- api/fields/workflow_run_fields.py | 19 +- api/libs/helper.py | 8 +- api/models/model.py | 119 - api/models/workflow.py | 40 - .../app_definition_query_repository.py | 2 +- ...hemy_execution_extra_content_repository.py | 2 +- api/services/agent/__init__.py | 5 +- api/services/agent/composer_service.py | 323 +-- api/services/agent/dsl_entities.py | 13 +- api/services/agent/dsl_service.py | 151 +- api/services/agent/home_snapshot_service.py | 21 +- api/services/agent/roster_package_importer.py | 6 +- api/services/agent/roster_service.py | 68 +- api/services/agent/workspace_service.py | 76 +- api/services/agent_service.py | 161 -- api/services/annotation_service.py | 86 +- api/services/app/console_gateway.py | 21 +- api/services/app/response_gateway.py | 16 +- api/services/app_definition_query_service.py | 2 +- api/services/app_dsl_service.py | 184 +- api/services/app_generate_service.py | 397 +-- api/services/app_package_service.py | 4 +- api/services/app_preview_details_adapters.py | 15 +- api/services/app_service.py | 47 +- api/services/data_migration/export_service.py | 14 +- api/services/data_migration/import_service.py | 78 +- .../human_input_delivery_test_service.py | 48 +- .../installed_app_generation_adapters.py | 21 +- api/services/knowledge/dataset_service.py | 300 +-- api/services/message_service.py | 2 +- api/services/plugin/app_invocation.py | 363 +++ .../rag_pipeline/pipeline_generate_service.py | 82 +- api/services/rag_pipeline/rag_pipeline.py | 964 +------- api/services/snippet_dsl_service.py | 88 +- api/services/snippet_generate_service.py | 174 +- api/services/snippet_service.py | 450 +--- api/services/trial_app_generation_adapters.py | 14 +- api/services/trigger/errors.py | 5 + api/services/trigger/schedule_service.py | 236 +- api/services/trigger/trigger_service.py | 166 +- api/services/trigger/webhook_service.py | 138 +- .../execution/adapters/node_factory.py | 21 +- .../workflow/node_output_inspector_service.py | 13 +- api/services/workflow/runtime_gateway.py | 266 ++ api/services/workflow/workflow_converter.py | 256 +- .../workflow_node_execution_trace_service.py | 22 +- api/services/workflow_run_service.py | 7 +- api/services/workflow_service.py | 1331 +---------- .../app_generate/resume_agent_app_task.py | 18 +- .../app_generate/workflow_execute_task.py | 502 ++-- api/tasks/async_workflow_tasks.py | 11 +- api/tasks/human_input_timeout_tasks.py | 48 +- .../priority_rag_pipeline_run_task.py | 32 +- .../rag_pipeline/rag_pipeline_run_task.py | 109 +- api/tasks/retry_document_indexing_task.py | 16 +- api/tasks/workflow_draft_var_tasks.py | 18 +- .../workflow/nodes/test_code.py | 2 +- .../workflow/nodes/test_http.py | 4 +- .../conftest.py | 11 +- .../controllers/openapi/test_app_dsl.py | 3 +- .../test_human_input_resume_node_execution.py | 12 +- .../services/dataset_collection_binding.py | 34 +- .../services/test_agent_service.py | 1028 -------- .../services/test_annotation_service.py | 133 +- .../services/test_app_dsl_service.py | 87 +- .../services/test_app_generate_service.py | 306 ++- .../test_conversation_variable_updater.py | 20 +- .../services/test_dataset_service_document.py | 24 +- .../test_dataset_service_permissions.py | 13 +- .../test_dataset_service_update_dataset.py | 4 +- .../test_human_input_delivery_test.py | 23 +- .../services/test_schedule_service.py | 119 +- .../test_webhook_service_relationships.py | 86 +- .../services/test_workflow_app_service.py | 6 +- .../test_workflow_draft_variable_service.py | 191 +- .../services/test_workflow_service.py | 465 +--- .../services/tools/conftest.py | 17 + .../tools/test_api_tools_manage_service.py | 27 +- .../tools/test_mcp_tools_manage_service.py | 12 +- .../tools/test_tools_transform_service.py | 48 +- .../test_workflow_tools_manage_service.py | 155 +- .../workflow/test_workflow_converter.py | 860 +------ .../workflow/test_workflow_deletion.py | 21 +- .../test_mail_human_input_delivery_task.py | 3 +- .../tasks/test_rag_pipeline_run_tasks.py | 4 +- .../trigger/test_trigger_e2e.py | 11 +- .../commands/test_data_migration_commands.py | 28 +- api/tests/unit_tests/conftest.py | 198 +- api/tests/unit_tests/controllers/conftest.py | 44 +- .../console/agent/test_agent_controllers.py | 40 +- .../controllers/console/app/test_app_apis.py | 88 +- .../console/app/test_app_response_models.py | 6 +- .../controllers/console/app/test_workflow.py | 1286 ++-------- .../console/app/test_workflow_admission.py | 150 ++ .../console/app/test_workflow_convert_api.py | 58 - .../test_workflow_human_input_debug_api.py | 281 +-- .../console/app/test_workflow_run_api.py | 4 +- .../test_datasource_content_preview.py | 31 +- .../test_rag_pipeline_workflow.py | 19 +- .../test_rag_pipeline_workflow_apis.py | 275 +-- .../console/explore/test_completion.py | 9 +- .../explore/test_installed_app_completion.py | 9 +- .../console/explore/test_message.py | 2 + .../console/explore/test_trial_audio.py | 10 + .../console/explore/test_trial_generation.py | 2 +- .../explore/test_trial_preview_details.py | 60 +- .../console/explore/test_workflow.py | 9 +- .../console/snippets/test_snippet_workflow.py | 227 +- .../console/workspace/test_snippets.py | 30 +- .../inner_api/plugin/test_plugin.py | 100 +- .../controllers/inner_api/test_agent_tools.py | 32 +- .../unit_tests/controllers/mcp/test_mcp.py | 7 +- .../openapi/test_app_run_rate_limit.py | 6 +- .../openapi/test_app_run_streaming.py | 7 +- .../service_api/app/test_completion.py | 61 +- .../service_api/app/test_workflow.py | 15 +- .../test_rag_pipeline_workflow.py | 82 +- .../test_trace_session_id_parsing.py | 3 + .../controllers/trigger/test_trigger.py | 4 +- .../controllers/trigger/test_webhook.py | 57 +- .../controllers/web/test_completion.py | 5 +- .../controllers/web/test_message_endpoints.py | 31 +- .../controllers/web/test_workflow.py | 3 + .../app/apps/agent_app/test_resolve_agent.py | 537 ----- .../core/app/apps/workflow/test_errors.py | 9 - .../app/features/test_annotation_reply.py | 185 -- ...est_conversation_variable_persist_layer.py | 114 - .../core/mcp/server/test_streamable_http.py | 148 +- .../plugin/test_backwards_invocation_app.py | 94 +- .../unit_tests/core/tools/test_base_tool.py | 21 +- .../core/tools/test_builtin_tools_extra.py | 109 +- .../unit_tests/core/tools/test_custom_tool.py | 4 +- .../core/tools/test_dataset_retriever_tool.py | 208 +- .../unit_tests/core/tools/test_mcp_tool.py | 13 +- .../unit_tests/core/tools/test_plugin_tool.py | 8 +- .../unit_tests/core/tools/test_tool_engine.py | 76 +- .../core/tools/utils/test_misc_utils_extra.py | 539 ----- .../core/tools/workflow_as_tool/test_tool.py | 445 +++- .../nodes/agent/test_message_transformer.py | 6 +- .../nodes/agent_v2/test_agent_node.py | 199 +- .../nodes/agent_v2/test_binding_resolver.py | 182 +- .../test_tool_node_invocation_behavior.py | 18 +- .../workflow/test_node_mapping_bootstrap.py | 24 +- ...st_sync_plugin_trigger_when_app_created.py | 31 - .../test_sync_webhook_when_app_created.py | 25 +- .../test_ext_application_services.py | 10 +- .../extensions/test_workflow_warm_shutdown.py | 4 +- .../unit_tests/fields/test_workflow_fields.py | 304 +++ .../libs/test_cron_compatibility.py | 8 +- api/tests/unit_tests/models/test_agent.py | 2 +- .../unit_tests/models/test_app_models.py | 51 - .../test_installed_app_session_accessors.py | 102 +- .../test_workflow_node_execution_accessors.py | 37 +- api/tests/unit_tests/pyrefly.toml | 4 - .../test_agent_execution_repository.py | 148 ++ ...t_app_definition_query_repository_agent.py | 2 +- .../test_app_preview_query_repository.py | 2 +- ...hemy_execution_extra_content_repository.py | 6 +- .../test_workflow_app_log_query_repository.py | 18 +- .../workflow/test_definition_repository.py | 1998 ++++++++++++++++ .../workflow/test_draft_repository.py | 446 ++++ .../services/agent/test_agent_dsl_service.py | 100 +- .../agent/test_agent_observability_service.py | 3 +- .../services/agent/test_agent_services.py | 236 +- .../services/agent/test_deletion_service.py | 2 +- .../agent/test_home_snapshot_service.py | 3 +- .../services/agent/test_log_service.py | 4 +- .../services/agent/test_retirement_service.py | 190 +- .../agent/test_roster_package_importer.py | 9 +- .../agent/test_roster_package_service.py | 13 +- .../agent/test_tool_invocation_service.py | 71 +- .../agent/test_workflow_package_service.py | 38 +- .../app/test_agent_app_access_service.py | 22 +- .../app/test_agent_app_feature_service.py | 15 +- .../app/test_agent_chat_attachments.py | 201 ++ .../services/app/test_console_gateway.py | 4 +- .../services/app/test_console_service.py | 51 +- .../services/app/test_response_gateway.py | 6 +- .../data_migration/test_export_service.py | 33 +- .../data_migration/test_import_service.py | 185 +- .../services/dataset_service_test_helpers.py | 50 +- .../test_runner_retrieval_boundary.py | 295 +++ .../test_pipeline_generate_service.py | 255 +- .../rag_pipeline/test_rag_pipeline_service.py | 694 +++--- .../services/test_agent_tool_inner_service.py | 198 -- .../services/test_annotation_service.py | 63 +- .../test_app_definition_query_service.py | 2 +- .../services/test_app_dsl_service.py | 159 +- .../services/test_app_generate_service.py | 573 +++-- ..._generate_service_streaming_integration.py | 27 +- .../test_app_preview_details_adapters.py | 28 +- .../unit_tests/services/test_app_service.py | 34 +- .../test_conversation_variable_updater.py | 4 +- .../services/test_dataset_service_dataset.py | 161 +- .../services/test_dataset_service_document.py | 11 +- .../test_human_input_delivery_test_service.py | 6 +- .../test_installed_app_generation_adapters.py | 48 +- .../unit_tests/services/test_package_icons.py | 4 +- .../services/test_schedule_service.py | 74 +- .../services/test_skill_management_service.py | 2 +- .../services/test_snippet_dsl_service.py | 65 +- .../services/test_snippet_generate_service.py | 249 +- .../services/test_snippet_service.py | 524 ++-- .../test_trial_app_generation_adapters.py | 41 +- .../test_webhook_service_additional.py | 8 +- .../test_webhook_service_relationships.py | 123 +- .../test_workflow_app_log_query_service.py | 14 +- ...t_workflow_node_execution_trace_service.py | 32 +- .../services/test_workflow_run_service.py | 42 +- .../test_workflow_run_service_pause.py | 31 +- .../services/test_workflow_service.py | 2087 +--------------- .../tools/test_api_provider_runtime.py | 155 ++ .../tools/test_builtin_provider_runtime.py | 289 +++ .../workflow/test_conversion_service.py | 262 ++ .../workflow/test_draft_var_loader_simple.py | 35 +- .../workflow/test_generation_read_boundary.py | 130 + .../workflow/test_human_input_persistence.py | 192 ++ .../test_node_output_inspector_service.py | 13 +- .../services/workflow/test_runtime_gateway.py | 2129 +++++++++++++++++ .../test_workflow_converter_additional.py | 432 +--- .../test_workflow_human_input_delivery.py | 453 ++-- .../tasks/test_human_input_timeout_tasks.py | 2 +- .../test_remove_app_and_related_data_task.py | 3 +- .../tasks/test_resume_agent_app_task.py | 8 +- .../test_retry_document_indexing_task.py | 4 +- .../tasks/test_workflow_execute_task.py | 164 +- api/tests/unit_tests/tools/test_api_tool.py | 24 +- api/tests/unit_tests/tools/test_mcp_tool.py | 28 +- 304 files changed, 16077 insertions(+), 19415 deletions(-) create mode 100644 api/controllers/console/app/workflow_admission.py create mode 100644 api/controllers/trigger/errors.py create mode 100644 api/extensions/application_services/snippets.py create mode 100644 api/services/plugin/app_invocation.py create mode 100644 api/services/trigger/errors.py create mode 100644 api/services/workflow/runtime_gateway.py delete mode 100644 api/tests/test_containers_integration_tests/services/test_agent_service.py create mode 100644 api/tests/test_containers_integration_tests/services/tools/conftest.py create mode 100644 api/tests/unit_tests/controllers/console/app/test_workflow_admission.py delete mode 100644 api/tests/unit_tests/controllers/console/app/test_workflow_convert_api.py delete mode 100644 api/tests/unit_tests/core/app/apps/agent_app/test_resolve_agent.py delete mode 100644 api/tests/unit_tests/core/app/apps/workflow/test_errors.py delete mode 100644 api/tests/unit_tests/core/app/features/test_annotation_reply.py delete mode 100644 api/tests/unit_tests/core/app/layers/test_conversation_variable_persist_layer.py delete mode 100644 api/tests/unit_tests/events/event_handlers/test_sync_plugin_trigger_when_app_created.py create mode 100644 api/tests/unit_tests/fields/test_workflow_fields.py create mode 100644 api/tests/unit_tests/repositories/test_agent_execution_repository.py create mode 100644 api/tests/unit_tests/repositories/workflow/test_definition_repository.py create mode 100644 api/tests/unit_tests/repositories/workflow/test_draft_repository.py create mode 100644 api/tests/unit_tests/services/app/test_agent_chat_attachments.py create mode 100644 api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py delete mode 100644 api/tests/unit_tests/services/test_agent_tool_inner_service.py create mode 100644 api/tests/unit_tests/services/tools/test_api_provider_runtime.py create mode 100644 api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py create mode 100644 api/tests/unit_tests/services/workflow/test_conversion_service.py create mode 100644 api/tests/unit_tests/services/workflow/test_generation_read_boundary.py create mode 100644 api/tests/unit_tests/services/workflow/test_human_input_persistence.py create mode 100644 api/tests/unit_tests/services/workflow/test_runtime_gateway.py diff --git a/api/.importlinter b/api/.importlinter index 4db00db0276cd4..e8bc504be34049 100644 --- a/api/.importlinter +++ b/api/.importlinter @@ -31,6 +31,8 @@ layers = # error makes this list shrink whenever a legacy dependency is removed. unmatched_ignore_imports_alerting = error ignore_imports = + core.app.apps.advanced_chat.app_runner -> extensions.workflow_warm_shutdown + core.app.apps.workflow.app_runner -> extensions.workflow_warm_shutdown core.app.app_config.easy_ui_based_app.dataset.manager -> services.knowledge.dataset_service core.app.apps.advanced_chat.app_generator -> controllers.console.app.workflow core.app.apps.advanced_chat.app_generator -> services.conversation_service @@ -89,7 +91,6 @@ ignore_imports = core.workflow.node_runtime -> services.tools.legacy_builtin_tools_manage_service # Node construction delegates datasource dependency assembly to the composition root. core.workflow.node_factory -> extensions.application_services.data_sources - core.workflow.nodes.agent.message_transformer -> services.tools.legacy_builtin_tools_manage_service core.workflow.nodes.agent_v2.agent_node -> services.agent.prompt_mentions core.workflow.nodes.agent_v2.agent_node -> services.agent.workspace_service core.workflow.nodes.agent_v2.dify_tools_builder -> services.tools.legacy_mcp_tools_manage_service @@ -116,7 +117,6 @@ ignore_imports = libs.oauth_bearer -> models libs.rsa -> extensions.ext_storage libs.workspace_permission -> services.enterprise.enterprise_service - services.app_generate_service -> controllers.console.app.workflow [importlinter:contract:no-direct-rsa-imports] # Note: `libs` itself is deliberately excluded from source_modules -- import-linter's diff --git a/api/celery_healthcheck.py b/api/celery_healthcheck.py index 23d856d7d04dc0..b4c3d3ecd36331 100644 --- a/api/celery_healthcheck.py +++ b/api/celery_healthcheck.py @@ -1,18 +1,18 @@ -# This module provides a lightweight Celery instance for use in Docker health checks. -# Unlike celery_entrypoint.py, this does NOT import app.py and therefore avoids -# initializing all Flask extensions (DB, Redis, storage, blueprints, etc.). -# Using this module keeps the health check fast and low-cost. -from celery import Celery - -from configs import dify_config -from extensions.ext_celery import get_celery_broker_transport_options, get_celery_ssl_options - -celery = Celery(broker=dify_config.CELERY_BROKER_URL) - -broker_transport_options = get_celery_broker_transport_options() -if broker_transport_options: - celery.conf.update(broker_transport_options=broker_transport_options) - -ssl_options = get_celery_ssl_options() -if ssl_options: - celery.conf.update(broker_use_ssl=ssl_options) +# This module provides a lightweight Celery instance for use in Docker health checks. +# Unlike celery_entrypoint.py, this does NOT import app.py and therefore avoids +# initializing all Flask extensions (DB, Redis, storage, blueprints, etc.). +# Using this module keeps the health check fast and low-cost. +from celery import Celery + +from configs import dify_config +from extensions.ext_celery import get_celery_broker_transport_options, get_celery_ssl_options + +celery = Celery(broker=dify_config.CELERY_BROKER_URL) + +broker_transport_options = get_celery_broker_transport_options() +if broker_transport_options: + celery.conf.update(broker_transport_options=broker_transport_options) + +ssl_options = get_celery_ssl_options() +if ssl_options: + celery.conf.update(broker_use_ssl=ssl_options) diff --git a/api/commands/data_migration.py b/api/commands/data_migration.py index 8c2627601a6c28..8c3525e059023c 100644 --- a/api/commands/data_migration.py +++ b/api/commands/data_migration.py @@ -12,6 +12,7 @@ from sqlalchemy.orm import Session from core.db.session_factory import session_factory +from extensions.ext_application_services import application_services from extensions.ext_database import db from models import Tenant from models.model import App @@ -109,7 +110,10 @@ def export_migration_data(input_file: str | None, output_file: str | None, overw raw_config = _load_json_object(input_file, "Export config") selection = ExportConfigParser().parse(raw_config) with session_factory.create_session() as session: - result = MigrationExportService().export(selection, session=session) + result = MigrationExportService( + workflow_tools=application_services().tools.workflows, + tool_providers=application_services().tools.tool_providers, + ).export(selection, session=session) MigrationPackageService().save_package(result.package, output_file, overwrite=overwrite) click.echo(click.style(f"Output written to {output_file}", fg="green")) _render_report(result.report_items, context=_with_output_path(result.report_context, output_file)) @@ -157,7 +161,13 @@ def import_migration_data( assert input_file is not None package = MigrationPackageService().load_package(input_file) with session_factory.create_session() as session: - result = MigrationImportService().import_package( + from extensions.application_services.workflow import build_app_dsl_service + + result = MigrationImportService( + workflow_tools=application_services().tools.workflows, + workflows=application_services().console_workflows, + app_dsl=build_app_dsl_service, + ).import_package( ImportRequest( package=package, cli_target_tenant=target_tenant, @@ -256,7 +266,10 @@ def migration_data_wizard() -> None: output_file=output_file, ) with session_factory.create_session() as session: - result = MigrationExportService().export(selection, session=session) + result = MigrationExportService( + workflow_tools=application_services().tools.workflows, + tool_providers=application_services().tools.tool_providers, + ).export(selection, session=session) MigrationPackageService().save_package(result.package, output_file, overwrite=overwrite) click.echo(click.style(f"Output written to {output_file}", fg="green")) _print_wizard_step("Report") diff --git a/api/controllers/console/agent/composer.py b/api/controllers/console/agent/composer.py index 68a0e942571e87..1b7006cc87fde5 100644 --- a/api/controllers/console/agent/composer.py +++ b/api/controllers/console/agent/composer.py @@ -2,8 +2,8 @@ from flask_restx import Resource from sqlalchemy.orm import Session -from werkzeug.exceptions import NotFound +from controllers.common.errors import NotFoundError from controllers.common.rbac import AgentId, PlainApp, RBACCheck, Workspace from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models from controllers.common.session import with_session @@ -19,6 +19,8 @@ with_current_tenant_id, with_current_user_id, ) +from extensions.application_services.snippets import build_snippet_service +from extensions.ext_application_services import application_services from fields.agent_fields import ( AgentAppComposerResponse, AgentComposerCandidatesResponse, @@ -36,7 +38,6 @@ WorkflowAgentComposerQuery, WorkflowComposerCopyFromRosterPayload, ) -from services.snippet_service import SnippetService register_schema_models( console_ns, ComposerSavePayload, WorkflowAgentComposerQuery, WorkflowComposerCopyFromRosterPayload @@ -202,11 +203,13 @@ def get(self, session: Session, tenant_id: str, current_user_id: str, app_model: return dump_response( AgentComposerCandidatesResponse, AgentComposerService.get_workflow_candidates( + variables=application_services().workflow_variables, session=session, tenant_id=tenant_id, app_id=app_model.id, node_id=node_id, user_id=current_user_id, + tool_providers=application_services().tools.tool_providers, ), ) @@ -276,12 +279,12 @@ def post( def _require_snippet_app_id(*, session: Session, tenant_id: str, snippet_id: UUID) -> str: - snippet = SnippetService(session=session).get_snippet_by_id( + snippet = build_snippet_service(session=session).get_snippet_by_id( snippet_id=str(snippet_id), # pyrefly: ignore[unnecessary-type-conversion] tenant_id=tenant_id, ) if snippet is None: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") return snippet.id @@ -426,11 +429,13 @@ def get(self, session: Session, tenant_id: str, current_user_id: str, snippet_id return dump_response( AgentComposerCandidatesResponse, AgentComposerService.get_workflow_candidates( + variables=application_services().workflow_variables, session=session, tenant_id=tenant_id, app_id=_require_snippet_app_id(session=session, tenant_id=tenant_id, snippet_id=snippet_id), node_id=node_id, user_id=current_user_id, + tool_providers=application_services().tools.tool_providers, ), ) @@ -582,5 +587,6 @@ def get(self, session: Session, tenant_id: str, current_user_id: str, agent_id: tenant_id=tenant_id, agent_id=str(agent_id), user_id=current_user_id, + tool_providers=application_services().tools.tool_providers, ), ) diff --git a/api/controllers/console/agent/roster.py b/api/controllers/console/agent/roster.py index 348abb7568e35d..91c35728376ca1 100644 --- a/api/controllers/console/agent/roster.py +++ b/api/controllers/console/agent/roster.py @@ -1,7 +1,7 @@ from typing import Literal from uuid import UUID -from flask import abort, request +from flask import request from flask_restx import Resource from pydantic import AliasChoices, BaseModel, Field, field_validator from sqlalchemy import func, or_, select @@ -9,6 +9,7 @@ from configs import dify_config from controllers.common.agent_access import resolve_agent_access_filter +from controllers.common.errors import AccessDeniedError, InvalidRequestError from controllers.common.rbac import AgentId, RBACCheck, Workspace from controllers.common.schema import ( query_params_from_model, @@ -412,7 +413,14 @@ def _serialize_agent_app_detail( roster_service = _agent_roster_service(session) payload = GenericAppDetailWithSite.model_validate( - AppResponseView(app_model, session=session, account=current_user, access_mode=access_mode), + AppResponseView( + app_model, + tool_providers=application_services().tools.tool_providers, + workflow_queries=application_services().tools.workflow_queries, + session=session, + account=current_user, + access_mode=access_mode, + ), from_attributes=True, ).model_dump(mode="json") agent = ( @@ -506,7 +514,15 @@ def _serialize_agent_app_pagination( "limit": app_pagination.per_page, "total": app_pagination.total, "has_more": app_pagination.has_next, - "data": [AppResponseView(app, session=session) for app in app_pagination.items], + "data": [ + AppResponseView( + app, + tool_providers=application_services().tools.tool_providers, + workflow_queries=application_services().tools.workflow_queries, + session=session, + ) + for app in app_pagination.items + ], }, ).model_dump(mode="json") payload["publication_counts"] = { @@ -605,7 +621,7 @@ def _parse_observability_time_range(start: str | None, end: str | None, account: try: return parse_time_range(start, end, timezone) except ValueError as exc: - abort(400, description=str(exc)) + raise InvalidRequestError(str(exc)) def _query_values(name: str, alias_name: str | None = None) -> list[str]: @@ -1165,7 +1181,7 @@ def get(self, session: Session, tenant_id: str, current_user: Account, agent_id: ), ) except ValueError as exc: - abort(400, description=str(exc)) + raise InvalidRequestError(str(exc)) return dump_response(AgentLogListResponse, payload) @@ -1205,7 +1221,7 @@ def get(self, session: Session, tenant_id: str, current_user: Account, agent_id: ), ) except ValueError as exc: - abort(400, description=str(exc)) + raise InvalidRequestError(str(exc)) return dump_response(AgentLogMessageListResponse, payload) @@ -1259,7 +1275,7 @@ def get( params=AgentStatisticsQueryParams(source=req_data.source, start=start, end=end, timezone=timezone), ) except ValueError as exc: - abort(400, description=str(exc)) + raise InvalidRequestError(str(exc)) return dump_response(AgentStatisticSummaryEnvelopeResponse, payload) @@ -1315,7 +1331,7 @@ def post(self, session: Session, tenant_id: str, current_user: Account, agent_id dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD and not FeatureService.get_workspace_plan(tenant_id).is_paid ): - abort(403, description="This feature requires a paid plan.") + raise AccessDeniedError("This feature requires a paid plan.") return dump_response( AgentConfigSnapshotRestoreResponse, diff --git a/api/controllers/console/app/completion.py b/api/controllers/console/app/completion.py index 61ce63e7d6b8e4..f7fe99549a1922 100644 --- a/api/controllers/console/app/completion.py +++ b/api/controllers/console/app/completion.py @@ -8,9 +8,9 @@ from flask_restx import Resource from pydantic import BaseModel, Field, field_validator from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, InternalServerError, NotFound import services +from controllers.common.errors import InternalServerError, InvalidRequestError, NotFoundError from controllers.common.fields import SimpleResultResponse from controllers.common.rbac import AgentId, PlainApp, RBACCheck from controllers.common.schema import register_response_schema_models, register_schema_models @@ -46,6 +46,7 @@ QuotaExceededError, ) from core.helper.trace_id_helper import get_external_trace_id +from extensions.ext_application_services import application_services from graphon.model_runtime.errors.invoke import InvokeError from libs import helper from libs.helper import uuid_value @@ -74,7 +75,7 @@ def _resolve_debugger_chat_streaming( if app_mode != AppMode.AGENT: return response_mode != "blocking" if response_mode_provided and response_mode == "blocking": - raise BadRequest("Agent App only supports streaming response mode.") + raise InvalidRequestError("Agent App only supports streaming response mode.") return True @@ -171,6 +172,8 @@ def post(self, req_data: CompletionMessagePayload, session: Session, current_use try: response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=current_user, @@ -182,7 +185,7 @@ def post(self, req_data: CompletionMessagePayload, session: Session, current_use # response-contract:ignore compact_generate_response return helper.compact_generate_response(response) except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") except services.errors.conversation.ConversationCompletedError: raise ConversationCompletedError() except services.errors.app_model_config.AppModelConfigBrokenError: @@ -384,7 +387,7 @@ def _resolve_current_user_agent_debug_conversation_id( account_id=current_user.id, ) if conversation_id is None: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") return conversation_id @@ -414,7 +417,7 @@ def _create_chat_message( start_new=draft_type == AgentConfigDraftType.DRAFT and not args_model.conversation_id, ) if args_model.conversation_id and args_model.conversation_id != debug_conversation_id: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") args["conversation_id"] = debug_conversation_id streaming = _resolve_debugger_chat_streaming( @@ -526,6 +529,8 @@ def _generate_chat_message( ): try: return AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=current_user, @@ -534,7 +539,7 @@ def _generate_chat_message( streaming=streaming, ) except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") except services.errors.conversation.ConversationCompletedError: raise ConversationCompletedError() except services.errors.app_model_config.AppModelConfigBrokenError: diff --git a/api/controllers/console/app/error.py b/api/controllers/console/app/error.py index c2808f149a5f37..c80c6d04554231 100644 --- a/api/controllers/console/app/error.py +++ b/api/controllers/console/app/error.py @@ -1,8 +1,8 @@ -from core.app.apps.agent_app.errors import ( +from libs.exception import BaseHTTPException +from services.app.generation.errors import ( AGENT_SESSION_CONFIGURATION_CHANGED_ERROR_CODE, AGENT_SESSION_CONFIGURATION_CHANGED_MESSAGE, ) -from libs.exception import BaseHTTPException class AppNotFoundError(BaseHTTPException): diff --git a/api/controllers/console/app/model_config.py b/api/controllers/console/app/model_config.py index 10673553d19769..01a34d1275fe1b 100644 --- a/api/controllers/console/app/model_config.py +++ b/api/controllers/console/app/model_config.py @@ -20,13 +20,14 @@ with_current_user_id, ) from core.agent.entities import AgentToolEntity -from core.tools.tool_manager import ToolManager from core.tools.utils.configuration import ToolParameterConfigurationManager from events.app_event import app_model_config_was_updated +from extensions.ext_application_services import application_services from libs.datetime_utils import naive_utc_now from libs.login import login_required from models.model import App, AppMode, AppModelConfig from services.app_model_config_service import AppModelConfigService +from services.tools.tool_manager import ToolManager register_schema_models(console_ns, AppModelConfigPayload) register_response_schema_models(console_ns, SimpleResultResponse) @@ -89,6 +90,8 @@ def post(self, session: Session, current_tenant_id: str, current_user_id: str, a # get tool try: tool_runtime = ToolManager.get_agent_tool_runtime( + tool_providers=application_services().tools.tool_providers, + workflow_queries=application_services().tools.workflow_queries, tenant_id=current_tenant_id, app_id=app_model.id, agent_tool=agent_tool_entity, @@ -129,6 +132,8 @@ def post(self, session: Session, current_tenant_id: str, current_user_id: str, a else: try: tool_runtime = ToolManager.get_agent_tool_runtime( + tool_providers=application_services().tools.tool_providers, + workflow_queries=application_services().tools.workflow_queries, tenant_id=current_tenant_id, app_id=app_model.id, agent_tool=agent_tool_entity, diff --git a/api/controllers/console/app/workflow.py b/api/controllers/console/app/workflow.py index 22e73742097dc2..e77955c0de8435 100644 --- a/api/controllers/console/app/workflow.py +++ b/api/controllers/console/app/workflow.py @@ -1,30 +1,22 @@ import json import logging -from collections.abc import Sequence -from datetime import datetime -from typing import Any, Literal, NotRequired, Self, TypedDict +from typing import Any, Literal, Self +from uuid import UUID -from flask import abort, request +from flask import request from flask_restx import Resource -from pydantic import ( - AliasChoices, - BaseModel, - ConfigDict, - Field, - RootModel, - ValidationError, - field_validator, - model_validator, -) -from sqlalchemy.orm import Session, sessionmaker -from werkzeug.exceptions import BadRequest, Forbidden, InternalServerError, NotFound +from pydantic import BaseModel, ConfigDict, Field, RootModel, ValidationError, field_validator, model_validator -import services -from configs import dify_config +import services.errors.conversation from controllers.common.controller_schemas import DefaultBlockConfigQuery, WorkflowListQuery, WorkflowUpdatePayload -from controllers.common.errors import InvalidArgumentError +from controllers.common.errors import ( + InternalServerError, + InvalidArgumentError, + InvalidRequestError, + NotFoundError, + UnsupportedMediaTypeError, +) from controllers.common.fields import GeneratedAppResponse, NewAppResponse, SimpleResultResponse -from controllers.common.rbac import PlainApp, RBACCheck from controllers.common.schema import ( query_params_from_model, register_response_schema_model, @@ -32,97 +24,45 @@ register_schema_models, ) from controllers.console import console_ns -from controllers.console.app.error import ( - ConversationCompletedError, - DraftWorkflowNotExist, - DraftWorkflowNotSync, -) -from controllers.console.app.permission_keys import get_app_permission_keys -from controllers.console.app.wraps import get_app_model, with_session -from controllers.console.wraps import ( - RBACPermission, - account_initialization_required, - edit_permission_required, - model_validate, - rbac_permission_required, - setup_required, - with_current_tenant_id, - with_current_user, -) +from controllers.console.app.error import ConversationCompletedError, DraftWorkflowNotSync +from controllers.console.app.workflow_admission import console_workflow_admission +from controllers.console.flask_admission import console_account_admission +from controllers.console.wraps import RBACPermission, model_validate from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError -from core.app.app_config.features.file_upload.manager import FileUploadConfigManager -from core.app.apps.base_app_queue_manager import AppQueueManager -from core.app.entities.app_invoke_entities import InvokeFrom -from core.app.file_access import DatabaseFileAccessController -from core.db.session_factory import session_factory -from core.helper import encrypter from core.helper.trace_id_helper import get_external_trace_id -from core.plugin.impl.exc import PluginInvokeError -from core.trigger.constants import TRIGGER_SCHEDULE_NODE_TYPE -from core.trigger.debug.event_selectors import ( - TriggerDebugEvent, - TriggerDebugEventPoller, - create_event_poller, - select_trigger_debug_events, -) -from core.workflow.llm_environment_variable import ( - LLM_ENVIRONMENT_VARIABLE_VALUE_TYPE, - LLMEnvironmentVariable, -) -from extensions.ext_database import db -from extensions.ext_redis import redis_client -from factories import file_factory, variable_factory +from extensions.ext_application_services import application_services from fields.base import ResponseModel from fields.conversation_variable_fields import WorkflowConversationVariableResponse -from fields.member_fields import SimpleAccount -from fields.workflow_run_fields import ( - WorkflowRunNodeExecutionResponse, - node_execution_response_source, +from fields.workflow_fields import ( + DefaultBlockConfigResponse, + DefaultBlockConfigsResponse, + PipelineVariableResponse, + WorkflowEnvironmentVariableResponse, + WorkflowPaginationResponse, + WorkflowPublishResponse, + WorkflowResponse, + WorkflowRestoreResponse, ) -from graphon.enums import NodeType -from graphon.file import File -from graphon.file import helpers as file_helpers -from graphon.graph_engine.manager import GraphEngineManager +from fields.workflow_run_fields import WorkflowRunNodeExecutionResponse from graphon.model_runtime.utils.encoders import jsonable_encoder -from graphon.variables import SecretVariable, SegmentType, VariableBase from graphon.variables.exc import VariableError from libs import helper -from libs.datetime_utils import naive_utc_now -from libs.helper import TimestampField, dump_response, to_timestamp, uuid_value -from libs.login import login_required -from models import Account, App +from libs.helper import TimestampField, dump_response, uuid_value +from machinery.context import RequestContext from models.model import AppMode -from models.workflow import Workflow -from repositories.workflow_collaboration_repository import WORKFLOW_ONLINE_USERS_PREFIX -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService -from services.app.access import resolve_app_access_filter -from services.app_generate_service import AppGenerateService from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError -from services.errors.llm import InvokeRateLimitError -from services.workflow_ref_service import WorkflowRefService -from services.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError, WorkflowService -from services.workflow_variable_reference_validator import ( - format_variable_reference_errors, - validate_variable_references, +from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError +from services.workflow.contracts import ( + RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE, + DraftSyncCommand, + WorkflowOwner, + WorkflowTriggerError, ) logger = logging.getLogger(__name__) - -_file_access_controller = DatabaseFileAccessController() LISTENING_RETRY_IN = 2000 -RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE = "source workflow must be published" -MAX_WORKFLOW_ONLINE_USERS_REQUEST_IDS = 1000 -WORKFLOW_ONLINE_USERS_REDIS_BATCH_SIZE = 50 -ENVIRONMENT_VARIABLE_SUPPORTED_TYPES = (SegmentType.STRING, SegmentType.NUMBER, SegmentType.SECRET) - - -class EnvironmentVariableResponseDict(TypedDict): - value_type: str - id: NotRequired[str] - name: NotRequired[str] - value: NotRequired[Any] - description: NotRequired[str | None] +from services.errors.llm import InvokeRateLimitError class SyncEnvironmentVariablePatchPayload(BaseModel): @@ -285,108 +225,6 @@ def normalize_app_ids(cls, app_ids: list[str]) -> list[str]: return list(dict.fromkeys(app_id.strip() for app_id in app_ids if app_id.strip())) -class PipelineVariableResponse(ResponseModel): - label: str - variable: str - type: str - belong_to_node_id: str - max_length: int | None = None - required: bool - unit: str | None = None - default_value: Any = Field(default=None) - options: list[str] | None = None - placeholder: str | None = None - tooltips: str | None = None - allowed_file_types: list[str] | None = None - allowed_file_extensions: list[str] | None = Field( - default=None, validation_alias=AliasChoices("allowed_file_extensions", "allow_file_extension") - ) - allowed_file_upload_methods: list[str] | None = Field( - default=None, validation_alias=AliasChoices("allowed_file_upload_methods", "allow_file_upload_methods") - ) - - -class WorkflowEnvironmentVariableResponse(ResponseModel): - value_type: str - id: str - name: str - value: Any - description: str - - -class WorkflowResponse(ResponseModel): - id: str - graph: dict[str, Any] = Field( - validation_alias=AliasChoices("graph_dict", "graph"), - ) - features: dict[str, Any] = Field( - validation_alias=AliasChoices("features_dict", "features"), - ) - hash: str = Field(validation_alias=AliasChoices("unique_hash", "hash")) - version: str - # NULL for drafts and for versions published before numbering was introduced; those - # render as "Untitled Version" instead of `#N`. Never 0, so clients must test for null. - version_number: int | None = None - marked_name: str - marked_comment: str - created_by: SimpleAccount | None = Field( - default=None, validation_alias=AliasChoices("created_by_account", "created_by") - ) - created_at: int - updated_by: SimpleAccount | None = Field( - default=None, validation_alias=AliasChoices("updated_by_account", "updated_by") - ) - updated_at: int - tool_published: bool - environment_variables: list[WorkflowEnvironmentVariableResponse] - conversation_variables: list[WorkflowConversationVariableResponse] - rag_pipeline_variables: list[PipelineVariableResponse] - - @field_validator("created_at", "updated_at", mode="before") - @classmethod - def _normalize_timestamp(cls, value: datetime | int | None) -> int: - timestamp = to_timestamp(value) - if timestamp is None: - raise ValueError("timestamp is required") - return timestamp - - @field_validator("environment_variables", mode="before") - @classmethod - def _serialize_environment_variables(cls, value: Any) -> list[Any]: - if value is None: - return [] - - return [_serialize_environment_variable(item) for item in value] - - -class WorkflowResponseSource: - def __init__(self, workflow: Workflow, *, session: Session) -> None: - self._workflow = workflow - self._session = session - - def __getattr__(self, name: str) -> object: - return getattr(self._workflow, name) # guard-ignore: no-new-getattr -- delegates model fields - - @property - def created_by_account(self) -> Account | None: - return self._workflow.get_created_by_account(session=self._session) - - @property - def updated_by_account(self) -> Account | None: - return self._workflow.get_updated_by_account(session=self._session) - - @property - def tool_published(self) -> bool: - return self._workflow.get_tool_published(session=self._session) - - -class WorkflowPaginationResponse(ResponseModel): - items: list[WorkflowResponse] - page: int - limit: int - has_more: bool - - class WorkflowOnlineUser(ResponseModel): user_id: str username: str @@ -402,35 +240,12 @@ class WorkflowOnlineUsersResponse(ResponseModel): data: list[WorkflowOnlineUsersByApp] -class WorkflowPublishResponse(ResponseModel): - result: str - created_at: int - warning: str | None = Field( - default=None, - description="Advisory warning for variable references that can read a skipped branch. Publish still succeeds.", - ) - - class SyncDraftWorkflowResponse(ResponseModel): result: str hash: str updated_at: int -class WorkflowRestoreResponse(ResponseModel): - result: str - hash: str - updated_at: int - - -class DefaultBlockConfigsResponse(RootModel[list[dict[str, Any]]]): - root: list[dict[str, Any]] - - -class DefaultBlockConfigResponse(RootModel[dict[str, Any]]): - root: dict[str, Any] - - class HumanInputFormPreviewResponse(ResponseModel): form_id: str node_id: str @@ -487,7 +302,11 @@ class DraftWorkflowTriggerRunAllPayload(BaseModel): DraftWorkflowTriggerRunPayload, DraftWorkflowTriggerRunAllPayload, ) + + register_response_schema_model(console_ns, WorkflowRunNodeExecutionResponse) + + register_response_schema_models( console_ns, WorkflowConversationVariableResponse, @@ -512,71 +331,6 @@ class DraftWorkflowTriggerRunAllPayload(BaseModel): ) -# TODO(QuantumGhost): Refactor existing node run API to handle file parameter parsing -# at the controller level rather than in the workflow logic. This would improve separation -# of concerns and make the code more maintainable. -def _parse_file(workflow: Workflow, files: list[dict] | None = None) -> Sequence[File]: - files = files or [] - - file_extra_config = FileUploadConfigManager.convert(workflow.features_dict, is_vision=False) - file_objs: Sequence[File] = [] - if file_extra_config is None: - return file_objs - file_objs = file_factory.build_from_mappings( - mappings=files, - tenant_id=workflow.tenant_id, - config=file_extra_config, - access_controller=_file_access_controller, - ) - return file_objs - - -def _serialize_environment_variable(value: Any) -> EnvironmentVariableResponseDict | Any: - match value: - case LLMEnvironmentVariable(): - return { - "id": value.id, - "name": value.name, - "value": value.value, - "value_type": LLM_ENVIRONMENT_VARIABLE_VALUE_TYPE, - "description": value.description, - } - - case SecretVariable(): - return { - "id": value.id, - "name": value.name, - "value": encrypter.full_mask_token(), - "value_type": value.value_type.value, - "description": value.description, - } - - case VariableBase(): - return { - "id": value.id, - "name": value.name, - "value": value.value, - "value_type": str(value.value_type.exposed_type()), - "description": value.description, - } - - case dict(): - value_type_str = value.get("value_type") - if not isinstance(value_type_str, str): - raise TypeError( - f"unexpected type for value_type field, value={value_type_str}, type={type(value_type_str)}" - ) - if value_type_str == LLM_ENVIRONMENT_VARIABLE_VALUE_TYPE: - return value - value_type = SegmentType(value_type_str).exposed_type() - if value_type not in ENVIRONMENT_VARIABLE_SUPPORTED_TYPES: - raise ValueError(f"Unsupported environment variable value type: {value_type}") - return value - - case _: - return value - - @console_ns.route("/apps//workflows/draft") class DraftWorkflowApi(Resource): @console_ns.doc("get_draft_workflow") @@ -588,41 +342,12 @@ class DraftWorkflowApi(Resource): console_ns.models[WorkflowResponse.__name__], ) @console_ns.response(404, "Draft workflow not found") - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - @with_session(write=False) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - def get(self, session: Session, app_model: App): - """ - Get draft workflow - """ - # fetch draft workflow by app_model - workflow_service = WorkflowService() - workflow = workflow_service.get_draft_workflow(app_model=app_model, session=session) - - if not workflow: - raise DraftWorkflowNotExist() - - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - # Return workflow with response-only Agent node job projection so the - # front-end can treat draft graph node data as the editing source. - response = WorkflowResponse.model_validate( - WorkflowResponseSource(workflow, session=session), from_attributes=True - ).model_dump(mode="json") - response["graph"] = WorkflowAgentPublishService.project_draft_bindings_to_graph( - session=session, - draft_workflow=workflow, + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) + def get(self, request_context: RequestContext, app_id: UUID): + return dump_response( + WorkflowResponse, application_services().console_workflows.draft(request_context, str(app_id)) ) - return response - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) @console_ns.doc("sync_draft_workflow") @console_ns.doc(description="Sync draft workflow configuration") @console_ns.expect(console_ns.models[SyncDraftWorkflowPayload.__name__]) @@ -633,15 +358,9 @@ def get(self, session: Session, app_model: App): ) @console_ns.response(400, "Invalid workflow configuration") @console_ns.response(403, "Permission denied") - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - def post(self, current_user: Account, app_model: App): - """ - Sync draft workflow - """ + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) + def post(self, request_context: RequestContext, app_id: UUID): content_type = request.headers.get("Content-Type", "") - if "application/json" in content_type: payload_data = request.get_json(silent=True) if not isinstance(payload_data, dict): @@ -653,51 +372,32 @@ def post(self, current_user: Account, app_model: App): except (ValueError, ValidationError): return {"message": "Invalid JSON data"}, 400 else: - abort(415) - workflow_service = WorkflowService() - + raise UnsupportedMediaTypeError() + patch = args_model.environment_variable_patch try: - environment_variable_patch = args_model.environment_variable_patch - environment_variable_upserts: list[VariableBase] | None = None - deleted_environment_variable_ids: list[str] = [] - if environment_variable_patch is not None: - environment_variable_upsert_mappings = Workflow.normalize_environment_variable_mappings( - environment_variable_patch.environment_variables, - ) - environment_variable_upserts = [ - variable_factory.build_environment_variable_from_mapping(obj) - for obj in environment_variable_upsert_mappings - ] - deleted_environment_variable_ids = environment_variable_patch.deleted_environment_variable_ids - conversation_variables = [ - variable_factory.build_conversation_variable_from_mapping(obj) - for obj in args_model.conversation_variables - ] - workflow = workflow_service.sync_draft_workflow( - app_model=app_model, - graph=args_model.graph, - features=args_model.features, - unique_hash=args_model.hash, - account=current_user, - environment_variables=[], - conversation_variables=conversation_variables, - session=db.session(), - environment_variable_upserts=environment_variable_upserts, - deleted_environment_variable_ids=deleted_environment_variable_ids, - preserve_environment_variables=True, - graph_only=args_model.is_collaborative, + result = application_services().console_workflows.sync( + request_context, + str(app_id), + DraftSyncCommand( + graph=args_model.graph, + features=args_model.features, + unique_hash=args_model.hash, + is_collaborative=args_model.is_collaborative, + environment_upserts=patch.environment_variables if patch is not None else None, + environment_deletions=patch.deleted_environment_variable_ids if patch is not None else [], + conversation_variables=args_model.conversation_variables, + ), ) - except WorkflowHashNotEqualError: - raise DraftWorkflowNotSync() - except VariableError as e: - raise InvalidArgumentError(description=str(e)) - + except WorkflowHashNotEqualError as error: + raise DraftWorkflowNotSync() from error + except VariableError as error: + raise InvalidArgumentError(description=str(error)) from error return dump_response( SyncDraftWorkflowResponse, { "result": "success", - "hash": workflow.unique_hash, - "updated_at": TimestampField().format(workflow.updated_at or workflow.created_at), + "hash": result.hash, + "updated_at": TimestampField().format(result.updated_at), }, ) @@ -711,47 +411,27 @@ class AdvancedChatDraftWorkflowRunApi(Resource): @console_ns.response(200, "Workflow run started successfully", console_ns.models[GeneratedAppResponse.__name__]) @console_ns.response(400, "Invalid request parameters") @console_ns.response(403, "Permission denied") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @with_current_user - @edit_permission_required - @with_session - @get_app_model(mode=[AppMode.ADVANCED_CHAT]) + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.ADVANCED_CHAT,)) @model_validate(AdvancedChatWorkflowRunPayload) - def post(self, payload: AdvancedChatWorkflowRunPayload, session: Session, current_user: Account, app_model: App): - """ - Run draft workflow - """ + def post(self, payload: AdvancedChatWorkflowRunPayload, request_context: RequestContext, app_id: UUID): args = payload.model_dump(exclude_none=True) - external_trace_id = get_external_trace_id(request) if external_trace_id: args["external_trace_id"] = external_trace_id - try: - response = AppGenerateService.generate( - session=session, - app_model=app_model, - user=current_user, - args=args, - invoke_from=InvokeFrom.DEBUGGER, - streaming=True, - ) - - return helper.compact_generate_response(response) - except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") - except services.errors.conversation.ConversationCompletedError: - raise ConversationCompletedError() - except InvokeRateLimitError as ex: - raise InvokeRateLimitHttpError(ex.description) - except ValueError as e: - raise e - except Exception: + result = application_services().console_workflows.generate(request_context, str(app_id), args) + return helper.compact_generate_response(result) + except services.errors.conversation.ConversationNotExistsError as error: + raise NotFoundError("Conversation Not Exists.") from error + except services.errors.conversation.ConversationCompletedError as error: + raise ConversationCompletedError() from error + except InvokeRateLimitError as error: + raise InvokeRateLimitHttpError(error.description) from error + except ValueError: + raise + except Exception as error: logger.exception("internal server error.") - raise InternalServerError() + raise InternalServerError() from error @console_ns.route("/apps//advanced-chat/workflows/draft/iteration/nodes//run") @@ -767,40 +447,23 @@ class AdvancedChatDraftRunIterationNodeApi(Resource): ) @console_ns.response(403, "Permission denied") @console_ns.response(404, "Node not found") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.ADVANCED_CHAT,)) @model_validate(IterationNodeRunPayload) - def post(self, payload: IterationNodeRunPayload, current_user: Account, app_model: App, node_id: str): - """ - Run draft workflow iteration node - """ - args = payload.model_dump(exclude_none=True) - + def post(self, payload: IterationNodeRunPayload, request_context: RequestContext, app_id: UUID, node_id: str): try: - response = AppGenerateService.generate_single_iteration( - app_model=app_model, - user=current_user, - node_id=node_id, - args=args, - session=db.session(), - streaming=True, + result = application_services().console_workflows.iteration( + request_context, str(app_id), node_id, payload.inputs ) - - return helper.compact_generate_response(response) - except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") - except services.errors.conversation.ConversationCompletedError: - raise ConversationCompletedError() - except ValueError as e: - raise e - except Exception: + return helper.compact_generate_response(result) + except services.errors.conversation.ConversationNotExistsError as error: + raise NotFoundError("Conversation Not Exists.") from error + except services.errors.conversation.ConversationCompletedError as error: + raise ConversationCompletedError() from error + except ValueError: + raise + except Exception as error: logger.exception("internal server error.") - raise InternalServerError() + raise InternalServerError() from error @console_ns.route("/apps//workflows/draft/iteration/nodes//run") @@ -816,40 +479,23 @@ class WorkflowDraftRunIterationNodeApi(Resource): ) @console_ns.response(403, "Permission denied") @console_ns.response(404, "Node not found") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) @model_validate(IterationNodeRunPayload) - def post(self, payload: IterationNodeRunPayload, current_user: Account, app_model: App, node_id: str): - """ - Run draft workflow iteration node - """ - args = payload.model_dump(exclude_none=True) - + def post(self, payload: IterationNodeRunPayload, request_context: RequestContext, app_id: UUID, node_id: str): try: - response = AppGenerateService.generate_single_iteration( - app_model=app_model, - user=current_user, - node_id=node_id, - args=args, - session=db.session(), - streaming=True, + result = application_services().console_workflows.iteration( + request_context, str(app_id), node_id, payload.inputs ) - - return helper.compact_generate_response(response) - except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") - except services.errors.conversation.ConversationCompletedError: - raise ConversationCompletedError() - except ValueError as e: - raise e - except Exception: + return helper.compact_generate_response(result) + except services.errors.conversation.ConversationNotExistsError as error: + raise NotFoundError("Conversation Not Exists.") from error + except services.errors.conversation.ConversationCompletedError as error: + raise ConversationCompletedError() from error + except ValueError: + raise + except Exception as error: logger.exception("internal server error.") - raise InternalServerError() + raise InternalServerError() from error @console_ns.route("/apps//advanced-chat/workflows/draft/loop/nodes//run") @@ -861,39 +507,23 @@ class AdvancedChatDraftRunLoopNodeApi(Resource): @console_ns.response(200, "Loop node run started successfully", console_ns.models[GeneratedAppResponse.__name__]) @console_ns.response(403, "Permission denied") @console_ns.response(404, "Node not found") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.ADVANCED_CHAT,)) @model_validate(LoopNodeRunPayload) - def post(self, args: LoopNodeRunPayload, current_user: Account, app_model: App, node_id: str): - """ - Run draft workflow loop node - """ - + def post(self, payload: LoopNodeRunPayload, request_context: RequestContext, app_id: UUID, node_id: str): try: - response = AppGenerateService.generate_single_loop( - app_model=app_model, - user=current_user, - node_id=node_id, - args=args, - session=db.session(), - streaming=True, + result = application_services().console_workflows.loop( + request_context, str(app_id), node_id, payload.inputs ) - - return helper.compact_generate_response(response) - except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") - except services.errors.conversation.ConversationCompletedError: - raise ConversationCompletedError() - except ValueError as e: - raise e - except Exception: + return helper.compact_generate_response(result) + except services.errors.conversation.ConversationNotExistsError as error: + raise NotFoundError("Conversation Not Exists.") from error + except services.errors.conversation.ConversationCompletedError as error: + raise ConversationCompletedError() from error + except ValueError: + raise + except Exception as error: logger.exception("internal server error.") - raise InternalServerError() + raise InternalServerError() from error @console_ns.route("/apps//workflows/draft/loop/nodes//run") @@ -909,39 +539,23 @@ class WorkflowDraftRunLoopNodeApi(Resource): ) @console_ns.response(403, "Permission denied") @console_ns.response(404, "Node not found") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) @model_validate(LoopNodeRunPayload) - def post(self, args: LoopNodeRunPayload, current_user: Account, app_model: App, node_id: str): - """ - Run draft workflow loop node - """ - + def post(self, payload: LoopNodeRunPayload, request_context: RequestContext, app_id: UUID, node_id: str): try: - response = AppGenerateService.generate_single_loop( - app_model=app_model, - user=current_user, - node_id=node_id, - args=args, - session=db.session(), - streaming=True, + result = application_services().console_workflows.loop( + request_context, str(app_id), node_id, payload.inputs ) - - return helper.compact_generate_response(response) - except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") - except services.errors.conversation.ConversationCompletedError: - raise ConversationCompletedError() - except ValueError as e: - raise e - except Exception: + return helper.compact_generate_response(result) + except services.errors.conversation.ConversationNotExistsError as error: + raise NotFoundError("Conversation Not Exists.") from error + except services.errors.conversation.ConversationCompletedError as error: + raise ConversationCompletedError() from error + except ValueError: + raise + except Exception as error: logger.exception("internal server error.") - raise InternalServerError() + raise InternalServerError() from error class HumanInputFormPreviewPayload(BaseModel): @@ -986,29 +600,12 @@ class AdvancedChatDraftHumanInputFormPreviewApi(Resource): @console_ns.doc(params={"app_id": "Application ID", "node_id": "Node ID"}) @console_ns.expect(console_ns.models[HumanInputFormPreviewPayload.__name__]) @console_ns.response(200, "Human input form preview", console_ns.models[HumanInputFormPreviewResponse.__name__]) - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.ADVANCED_CHAT]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT, modes=(AppMode.ADVANCED_CHAT,)) @model_validate(HumanInputFormPreviewPayload) - def post(self, args: HumanInputFormPreviewPayload, current_user: Account, app_model: App, node_id: str): - """ - Preview human input form content and placeholders - """ - inputs = args.inputs - - workflow_service = WorkflowService() - preview = workflow_service.get_human_input_form_preview( - app_model=app_model, - account=current_user, - node_id=node_id, - inputs=inputs, - session=db.session(), + def post(self, args: HumanInputFormPreviewPayload, request_context: RequestContext, app_id: UUID, node_id: str): + return jsonable_encoder( + application_services().human_input_debug.preview_form(request_context, str(app_id), node_id, args.inputs) ) - return jsonable_encoder(preview) @console_ns.route("/apps//advanced-chat/workflows/draft/human-input/nodes//form/run") @@ -1022,29 +619,19 @@ class AdvancedChatDraftHumanInputFormRunApi(Resource): "Human input form submission result", console_ns.models[HumanInputFormSubmitResponse.__name__], ) - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.ADVANCED_CHAT,)) @model_validate(HumanInputFormSubmitPayload) - def post(self, args: HumanInputFormSubmitPayload, current_user: Account, app_model: App, node_id: str): - """ - Submit human input form preview - """ - workflow_service = WorkflowService() - result = workflow_service.submit_human_input_form_preview( - app_model=app_model, - account=current_user, - node_id=node_id, - form_inputs=args.form_inputs, - inputs=args.inputs, - action=args.action, - session=db.session(), + def post(self, args: HumanInputFormSubmitPayload, request_context: RequestContext, app_id: UUID, node_id: str): + return jsonable_encoder( + application_services().human_input_debug.submit_form( + request_context, + str(app_id), + node_id, + inputs=args.inputs, + form_inputs=args.form_inputs, + action=args.action, + ) ) - return jsonable_encoder(result) @console_ns.route("/apps//workflows/draft/human-input/nodes//form/preview") @@ -1054,29 +641,12 @@ class WorkflowDraftHumanInputFormPreviewApi(Resource): @console_ns.doc(params={"app_id": "Application ID", "node_id": "Node ID"}) @console_ns.expect(console_ns.models[HumanInputFormPreviewPayload.__name__]) @console_ns.response(200, "Human input form preview", console_ns.models[HumanInputFormPreviewResponse.__name__]) - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT, modes=(AppMode.WORKFLOW,)) @model_validate(HumanInputFormPreviewPayload) - def post(self, args: HumanInputFormPreviewPayload, current_user: Account, app_model: App, node_id: str): - """ - Preview human input form content and placeholders - """ - inputs = args.inputs - - workflow_service = WorkflowService() - preview = workflow_service.get_human_input_form_preview( - app_model=app_model, - account=current_user, - node_id=node_id, - inputs=inputs, - session=db.session(), + def post(self, args: HumanInputFormPreviewPayload, request_context: RequestContext, app_id: UUID, node_id: str): + return jsonable_encoder( + application_services().human_input_debug.preview_form(request_context, str(app_id), node_id, args.inputs) ) - return jsonable_encoder(preview) @console_ns.route("/apps//workflows/draft/human-input/nodes//form/run") @@ -1090,29 +660,19 @@ class WorkflowDraftHumanInputFormRunApi(Resource): "Human input form submission result", console_ns.models[HumanInputFormSubmitResponse.__name__], ) - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) @model_validate(HumanInputFormSubmitPayload) - def post(self, args: HumanInputFormSubmitPayload, current_user: Account, app_model: App, node_id: str): - """ - Submit human input form preview - """ - workflow_service = WorkflowService() - result = workflow_service.submit_human_input_form_preview( - app_model=app_model, - account=current_user, - node_id=node_id, - form_inputs=args.form_inputs, - inputs=args.inputs, - action=args.action, - session=db.session(), + def post(self, args: HumanInputFormSubmitPayload, request_context: RequestContext, app_id: UUID, node_id: str): + return jsonable_encoder( + application_services().human_input_debug.submit_form( + request_context, + str(app_id), + node_id, + inputs=args.inputs, + form_inputs=args.form_inputs, + action=args.action, + ) ) - return jsonable_encoder(result) @console_ns.route("/apps//workflows/draft/human-input/nodes//delivery-test") @@ -1122,28 +682,17 @@ class WorkflowDraftHumanInputDeliveryTestApi(Resource): @console_ns.doc(params={"app_id": "Application ID", "node_id": "Node ID"}) @console_ns.expect(console_ns.models[HumanInputDeliveryTestPayload.__name__]) @console_ns.response(200, "Human input delivery test result", console_ns.models[EmptyObjectResponse.__name__]) - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN) @model_validate(HumanInputDeliveryTestPayload) - def post(self, args: HumanInputDeliveryTestPayload, current_user: Account, app_model: App, node_id: str): - """ - Test human input delivery - """ - workflow_service = WorkflowService() - workflow_service.test_human_input_delivery( - app_model=app_model, - account=current_user, - node_id=node_id, - delivery_method_id=args.delivery_method_id, + def post(self, args: HumanInputDeliveryTestPayload, request_context: RequestContext, app_id: UUID, node_id: str): + application_services().human_input_debug.test_delivery( + request_context, + str(app_id), + node_id, inputs=args.inputs, - session=db.session(), + delivery_method_id=args.delivery_method_id, ) - return jsonable_encoder({}) + return {} @console_ns.route("/apps//workflows/draft/run") @@ -1158,38 +707,18 @@ class DraftWorkflowRunApi(Resource): console_ns.models[GeneratedAppResponse.__name__], ) @console_ns.response(403, "Permission denied") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @with_current_user - @edit_permission_required - @with_session - @get_app_model(mode=[AppMode.WORKFLOW]) + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) @model_validate(DraftWorkflowRunPayload) - def post(self, payload: DraftWorkflowRunPayload, session: Session, current_user: Account, app_model: App): - """ - Run draft workflow - """ + def post(self, payload: DraftWorkflowRunPayload, request_context: RequestContext, app_id: UUID): args = payload.model_dump(exclude_none=True) - external_trace_id = get_external_trace_id(request) if external_trace_id: args["external_trace_id"] = external_trace_id - try: - response = AppGenerateService.generate( - session=session, - app_model=app_model, - user=current_user, - args=args, - invoke_from=InvokeFrom.DEBUGGER, - streaming=True, - ) - - return helper.compact_generate_response(response) - except InvokeRateLimitError as ex: - raise InvokeRateLimitHttpError(ex.description) + result = application_services().console_workflows.generate(request_context, str(app_id), args) + return helper.compact_generate_response(result) + except InvokeRateLimitError as error: + raise InvokeRateLimitHttpError(error.description) from error @console_ns.route("/apps//workflow-runs/tasks//stop") @@ -1200,23 +729,9 @@ class WorkflowTaskStopApi(Resource): @console_ns.response(200, "Task stopped successfully", console_ns.models[SimpleResultResponse.__name__]) @console_ns.response(404, "Task not found") @console_ns.response(403, "Permission denied") - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - def post(self, app_model: App, task_id: str): - """ - Stop workflow task - """ - # Stop using both mechanisms for backward compatibility - # Legacy stop flag mechanism (without user check) - AppQueueManager.set_stop_flag_no_user_check(task_id) - - # New graph engine command channel mechanism - GraphEngineManager(redis_client).send_stop_command(task_id) - + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN) + def post(self, request_context: RequestContext, app_id: UUID, task_id: str): + application_services().console_workflows.stop(task_id) return {"result": "success"} @@ -1233,61 +748,19 @@ class DraftWorkflowNodeRunApi(Resource): ) @console_ns.response(403, "Permission denied") @console_ns.response(404, "Node not found") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN) @model_validate(DraftWorkflowNodeRunPayload) - def post(self, payload: DraftWorkflowNodeRunPayload, current_user: Account, app_model: App, node_id: str): - """ - Run draft workflow node - """ - args = payload.model_dump(exclude_none=True) - - user_inputs = payload.inputs - if user_inputs is None: - raise ValueError("missing inputs") - - workflow_srv = WorkflowService() - # fetch draft workflow by app_model - draft_workflow = workflow_srv.get_draft_workflow(app_model=app_model, session=db.session()) - if not draft_workflow: - raise ValueError("Workflow not initialized") - files = _parse_file(draft_workflow, args.get("files")) - workflow_service = WorkflowService() - - workflow_node_execution = workflow_service.run_draft_workflow_node( - app_model=app_model, - draft_workflow=draft_workflow, - node_id=node_id, - user_inputs=user_inputs, - account=current_user, - query=args.get("query", ""), - files=files, + def post(self, payload: DraftWorkflowNodeRunPayload, request_context: RequestContext, app_id: UUID, node_id: str): + return dump_response( + WorkflowRunNodeExecutionResponse, + application_services().console_workflows.run_node( + request_context, + str(app_id), + node_id, + payload.model_dump(exclude_none=True), + ), ) - return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(workflow_node_execution, session=db.session()), from_attributes=True - ).model_dump(mode="json") - - -def _advisory_variable_reference_warning(graph_text: str | None) -> str | None: - """Return a non-blocking publish warning. A checker failure must not fail publish.""" - if not graph_text: - return None - try: - graph = json.loads(graph_text) - if not isinstance(graph, dict): - return None - issues = validate_variable_references(graph) - return format_variable_reference_errors(issues) if issues else None - except Exception: - logger.warning("Skipped advisory variable reference check", exc_info=True) - return None - @console_ns.route("/apps//workflows/publish") class PublishedWorkflowApi(Resource): @@ -1299,66 +772,25 @@ class PublishedWorkflowApi(Resource): "Published workflow retrieved successfully, or null if not found", console_ns.models[WorkflowResponse.__name__], ) - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - def get(self, app_model: App): - """ - Get published workflow - """ - # fetch published workflow by app_model - workflow_service = WorkflowService() - session = db.session() - workflow = workflow_service.get_published_workflow(app_model=app_model, session=session) - - # return workflow, if not found, return None - if workflow is None: - return None - - return dump_response(WorkflowResponse, WorkflowResponseSource(workflow, session=session)) + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) + def get(self, request_context: RequestContext, app_id: UUID): + result = application_services().console_workflows.published(request_context, str(app_id)) + return dump_response(WorkflowResponse, result) if result is not None else None @console_ns.expect(console_ns.models[PublishWorkflowPayload.__name__]) @console_ns.response(200, "Workflow published successfully", console_ns.models[WorkflowPublishResponse.__name__]) - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_RELEASE_AND_VERSION, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_RELEASE_AND_VERSION) @model_validate(PublishWorkflowPayload) - def post(self, args: PublishWorkflowPayload, current_user: Account, app_model: App): - """ - Publish workflow - """ - - workflow_service = WorkflowService() - with sessionmaker(db.engine).begin() as session: - workflow = workflow_service.publish_workflow( - session=session, - app_model=app_model, - account=current_user, - marked_name=args.marked_name or "", - marked_comment=args.marked_comment or "", - ) - - # Update app_model within the same session to ensure atomicity - app_model_in_session = session.get(App, app_model.id) - if app_model_in_session: - app_model_in_session.workflow_id = workflow.id - app_model_in_session.updated_by = current_user.id - app_model_in_session.updated_at = naive_utc_now() - - workflow_created_at = TimestampField().format(workflow.created_at) - graph_text = workflow.graph - - warning = _advisory_variable_reference_warning(graph_text) + def post(self, args: PublishWorkflowPayload, request_context: RequestContext, app_id: UUID): + publication, warning = application_services().console_workflows.publish( + request_context, + str(app_id), + marked_name=args.marked_name or "", + marked_comment=args.marked_comment or "", + ) payload: dict[str, object] = { "result": "success", - "created_at": workflow_created_at, + "created_at": TimestampField().format(publication.created_at), } if warning: payload["warning"] = warning @@ -1375,19 +807,9 @@ class DefaultBlockConfigsApi(Resource): "Default block configurations retrieved successfully", console_ns.models[DefaultBlockConfigsResponse.__name__], ) - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - def get(self, app_model: App): - """ - Get default block config - """ - # Get default block configs - workflow_service = WorkflowService() - return workflow_service.get_default_block_configs() + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) + def get(self, request_context: RequestContext, app_id: UUID): + return application_services().console_workflows.default_blocks() @console_ns.route("/apps//workflows/default-workflow-block-configs/") @@ -1402,28 +824,16 @@ class DefaultBlockConfigApi(Resource): ) @console_ns.response(404, "Block type not found") @console_ns.doc(params=query_params_from_model(DefaultBlockConfigQuery)) - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) @model_validate(DefaultBlockConfigQuery) - def get(self, args: DefaultBlockConfigQuery, app_model: App, block_type: str): - """ - Get default block config - """ - + def get(self, args: DefaultBlockConfigQuery, request_context: RequestContext, app_id: UUID, block_type: str): filters = None if args.q: try: filters = json.loads(args.q) - except json.JSONDecodeError: - raise ValueError("Invalid filters") - - # Get default block configs - workflow_service = WorkflowService() - return workflow_service.get_default_block_config(node_type=block_type, filters=filters) + except json.JSONDecodeError as error: + raise ValueError("Invalid filters") from error + return application_services().console_workflows.default_block(block_type, filters) @console_ns.route("/apps//convert-to-workflow") @@ -1439,40 +849,16 @@ class ConvertToWorkflowApi(Resource): ) @console_ns.response(400, "Application cannot be converted") @console_ns.response(403, "Permission denied") - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.CHAT, AppMode.COMPLETION]) - @with_current_user - @with_current_tenant_id - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_EDIT, PlainApp())) + @console_workflow_admission(permission=RBACPermission.APP_EDIT, modes=(AppMode.CHAT, AppMode.COMPLETION)) @model_validate(ConvertToWorkflowPayload) - def post(self, payload: ConvertToWorkflowPayload, current_tenant_id: str, current_user: Account, app_model: App): - """ - Convert basic mode of chatbot app to workflow mode - Convert expert mode of chatbot app to workflow mode - Convert Completion App to Workflow App - """ - args = payload.model_dump(exclude_none=True) - - # convert to workflow mode - workflow_service = WorkflowService() - new_app_model = workflow_service.convert_to_workflow( - app_model=app_model, account=current_user, args=args, session=db.session() + def post(self, payload: ConvertToWorkflowPayload, request_context: RequestContext, app_id: UUID): + return application_services().console_workflows.convert( + request_context, str(app_id), payload.model_dump(exclude_none=True) ) - # return app id - return { - "new_app_id": new_app_model.id, - "permission_keys": get_app_permission_keys(str(current_tenant_id), current_user.id, str(new_app_model.id)), - } - @console_ns.route("/apps//workflows/draft/features") class WorkflowFeaturesApi(Resource): - """Update draft workflow features.""" - @console_ns.expect(console_ns.models[WorkflowFeaturesPayload.__name__]) @console_ns.doc("update_workflow_features") @console_ns.doc(description="Update draft workflow features") @@ -1482,22 +868,14 @@ class WorkflowFeaturesApi(Resource): "Workflow features updated successfully", console_ns.models[SimpleResultResponse.__name__], ) - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) @model_validate(WorkflowFeaturesPayload) - def post(self, args: WorkflowFeaturesPayload, current_user: Account, app_model: App): - features = args.features.model_dump(mode="json", exclude_unset=True) - - workflow_service = WorkflowService() - workflow_service.update_draft_workflow_features( - app_model=app_model, features=features, account=current_user, session=db.session() + def post(self, args: WorkflowFeaturesPayload, request_context: RequestContext, app_id: UUID): + application_services().console_workflows.update_features( + request_context, + str(app_id), + args.features.model_dump(mode="json", exclude_unset=True), ) - return {"result": "success"} @@ -1512,46 +890,20 @@ class PublishedAllWorkflowApi(Resource): "Published workflows retrieved successfully", console_ns.models[WorkflowPaginationResponse.__name__], ) - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - @with_current_user - @edit_permission_required - @with_session(write=False) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) @model_validate(WorkflowListQuery) - def get(self, args: WorkflowListQuery, session: Session, current_user: Account, app_model: App): - """ - Get published workflows - """ - - page = args.page - limit = args.limit - user_id = args.user_id - named_only = args.named_only - - if user_id: - if user_id != current_user.id: - raise Forbidden() - - workflow_service = WorkflowService() - workflows, has_more = workflow_service.get_all_published_workflow( - session=session, - app_model=app_model, - page=page, - limit=limit, - user_id=user_id, - named_only=named_only, + def get(self, args: WorkflowListQuery, request_context: RequestContext, app_id: UUID): + items, has_more = application_services().console_workflows.versions( + request_context, + str(app_id), + page=args.page, + limit=args.limit, + user_id=args.user_id, + named_only=args.named_only, + ) + return dump_response( + WorkflowPaginationResponse, {"items": items, "page": args.page, "limit": args.limit, "has_more": has_more} ) - return WorkflowPaginationResponse.model_validate( - { - "items": [WorkflowResponseSource(workflow, session=session) for workflow in workflows], - "page": page, - "limit": limit, - "has_more": has_more, - } - ).model_dump(mode="json") @console_ns.route("/apps//workflows//restore") @@ -1562,35 +914,17 @@ class DraftWorkflowRestoreApi(Resource): @console_ns.response(200, "Workflow restored successfully", console_ns.models[WorkflowRestoreResponse.__name__]) @console_ns.response(400, "Source workflow must be published") @console_ns.response(404, "Workflow not found") - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_RELEASE_AND_VERSION, PlainApp())) - def post(self, current_user: Account, app_model: App, workflow_id: str): - workflow_service = WorkflowService() - + @console_workflow_admission(permission=RBACPermission.APP_RELEASE_AND_VERSION) + def post(self, request_context: RequestContext, app_id: UUID, workflow_id: str): try: - workflow = workflow_service.restore_published_workflow_to_draft( - app_model=app_model, - workflow_id=workflow_id, - account=current_user, - session=db.session(), - ) - except IsDraftWorkflowError as exc: - raise BadRequest(RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE) from exc - except WorkflowNotFoundError as exc: - raise NotFound(str(exc)) from exc - except ValueError as exc: - raise BadRequest(str(exc)) from exc - - return { - "result": "success", - "hash": workflow.unique_hash, - "updated_at": TimestampField().format(workflow.updated_at or workflow.created_at), - } + result = application_services().console_workflows.restore(request_context, str(app_id), workflow_id) + except IsDraftWorkflowError as error: + raise InvalidRequestError(RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE) from error + except WorkflowNotFoundError as error: + raise NotFoundError(str(error)) from error + except ValueError as error: + raise InvalidRequestError(str(error)) from error + return {"result": "success", "hash": result.hash, "updated_at": TimestampField().format(result.updated_at)} @console_ns.route("/apps//workflows/") @@ -1602,82 +936,24 @@ class WorkflowByIdApi(Resource): @console_ns.response(200, "Workflow updated successfully", console_ns.models[WorkflowResponse.__name__]) @console_ns.response(404, "Workflow not found") @console_ns.response(403, "Permission denied") - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_EDIT, PlainApp())) + @console_workflow_admission(permission=RBACPermission.APP_EDIT) @model_validate(WorkflowUpdatePayload) - def patch(self, args: WorkflowUpdatePayload, current_user: Account, app_model: App, workflow_id: str): - """ - Update workflow attributes - """ - - # Prepare update data - update_data = {} - if args.marked_name is not None: - update_data["marked_name"] = args.marked_name - if args.marked_comment is not None: - update_data["marked_comment"] = args.marked_comment - - if not update_data: + def patch(self, args: WorkflowUpdatePayload, request_context: RequestContext, app_id: UUID, workflow_id: str): + changes = args.model_dump(exclude_none=True) + if not changes: return {"message": "No valid fields to update"}, 400 + return dump_response( + WorkflowResponse, + application_services().console_workflows.update(request_context, str(app_id), workflow_id, changes), + ) - workflow_service = WorkflowService() - workflow_ref = WorkflowRefService.create_app_workflow_ref(app_model, workflow_id) - - # Create a session and manage the transaction - with sessionmaker(db.engine, expire_on_commit=False).begin() as session: - workflow = workflow_service.update_workflow( - session=session, - account_id=current_user.id, - data=update_data, - workflow_ref=workflow_ref, - ) - - if not workflow: - raise NotFound("Workflow not found") - - response = dump_response(WorkflowResponse, WorkflowResponseSource(workflow, session=session)) - - return response - - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_EDIT, PlainApp())) @console_ns.response(204, "Workflow deleted successfully") - def delete(self, current_user: Account, app_model: App, workflow_id: str): - """ - Delete workflow - """ - workflow_service = WorkflowService() - workflow_ref = WorkflowRefService.create_app_workflow_ref(app_model, workflow_id) - - # Create a session and manage the transaction - with sessionmaker(db.engine).begin() as session: - try: - retirement_candidates = workflow_service.delete_workflow( - session=session, - workflow_ref=workflow_ref, - ) - except WorkflowInUseError as e: - abort(400, description=str(e)) - except DraftWorkflowDeletionError as e: - abort(400, description=str(e)) - except ValueError as e: - raise NotFound(str(e)) - - WorkflowAgentRetirementService.retire_unowned( - tenant_id=app_model.tenant_id, - agent_ids=retirement_candidates, - account_id=current_user.id, - ) + @console_workflow_admission(permission=RBACPermission.APP_EDIT) + def delete(self, request_context: RequestContext, app_id: UUID, workflow_id: str): + try: + application_services().console_workflows.delete(request_context, WorkflowOwner(str(app_id)), workflow_id) + except (WorkflowInUseError, DraftWorkflowDeletionError) as error: + raise InvalidRequestError(str(error)) return None, 204 @@ -1693,35 +969,16 @@ class DraftWorkflowNodeLastRunApi(Resource): ) @console_ns.response(404, "Node last run not found") @console_ns.response(403, "Permission denied") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - def get(self, app_model: App, node_id: str): - srv = WorkflowService() - workflow = srv.get_draft_workflow(app_model, session=db.session()) - if not workflow: - raise NotFound("Workflow not found") - node_exec = srv.get_node_last_run( - app_model=app_model, - workflow=workflow, - node_id=node_id, + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT, require_editor=False) + def get(self, request_context: RequestContext, app_id: UUID, node_id: str): + return dump_response( + WorkflowRunNodeExecutionResponse, + application_services().console_workflows.last_run(request_context, str(app_id), node_id), ) - if node_exec is None: - raise NotFound("last run not found") - return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(node_exec, session=db.session()), from_attributes=True - ).model_dump(mode="json") @console_ns.route("/apps//workflows/draft/trigger/run") class DraftWorkflowTriggerRunApi(Resource): - """ - Full workflow debug - Polling API for trigger events - Path: /apps//workflows/draft/trigger/run - """ - @console_ns.doc("poll_draft_workflow_trigger_run") @console_ns.doc(description="Poll for trigger events and execute full workflow when event arrives") @console_ns.doc(params={"app_id": "Application ID"}) @@ -1733,64 +990,27 @@ class DraftWorkflowTriggerRunApi(Resource): ) @console_ns.response(403, "Permission denied") @console_ns.response(500, "Internal server error") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @with_current_user - @edit_permission_required - @with_session - @get_app_model(mode=[AppMode.WORKFLOW]) + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) @model_validate(DraftWorkflowTriggerRunPayload) - def post(self, args: DraftWorkflowTriggerRunPayload, session: Session, current_user: Account, app_model: App): - """ - Poll for trigger events and execute full workflow when event arrives - """ - node_id = args.node_id - workflow_service = WorkflowService() - draft_workflow = workflow_service.get_draft_workflow(app_model, session=session) - if not draft_workflow: - raise ValueError("Workflow not found") - - poller: TriggerDebugEventPoller = create_event_poller( - draft_workflow=draft_workflow, - tenant_id=app_model.tenant_id, - user_id=current_user.id, - app_id=app_model.id, - node_id=node_id, - ) - event: TriggerDebugEvent | None = None + def post(self, args: DraftWorkflowTriggerRunPayload, request_context: RequestContext, app_id: UUID): try: - event = poller.poll() - if not event: - return jsonable_encoder({"status": "waiting", "retry_in": LISTENING_RETRY_IN}) - return helper.compact_generate_response( - AppGenerateService.generate( - session=session, - app_model=app_model, - user=current_user, - args=event.workflow_args, - invoke_from=InvokeFrom.DEBUGGER, - streaming=True, - root_node_id=node_id, - ) + result = application_services().console_workflows.trigger( + request_context, str(app_id), [args.node_id], single_node=False, select_all=False ) - except InvokeRateLimitError as ex: - raise InvokeRateLimitHttpError(ex.description) - except PluginInvokeError as e: - return jsonable_encoder({"status": "error", "error": e.to_user_friendly_error()}), 400 - except Exception as e: - logger.exception("Error polling trigger debug event") - raise e + except WorkflowTriggerError as error: + payload = {"status": "error"} + if error.message is not None: + payload["error"] = error.message + return payload, 400 + except InvokeRateLimitError as error: + raise InvokeRateLimitHttpError(error.description) from error + if result is None: + return {"status": "waiting", "retry_in": LISTENING_RETRY_IN} + return helper.compact_generate_response(result) @console_ns.route("/apps//workflows/draft/nodes//trigger/run") class DraftWorkflowTriggerNodeApi(Resource): - """ - Single node debug - Polling API for trigger events - Path: /apps//workflows/draft/nodes//trigger/run - """ - @console_ns.doc("poll_draft_workflow_trigger_node") @console_ns.doc(description="Poll for trigger events and execute single node when event arrives") @console_ns.doc(params={"app_id": "Application ID", "node_id": "Node ID"}) @@ -1801,80 +1021,26 @@ class DraftWorkflowTriggerNodeApi(Resource): ) @console_ns.response(403, "Permission denied") @console_ns.response(500, "Internal server error") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required - def post(self, current_user: Account, app_model: App, node_id: str): - """ - Poll for trigger events and execute single node when event arrives - """ - - workflow_service = WorkflowService() - draft_workflow = workflow_service.get_draft_workflow(app_model, session=db.session()) - if not draft_workflow: - raise ValueError("Workflow not found") - - node_config = draft_workflow.get_node_config_by_id(node_id=node_id) - if not node_config: - raise ValueError("Node data not found for node %s", node_id) - node_type: NodeType = draft_workflow.get_node_type_from_node_config(node_config) - event: TriggerDebugEvent | None = None - # for schedule trigger, when run single node, just execute directly - if node_type == TRIGGER_SCHEDULE_NODE_TYPE: - event = TriggerDebugEvent( - workflow_args={}, - node_id=node_id, - ) - # for other trigger types, poll for the event - else: - try: - poller: TriggerDebugEventPoller = create_event_poller( - draft_workflow=draft_workflow, - tenant_id=app_model.tenant_id, - user_id=current_user.id, - app_id=app_model.id, - node_id=node_id, - ) - event = poller.poll() - except PluginInvokeError as e: - return jsonable_encoder({"status": "error", "error": e.to_user_friendly_error()}), 400 - except Exception as e: - logger.exception("Error polling trigger debug event") - raise e - if not event: - return jsonable_encoder({"status": "waiting", "retry_in": LISTENING_RETRY_IN}) - - raw_files = event.workflow_args.get("files") - files = _parse_file(draft_workflow, raw_files if isinstance(raw_files, list) else None) + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) + def post(self, request_context: RequestContext, app_id: UUID, node_id: str): try: - node_execution = workflow_service.run_draft_workflow_node( - app_model=app_model, - draft_workflow=draft_workflow, - node_id=node_id, - user_inputs=event.workflow_args.get("inputs") or {}, - account=current_user, - query="", - files=files, + result = application_services().console_workflows.trigger( + request_context, str(app_id), [node_id], single_node=True, select_all=False ) - return jsonable_encoder(node_execution) - except Exception as e: - logger.exception("Error running draft workflow trigger node") - return jsonable_encoder( - {"status": "error", "error": "An unexpected error occurred while running the node."} - ), 400 + except WorkflowTriggerError as error: + payload = {"status": "error"} + if error.message is not None: + payload["error"] = error.message + return payload, 400 + except InvokeRateLimitError as error: + raise InvokeRateLimitHttpError(error.description) from error + if result is None: + return {"status": "waiting", "retry_in": LISTENING_RETRY_IN} + return jsonable_encoder(result) @console_ns.route("/apps//workflows/draft/trigger/run-all") class DraftWorkflowTriggerRunAllApi(Resource): - """ - Full workflow debug - Polling API for trigger events - Path: /apps//workflows/draft/trigger/run-all - """ - @console_ns.doc("draft_workflow_trigger_run_all") @console_ns.doc(description="Full workflow debug when the start node is a trigger") @console_ns.doc(params={"app_id": "Application ID"}) @@ -1882,61 +1048,23 @@ class DraftWorkflowTriggerRunAllApi(Resource): @console_ns.response(200, "Workflow executed successfully", console_ns.models[GeneratedAppResponse.__name__]) @console_ns.response(403, "Permission denied") @console_ns.response(500, "Internal server error") - @setup_required - @login_required - @account_initialization_required - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @with_session - @get_app_model(mode=[AppMode.WORKFLOW]) + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) @model_validate(DraftWorkflowTriggerRunAllPayload) - def post(self, args: DraftWorkflowTriggerRunAllPayload, session: Session, current_user: Account, app_model: App): - """ - Full workflow debug when the start node is a trigger - """ - - node_ids = args.node_ids - workflow_service = WorkflowService() - draft_workflow = workflow_service.get_draft_workflow(app_model, session=session) - if not draft_workflow: - raise ValueError("Workflow not found") - + def post(self, args: DraftWorkflowTriggerRunAllPayload, request_context: RequestContext, app_id: UUID): try: - trigger_debug_event: TriggerDebugEvent | None = select_trigger_debug_events( - draft_workflow=draft_workflow, - app_model=app_model, - user_id=current_user.id, - node_ids=node_ids, + result = application_services().console_workflows.trigger( + request_context, str(app_id), args.node_ids, single_node=False, select_all=True ) - except PluginInvokeError as e: - return jsonable_encoder({"status": "error", "error": e.to_user_friendly_error()}), 400 - except Exception as e: - logger.exception("Error polling trigger debug event") - raise e - if trigger_debug_event is None: - return jsonable_encoder({"status": "waiting", "retry_in": LISTENING_RETRY_IN}) - - try: - response = AppGenerateService.generate( - session=session, - app_model=app_model, - user=current_user, - args=trigger_debug_event.workflow_args, - invoke_from=InvokeFrom.DEBUGGER, - streaming=True, - root_node_id=trigger_debug_event.node_id, - ) - return helper.compact_generate_response(response) - except InvokeRateLimitError as ex: - raise InvokeRateLimitHttpError(ex.description) - except Exception: - logger.exception("Error running draft workflow trigger run-all") - return jsonable_encoder( - { - "status": "error", - } - ), 400 + except WorkflowTriggerError as error: + payload = {"status": "error"} + if error.message is not None: + payload["error"] = error.message + return payload, 400 + except InvokeRateLimitError as error: + raise InvokeRateLimitHttpError(error.description) from error + if result is None: + return {"status": "waiting", "retry_in": LISTENING_RETRY_IN} + return helper.compact_generate_response(result) @console_ns.route("/apps/workflows/online-users") @@ -1949,85 +1077,11 @@ class WorkflowOnlineUsersApi(Resource): ) @console_ns.doc("get_workflow_online_users") @console_ns.doc(description="Get workflow online users") - @setup_required - @login_required - @account_initialization_required - @with_current_user - @with_current_tenant_id + @console_account_admission() @model_validate(WorkflowOnlineUsersPayload) - def post(self, args: WorkflowOnlineUsersPayload, current_tenant_id: str, current_user: Account): - app_ids = args.app_ids - if len(app_ids) > MAX_WORKFLOW_ONLINE_USERS_REQUEST_IDS: - raise BadRequest(f"Maximum {MAX_WORKFLOW_ONLINE_USERS_REQUEST_IDS} app_ids are allowed per request.") - - if not app_ids: - return {"data": []} - - access_filter = None - workflow_service = WorkflowService() - with session_factory.create_session() as session: - if dify_config.RBAC_ENABLED: - access_filter = resolve_app_access_filter(current_tenant_id, current_user.id, session=session) - app_maintainers = workflow_service.get_tenant_app_maintainers(app_ids, current_tenant_id, session=session) - - accessible_app_ids = set(app_maintainers) - if access_filter is not None: - accessible_app_ids = { - app_id - for app_id, maintainer in app_maintainers.items() - if access_filter.is_app_accessible(app_id, maintainer, current_user.id) - } - ordered_accessible_app_ids = [app_id for app_id in app_ids if app_id in accessible_app_ids] - - users_json_by_app_id: dict[str, Any] = {} - for start_index in range(0, len(ordered_accessible_app_ids), WORKFLOW_ONLINE_USERS_REDIS_BATCH_SIZE): - app_id_batch = ordered_accessible_app_ids[ - start_index : start_index + WORKFLOW_ONLINE_USERS_REDIS_BATCH_SIZE - ] - pipe = redis_client.pipeline(transaction=False) - for app_id in app_id_batch: - pipe.hgetall(f"{WORKFLOW_ONLINE_USERS_PREFIX}{app_id}") - - users_json_batch = pipe.execute() - for app_id, users_json in zip(app_id_batch, users_json_batch): - users_json_by_app_id[app_id] = users_json - - results = [] - for app_id in ordered_accessible_app_ids: - users_json = users_json_by_app_id.get(app_id, {}) - - users = [] - for _, user_info_json in users_json.items(): - try: - user_info = json.loads(user_info_json) - except Exception: - continue - - if not isinstance(user_info, dict): - continue - - user_id = user_info.get("user_id") - username = user_info.get("username") - if not isinstance(user_id, str) or not isinstance(username, str): - continue - - avatar = user_info.get("avatar") - if avatar is not None and not isinstance(avatar, str): - avatar = None - - if isinstance(avatar, str) and avatar and not avatar.startswith(("http://", "https://")): - try: - avatar = file_helpers.get_signed_file_url(avatar) - except Exception as exc: - logger.warning( - "Failed to sign workflow online user avatar; using original value. " - "app_id=%s avatar=%s error=%s", - app_id, - avatar, - exc, - ) - - users.append({"user_id": user_id, "username": username, "avatar": avatar}) - results.append({"app_id": app_id, "users": users}) - - return WorkflowOnlineUsersResponse.model_validate({"data": results}).model_dump(mode="json") + def post(self, args: WorkflowOnlineUsersPayload, request_context: RequestContext): + try: + data = application_services().console_workflows.online_users(request_context, args.app_ids) + except ValueError as error: + raise InvalidRequestError(str(error)) from error + return dump_response(WorkflowOnlineUsersResponse, {"data": data}) diff --git a/api/controllers/console/app/workflow_admission.py b/api/controllers/console/app/workflow_admission.py new file mode 100644 index 00000000000000..efa158729cfbaf --- /dev/null +++ b/api/controllers/console/app/workflow_admission.py @@ -0,0 +1,58 @@ +"""Console workflow admission and domain-to-HTTP error translation.""" + +from collections.abc import Callable +from functools import wraps +from typing import Concatenate + +from controllers.common.errors import AccessDeniedError, NotFoundError +from controllers.common.rbac import PlainApp, RBACCheck +from controllers.console.app.error import AppNotFoundError, DraftWorkflowNotExist, DraftWorkflowNotSync +from controllers.console.flask_admission import console_account_admission +from controllers.console.wraps import RBACPermission +from extensions.ext_application_services import application_services +from machinery.context import RequestContext +from models.account import TenantAccountRole +from models.model import AppMode +from services.app.console_service import ConsoleAppNotFoundError +from services.errors.app import WorkflowHashNotEqualError, WorkflowNotFoundError +from services.errors.base import NoPermissionError +from services.workflow.contracts import DraftWorkflowMissingError + +_EDIT_ROLES = frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}) + + +def console_workflow_admission[T, **P, R]( + *, + permission: RBACPermission, + modes: tuple[AppMode, ...] = (AppMode.ADVANCED_CHAT, AppMode.WORKFLOW), + require_editor: bool = True, +): + def decorator(view: Callable[Concatenate[T, RequestContext, P], R]): + @console_account_admission( + allowed_roles=_EDIT_ROLES if require_editor else None, + rbac_checks=[RBACCheck(permission, PlainApp())], + ) + @wraps(view) + def admitted(self: T, context: RequestContext, /, *args: P.args, **kwargs: P.kwargs) -> R: + app_id = kwargs.get("app_id") + if app_id is None: + raise ValueError("missing app_id in path parameters") + try: + app = application_services().apps.console.get_reference(context, str(app_id)) + if app.mode not in modes: + raise AppNotFoundError(f"App mode is not in the supported list: {set(modes)}") + return view(self, context, *args, **kwargs) + except ConsoleAppNotFoundError as error: + raise AppNotFoundError() from error + except DraftWorkflowMissingError as error: + raise DraftWorkflowNotExist() from error + except WorkflowHashNotEqualError as error: + raise DraftWorkflowNotSync() from error + except WorkflowNotFoundError as error: + raise NotFoundError(str(error)) from error + except NoPermissionError as error: + raise AccessDeniedError() from error + + return admitted + + return decorator diff --git a/api/controllers/console/app/workflow_node_output_inspector.py b/api/controllers/console/app/workflow_node_output_inspector.py index ae16bbd3865619..a42e1bf764426c 100644 --- a/api/controllers/console/app/workflow_node_output_inspector.py +++ b/api/controllers/console/app/workflow_node_output_inspector.py @@ -41,6 +41,7 @@ rbac_permission_required, setup_required, ) +from extensions.ext_application_services import application_services from extensions.ext_database import db from libs.exception import BaseHTTPException from libs.login import login_required @@ -81,7 +82,7 @@ def _service() -> NodeOutputInspectorService: """One-line factory so tests can monkeypatch a stub if needed.""" - return NodeOutputInspectorService() + return NodeOutputInspectorService(binding_resolver=application_services().workflow_agent_bindings) def _serve_snapshot(app_model: App, run_id: UUID) -> dict: diff --git a/api/controllers/console/datasets/rag_pipeline/datasource_content_preview.py b/api/controllers/console/datasets/rag_pipeline/datasource_content_preview.py index af4993d9d51513..a030e3cb2b989c 100644 --- a/api/controllers/console/datasets/rag_pipeline/datasource_content_preview.py +++ b/api/controllers/console/datasets/rag_pipeline/datasource_content_preview.py @@ -10,11 +10,9 @@ from controllers.console.datasets.wraps import get_rag_pipeline from controllers.console.wraps import account_initialization_required, model_validate, setup_required, with_current_user from extensions.ext_application_services import application_services -from extensions.ext_database import db from libs.login import login_required from models import Account from models.dataset import Pipeline -from services.rag_pipeline.rag_pipeline import RagPipelineService class Parser(BaseModel): @@ -43,8 +41,7 @@ def post(self, req_data: Parser, current_user: Account, pipeline: Pipeline, node inputs = req_data.inputs datasource_type = req_data.datasource_type - rag_pipeline_service = RagPipelineService(db.session()) - preview_content = rag_pipeline_service.run_datasource_node_preview( + preview_content = application_services().knowledge.pipeline_execution.run_datasource_node_preview( pipeline=pipeline, node_id=node_id, user_inputs=inputs, diff --git a/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py b/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py index fd08089749d7f9..f0cca62d15920b 100644 --- a/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py +++ b/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py @@ -3,15 +3,21 @@ from typing import Any, Literal, cast from uuid import UUID -from flask import abort, request +from flask import request from flask_restx import Resource from pydantic import BaseModel, Field, RootModel, ValidationError from sqlalchemy.orm import Session, sessionmaker -from werkzeug.exceptions import BadRequest, Forbidden, InternalServerError, NotFound import services.errors.base from configs import dify_config from controllers.common.controller_schemas import DefaultBlockConfigQuery, WorkflowListQuery, WorkflowUpdatePayload +from controllers.common.errors import ( + AccessDeniedError, + InternalServerError, + InvalidRequestError, + NotFoundError, + UnsupportedMediaTypeError, +) from controllers.common.fields import SimpleResultResponse from controllers.common.rbac import DatasetByPipeline, DatasetId, RBACCheck from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models @@ -21,16 +27,9 @@ DraftWorkflowNotExist, DraftWorkflowNotSync, ) -from controllers.console.app.workflow import ( - RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE, - DefaultBlockConfigResponse, - DefaultBlockConfigsResponse, - WorkflowPaginationResponse, - WorkflowResponse, - WorkflowResponseSource, -) from controllers.console.app.wraps import with_session from controllers.console.datasets.wraps import get_rag_pipeline, load_rag_pipeline +from controllers.console.flask_admission import console_account_admission from controllers.console.wraps import ( RBACPermission, account_initialization_required, @@ -43,13 +42,18 @@ ) from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError from core.app.apps.base_app_queue_manager import AppQueueManager -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom from core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntity +from enums.account import TenantAccountRole from extensions.ext_application_services import application_services from extensions.ext_database import db -from factories import variable_factory from fields.base import ResponseModel +from fields.workflow_fields import ( + DefaultBlockConfigResponse, + DefaultBlockConfigsResponse, + WorkflowPaginationResponse, + WorkflowResponse, +) from fields.workflow_run_fields import ( WorkflowRunDetailResponse, WorkflowRunNodeExecutionListResponse, @@ -62,21 +66,26 @@ from libs import helper from libs.helper import TimestampField, UUIDStrOrEmpty, dump_response from libs.login import login_required +from machinery.context import RequestContext from models import Account from models.dataset import Pipeline from models.model import EndUser -from models.workflow import Workflow -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService +from services.app.generation.response import convert_to_event_stream from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError from services.errors.llm import InvokeRateLimitError -from services.errors.rag_pipeline import RagPipelineResourceNotFoundError +from services.errors.rag_pipeline import RagPipelinePublicationError, RagPipelineResourceNotFoundError +from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError from services.knowledge.dataset_service import DatasetService from services.rag_pipeline.pipeline_generate_service import PipelineGenerateService from services.rag_pipeline.rag_pipeline import RagPipelineService from services.rag_pipeline.rag_pipeline_manage_service import RagPipelineManageService from services.rag_pipeline.rag_pipeline_transform_service import RagPipelineTransformService +from services.workflow.contracts import ( + RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE, + DraftSyncCommand, + WorkflowOwner, +) from services.workflow_ref_service import WorkflowRefService -from services.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError, WorkflowService logger = logging.getLogger(__name__) @@ -219,24 +228,21 @@ def get(self, session: Session, pipeline: Pipeline): """ # fetch draft workflow by app_model rag_pipeline_service = RagPipelineService(session) - workflow = rag_pipeline_service.get_draft_workflow(pipeline=pipeline) + workflow = rag_pipeline_service.get_draft_workflow_record(pipeline=pipeline) if not workflow: raise DraftWorkflowNotExist() # return workflow, if not found, return 404 - return dump_response(WorkflowResponse, WorkflowResponseSource(workflow, session=session)) + return dump_response(WorkflowResponse, workflow) - @setup_required - @login_required - @account_initialization_required - @with_current_user - @get_rag_pipeline - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())) + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())], + ) @console_ns.expect(console_ns.models[DraftWorkflowSyncPayload.__name__]) @console_ns.response(200, "Success", console_ns.models[RagPipelineWorkflowSyncResponse.__name__]) - def post(self, current_user: Account, pipeline: Pipeline): + def post(self, context: RequestContext, pipeline_id: UUID): """ Sync draft workflow """ @@ -251,35 +257,32 @@ def post(self, current_user: Account, pipeline: Pipeline): except (ValueError, ValidationError): return {"message": "Invalid JSON data"}, 400 else: - abort(415) - rag_pipeline_service = RagPipelineService(db.session()) + raise UnsupportedMediaTypeError() try: - environment_variables_list = Workflow.normalize_environment_variable_mappings( - payload.environment_variables or [], - ) - environment_variables = [ - variable_factory.build_environment_variable_from_mapping(obj) for obj in environment_variables_list - ] - conversation_variables_list = payload.conversation_variables or [] - conversation_variables = [ - variable_factory.build_conversation_variable_from_mapping(obj) for obj in conversation_variables_list - ] - workflow = rag_pipeline_service.sync_draft_workflow( - pipeline=pipeline, - graph=payload.graph, - unique_hash=payload.hash, - account=current_user, - environment_variables=environment_variables, - conversation_variables=conversation_variables, - rag_pipeline_variables=payload.rag_pipeline_variables or [], + workflow = application_services().workflow_drafts.sync( + context, + WorkflowOwner(str(pipeline_id), "pipeline"), + DraftSyncCommand( + graph=payload.graph, + features={}, + unique_hash=payload.hash, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=payload.environment_variables or [], + conversation_variables=payload.conversation_variables or [], + rag_pipeline_variables=payload.rag_pipeline_variables or [], + ), ) + except WorkflowNotFoundError as exc: + raise NotFoundError(str(exc)) from exc except WorkflowHashNotEqualError: raise DraftWorkflowNotSync() return { "result": "success", - "hash": workflow.unique_hash, + "hash": workflow.hash, "updated_at": TimestampField().format(workflow.updated_at or workflow.created_at), } @@ -309,13 +312,12 @@ def post(self, req_data: NodeRunPayload, current_user: Account, pipeline: Pipeli user=current_user, node_id=node_id, args=args, - session=db.session(), streaming=True, ) return helper.compact_generate_response(response) except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") except services.errors.conversation.ConversationCompletedError: raise ConversationCompletedError() except ValueError as e: @@ -350,13 +352,12 @@ def post(self, req_data: NodeRunPayload, current_user: Account, pipeline: Pipeli user=current_user, node_id=node_id, args=args, - session=db.session(), streaming=True, ) return helper.compact_generate_response(response) except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") except services.errors.conversation.ConversationCompletedError: raise ConversationCompletedError() except ValueError as e: @@ -388,7 +389,6 @@ def post(self, req_data: DraftWorkflowRunPayload, session: Session, current_user try: response = PipelineGenerateService.generate( generator=application_services().knowledge.pipeline_generator, - session=session, pipeline=pipeline, user=current_user, args=args, @@ -424,7 +424,6 @@ def post(self, req_data: PublishedWorkflowRunPayload, session: Session, current_ try: response = PipelineGenerateService.generate( generator=application_services().knowledge.pipeline_generator, - session=session, pipeline=pipeline, user=current_user, args=args, @@ -454,10 +453,9 @@ def post(self, req_data: DatasourceNodeRunPayload, current_user: Account, pipeli Run rag pipeline datasource """ - rag_pipeline_service = RagPipelineService(db.session()) return helper.compact_generate_response( - PipelineGenerator.convert_to_event_stream( - rag_pipeline_service.run_datasource_workflow_node( + convert_to_event_stream( + application_services().knowledge.pipeline_execution.run_datasource_workflow_node( pipeline=pipeline, node_id=node_id, user_inputs=req_data.inputs, @@ -488,10 +486,9 @@ def post(self, req_data: DatasourceNodeRunPayload, current_user: Account, pipeli Run rag pipeline datasource """ - rag_pipeline_service = RagPipelineService(db.session()) return helper.compact_generate_response( - PipelineGenerator.convert_to_event_stream( - rag_pipeline_service.run_datasource_workflow_node( + convert_to_event_stream( + application_services().knowledge.pipeline_execution.run_datasource_workflow_node( pipeline=pipeline, node_id=node_id, user_inputs=req_data.inputs, @@ -527,8 +524,7 @@ def post(self, req_data: NodeRunRequiredPayload, current_user: Account, pipeline """ inputs = req_data.inputs - rag_pipeline_service = RagPipelineService(db.session()) - workflow_node_execution = rag_pipeline_service.run_draft_workflow_node( + workflow_node_execution = application_services().knowledge.pipeline_execution.run_draft_workflow_node( pipeline=pipeline, node_id=node_id, user_inputs=inputs, account=current_user ) @@ -536,7 +532,12 @@ def post(self, req_data: NodeRunRequiredPayload, current_user: Account, pipeline raise ValueError("Workflow node execution not found") return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(workflow_node_execution, session=db.session()), from_attributes=True + node_execution_response_source( + workflow_node_execution, + session=db.session(), + tool_providers=application_services().tools.tool_providers, + ), + from_attributes=True, ).model_dump(mode="json") @@ -582,41 +583,30 @@ def get(self, pipeline: Pipeline): # fetch published workflow by pipeline session = db.session() rag_pipeline_service = RagPipelineService(session) - workflow = rag_pipeline_service.get_published_workflow(pipeline=pipeline) + workflow = rag_pipeline_service.get_published_workflow_record(pipeline=pipeline) # return workflow, if not found, return None if workflow is None: return None - return dump_response(WorkflowResponse, WorkflowResponseSource(workflow, session=session)) + return dump_response(WorkflowResponse, workflow) @console_ns.response(200, "Success", console_ns.models[RagPipelineWorkflowPublishResponse.__name__]) - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())) - @with_current_user - @get_rag_pipeline - def post(self, current_user: Account, pipeline: Pipeline): - """ - Publish workflow - """ - rag_pipeline_service = RagPipelineService(db.session()) - workflow = rag_pipeline_service.publish_workflow( - session=db.session(), - pipeline=pipeline, - account=current_user, - ) - pipeline.is_published = True - pipeline.workflow_id = workflow.id - db.session.commit() - workflow_created_at = TimestampField().format(workflow.created_at) - - return { - "result": "success", - "created_at": workflow_created_at, - } + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())], + ) + def post(self, context: RequestContext, pipeline_id: str): + """Publish a prepared Pipeline and its dataset settings atomically.""" + try: + workflow = application_services().knowledge.pipeline_publication.publish(context, str(pipeline_id)) + except RagPipelinePublicationError as error: + raise InvalidRequestError(str(error)) from error + except WorkflowHashNotEqualError as error: + raise DraftWorkflowNotSync() from error + except WorkflowNotFoundError as error: + raise NotFoundError(str(error)) from error + return {"result": "success", "created_at": TimestampField().format(workflow.created_at)} @console_ns.route("/rag/pipelines//workflows/default-workflow-block-configs") @@ -702,7 +692,7 @@ def get(self, req_data: WorkflowListQuery, current_user: Account, pipeline: Pipe if user_id: if user_id != current_user.id: - raise Forbidden() + raise AccessDeniedError() rag_pipeline_service = RagPipelineService(db.session()) with sessionmaker(db.engine).begin() as session: @@ -715,44 +705,42 @@ def get(self, req_data: WorkflowListQuery, current_user: Account, pipeline: Pipe named_only=named_only, ) - return WorkflowPaginationResponse.model_validate( - { - "items": [WorkflowResponseSource(workflow, session=session) for workflow in workflows], - "page": page, - "limit": limit, - "has_more": has_more, - } - ).model_dump(mode="json") + return WorkflowPaginationResponse.model_validate( + { + "items": workflows, + "page": page, + "limit": limit, + "has_more": has_more, + } + ).model_dump(mode="json") @console_ns.route("/rag/pipelines//workflows//restore") class RagPipelineDraftWorkflowRestoreApi(Resource): @console_ns.response(200, "Success", console_ns.models[RagPipelineWorkflowSyncResponse.__name__]) - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())) - @with_current_user - @get_rag_pipeline - def post(self, current_user: Account, pipeline: Pipeline, workflow_id: str): - rag_pipeline_service = RagPipelineService(db.session()) + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())], + ) + def post(self, context: RequestContext, pipeline_id: str, workflow_id: str): try: - workflow = rag_pipeline_service.restore_published_workflow_to_draft( - pipeline=pipeline, - workflow_id=workflow_id, - account=current_user, + workflow = application_services().workflow_drafts.restore( + context, + WorkflowOwner(str(pipeline_id), "pipeline"), + workflow_id, ) + except WorkflowHashNotEqualError as exc: + raise DraftWorkflowNotSync() from exc except IsDraftWorkflowError as exc: # Use a stable, predefined message to keep the 400 response consistent - raise BadRequest(RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE) from exc + raise InvalidRequestError(RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE) from exc except WorkflowNotFoundError as exc: - raise NotFound(str(exc)) from exc + raise NotFoundError(str(exc)) from exc return { "result": "success", - "hash": workflow.unique_hash, + "hash": workflow.hash, "updated_at": TimestampField().format(workflow.updated_at or workflow.created_at), } @@ -794,46 +782,24 @@ def patch(self, req_data: WorkflowUpdatePayload, current_user: Account, pipeline ) if not workflow: - raise NotFound("Workflow not found") + raise NotFoundError("Workflow not found") - return dump_response(WorkflowResponse, WorkflowResponseSource(workflow, session=session)) + return dump_response(WorkflowResponse, workflow) @console_ns.response(204, "Workflow deleted successfully") - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())) - @with_current_user - @get_rag_pipeline - def delete(self, current_user: Account, pipeline: Pipeline, workflow_id: str): - """ - Delete a published workflow version that is not currently active on the pipeline. - """ - if pipeline.workflow_id == workflow_id: - abort(400, description=f"Cannot delete workflow that is currently in use by pipeline '{pipeline.id}'") - - workflow_service = WorkflowService() - workflow_ref = WorkflowRefService.create_pipeline_workflow_ref(pipeline, workflow_id) - - with sessionmaker(db.engine).begin() as session: - try: - retirement_candidates = workflow_service.delete_workflow( - session=session, - workflow_ref=workflow_ref, - ) - except WorkflowInUseError as e: - abort(400, description=str(e)) - except DraftWorkflowDeletionError as e: - abort(400, description=str(e)) - except ValueError as e: - raise NotFound(str(e)) - - WorkflowAgentRetirementService.retire_unowned( - tenant_id=pipeline.tenant_id, - agent_ids=retirement_candidates, - account_id=current_user.id, - ) + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())], + ) + def delete(self, context: RequestContext, pipeline_id: UUID, workflow_id: str): + try: + application_services().console_workflows.delete( + context, WorkflowOwner(str(pipeline_id), "pipeline"), workflow_id + ) + except (WorkflowInUseError, DraftWorkflowDeletionError) as error: + raise InvalidRequestError(str(error)) + except WorkflowNotFoundError as error: + raise NotFoundError(str(error)) from error return None, 204 @@ -987,7 +953,7 @@ def get(self, session: Session, pipeline: Pipeline, run_id: UUID): rag_pipeline_service = RagPipelineService(session) workflow_run = rag_pipeline_service.get_rag_pipeline_workflow_run(pipeline=pipeline, run_id=run_id_str) if workflow_run is None: - raise NotFound("Workflow run not found") + raise NotFoundError("Workflow run not found") return WorkflowRunDetailResponse.model_validate( workflow_run_response_source(workflow_run, session=session), from_attributes=True @@ -1015,9 +981,7 @@ def get(self, current_user: Account, pipeline: Pipeline, run_id: UUID): rag_pipeline_service = RagPipelineService(db.session()) user = cast("Account | EndUser", current_user) node_executions = rag_pipeline_service.get_rag_pipeline_workflow_run_node_executions( - pipeline=pipeline, - run_id=run_id_str, - user=user, + pipeline=pipeline, run_id=run_id_str, user=user, tool_providers=application_services().tools.tool_providers ) return WorkflowRunNodeExecutionListResponse.model_validate( @@ -1054,16 +1018,19 @@ def get(self, pipeline: Pipeline, node_id: str): rag_pipeline_service = RagPipelineService(db.session()) workflow = rag_pipeline_service.get_draft_workflow(pipeline=pipeline) if not workflow: - raise NotFound("Workflow not found") + raise NotFoundError("Workflow not found") node_exec = rag_pipeline_service.get_node_last_run( pipeline=pipeline, workflow=workflow, node_id=node_id, ) if node_exec is None: - raise NotFound("last run not found") + raise NotFoundError("last run not found") return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(node_exec, session=db.session()), from_attributes=True + node_execution_response_source( + node_exec, session=db.session(), tool_providers=application_services().tools.tool_providers + ), + from_attributes=True, ).model_dump(mode="json") @@ -1081,20 +1048,20 @@ class RagPipelineTransformApi(Resource): def post(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID): dataset = DatasetService.get_dataset_for_tenant(str(dataset_id), current_tenant_id, session=session) if dataset is None: - raise NotFound("Dataset not found.") + raise NotFoundError("Dataset not found.") if not dify_config.RBAC_ENABLED: if not (current_user.has_edit_permission or current_user.is_dataset_operator): - raise Forbidden() + raise AccessDeniedError() try: DatasetService.check_dataset_permission(dataset, current_user, session) except services.errors.base.NoPermissionError as exc: - raise Forbidden(str(exc)) from exc + raise AccessDeniedError(str(exc)) from exc try: return RagPipelineTransformService().transform_dataset(dataset, current_user.id, session) except RagPipelineResourceNotFoundError as exc: - raise NotFound(str(exc)) from exc + raise NotFoundError(str(exc)) from exc @console_ns.route("/rag/pipelines//workflows/draft/datasource/variables-inspect") @@ -1119,14 +1086,18 @@ def post(self, req_data: DatasourceVariablesPayload, current_user: Account, pipe """ args = req_data.model_dump() - rag_pipeline_service = RagPipelineService(db.session()) - workflow_node_execution = rag_pipeline_service.set_datasource_variables( + workflow_node_execution = application_services().knowledge.pipeline_execution.set_datasource_variables( pipeline=pipeline, args=args, current_user=current_user, ) return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(workflow_node_execution, session=db.session()), from_attributes=True + node_execution_response_source( + workflow_node_execution, + session=db.session(), + tool_providers=application_services().tools.tool_providers, + ), + from_attributes=True, ).model_dump(mode="json") @@ -1144,6 +1115,6 @@ def get(self, req_data: RagPipelineRecommendedPluginQuery, current_tenant_id: st rag_pipeline_service = RagPipelineService(db.session()) recommended_plugins = rag_pipeline_service.get_recommended_plugins( - req_data.type, current_user, current_tenant_id + req_data.type, current_user, current_tenant_id, tool_providers=application_services().tools.tool_providers ) return recommended_plugins diff --git a/api/controllers/console/explore/message.py b/api/controllers/console/explore/message.py index 188b9a59409aae..3ca5fd832e4114 100644 --- a/api/controllers/console/explore/message.py +++ b/api/controllers/console/explore/message.py @@ -10,9 +10,9 @@ from flask import Response from flask_restx import Resource from pydantic import BaseModel -from werkzeug.exceptions import HTTPException, InternalServerError, Unauthorized from controllers.common.controller_schemas import MessageFeedbackPayload, MessageListQuery +from controllers.common.errors import AuthenticationRequiredError, InternalServerError, raise_unexpected_error from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models from controllers.console import console_ns from controllers.console.app.error import ( @@ -46,6 +46,7 @@ from services.account_errors import AccountNotFoundError from services.app_definition_query_service import AppDefinitionUnavailableError from services.errors.app import MoreLikeThisDisabledError +from services.errors.app_model_config import AppModelConfigBrokenError from services.errors.base import BaseServiceError from services.errors.conversation import ConversationNotExistsError from services.errors.message import ( @@ -81,10 +82,10 @@ def decorated(*args: P.args, **kwargs: P.kwargs) -> R: return view(*args, **kwargs) except InstalledAppNotFoundError as error: raise InstalledAppNotFoundHTTPError() from error - except AppDefinitionUnavailableError as error: + except (AppDefinitionUnavailableError, AppModelConfigBrokenError) as error: raise AppUnavailableError() from error except (AccountNotFoundError, SuggestedQuestionsActorNotFoundError) as error: - raise Unauthorized("Account no longer exists.") from error + raise AuthenticationRequiredError("Account no longer exists.") from error except MessageNotChatAppError as error: raise NotChatAppError() from error except InstalledAppNotCompletionError as error: @@ -109,11 +110,10 @@ def decorated(*args: P.args, **kwargs: P.kwargs) -> R: raise ProviderModelCurrentlyNotSupportError() from error except InvokeError as error: raise CompletionRequestError(error.description) from error - except (HTTPException, ValueError): + except ValueError: raise except Exception as error: - logger.exception("Installed-app message operation failed") - raise InternalServerError() from error + raise_unexpected_error(error, message="Installed-app message operation failed") return decorated diff --git a/api/controllers/console/explore/trial.py b/api/controllers/console/explore/trial.py index 6de344785cd2c7..59ec87c2f4b35a 100644 --- a/api/controllers/console/explore/trial.py +++ b/api/controllers/console/explore/trial.py @@ -1,8 +1,8 @@ import logging -from collections.abc import Callable, Mapping +from collections.abc import Generator, Mapping +from contextlib import contextmanager from dataclasses import asdict from datetime import datetime -from functools import wraps from http import HTTPStatus from typing import Any, Literal from uuid import UUID @@ -10,11 +10,16 @@ from flask import Response, request from flask_restx import Resource from pydantic import AliasChoices, BaseModel, Field, field_validator -from werkzeug.exceptions import HTTPException, InternalServerError, NotFound, Unauthorized import services from configs import dify_config from controllers.common.audio_response import audio_binary_response +from controllers.common.errors import ( + AuthenticationRequiredError, + InternalServerError, + NotFoundError, + raise_unexpected_error, +) from controllers.common.fields import ( AudioBinaryResponse, AudioTranscriptResponse, @@ -63,7 +68,6 @@ from controllers.console.remote_files import RemoteFileUploadPayload, upload_remote_file from controllers.console.wraps import cloud_edition_billing_resource_check, model_validate from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError -from core.app.apps.agent_app.errors import AgentAppGeneratorError, AgentAppNotPublishedError from core.app.entities.app_invoke_entities import InvokeFrom from core.errors.error import ( AppInvokeQuotaExceededError, @@ -89,6 +93,7 @@ from models.enums import CreatorUserRole from models.model import AppMode from services.account_errors import AccountNotFoundError +from services.app.generation.errors import AgentAppGeneratorError, AgentAppNotPublishedError from services.app_definition_query_service import AppDefinitionUnavailableError from services.app_preview_query_service import ( AppPreviewOwnerUnavailableError, @@ -531,7 +536,7 @@ def post(self, req_data: WorkflowRunRequest, request_context: RequestContext, tr except AppDefinitionUnavailableError as error: raise AppUnavailableError() from error except AccountNotFoundError as error: - raise Unauthorized("Account no longer exists.") from error + raise AuthenticationRequiredError("Account no longer exists.") from error except services.errors.app_model_config.AppModelConfigBrokenError as error: raise AppUnavailableError() from error except ProviderTokenNotInitError as ex: @@ -600,9 +605,9 @@ def post(self, req_data: ChatRequest, request_context: RequestContext, trial_app except AppDefinitionUnavailableError as error: raise AppUnavailableError() from error except AccountNotFoundError as error: - raise Unauthorized("Account no longer exists.") from error + raise AuthenticationRequiredError("Account no longer exists.") from error except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") except services.errors.conversation.ConversationCompletedError: raise ConversationCompletedError() except services.errors.app_model_config.AppModelConfigBrokenError: @@ -671,11 +676,11 @@ def get(self, request_context: RequestContext, trial_app: TrialAppRef, message_i except AppDefinitionUnavailableError as error: raise AppUnavailableError() from error except SuggestedQuestionsActorNotFoundError as error: - raise Unauthorized("Account no longer exists.") from error + raise AuthenticationRequiredError("Account no longer exists.") from error except MessageNotExistsError: - raise NotFound("Message not found") + raise NotFoundError("Message not found") except ConversationNotExistsError: - raise NotFound("Conversation not found") + raise NotFoundError("Conversation not found") except SuggestedQuestionsAfterAnswerDisabledError: raise AppSuggestedQuestionsAfterAnswerDisabledError() except ProviderTokenNotInitError as ex: @@ -693,38 +698,34 @@ def get(self, request_context: RequestContext, trial_app: TrialAppRef, message_i return dump_response(SuggestedQuestionsResponse, {"data": questions}) -def _trial_audio_errors[**P, R](view: Callable[P, R]) -> Callable[P, R]: - @wraps(view) - def decorated(*args: P.args, **kwargs: P.kwargs) -> R: - try: - return view(*args, **kwargs) - except (AppDefinitionUnavailableError, services.errors.app_model_config.AppModelConfigBrokenError) as error: - raise AppUnavailableError() from error - except NoAudioUploadedServiceError as error: - raise NoAudioUploadedError() from error - except AudioTooLargeServiceError as error: - raise AudioTooLargeError(str(error)) from error - except UnsupportedAudioTypeServiceError as error: - raise UnsupportedAudioTypeError() from error - except ProviderNotSupportSpeechToTextServiceError as error: - raise ProviderNotSupportSpeechToTextError() from error - except SpeechToTextDisabledServiceError as error: - raise SpeechToTextDisabledError() from error - except ProviderTokenNotInitError as error: - raise ProviderNotInitializeError(error.description) from error - except QuotaExceededError as error: - raise ProviderQuotaExceededError() from error - except ModelCurrentlyNotSupportError as error: - raise ProviderModelCurrentlyNotSupportError() from error - except InvokeError as error: - raise CompletionRequestError(error.description) from error - except (HTTPException, ValueError): - raise - except Exception as error: - logger.exception("Trial audio operation failed") - raise InternalServerError() from error - - return decorated +@contextmanager +def _trial_audio_errors() -> Generator[None]: + try: + yield + except (AppDefinitionUnavailableError, services.errors.app_model_config.AppModelConfigBrokenError) as error: + raise AppUnavailableError() from error + except NoAudioUploadedServiceError as error: + raise NoAudioUploadedError() from error + except AudioTooLargeServiceError as error: + raise AudioTooLargeError(str(error)) from error + except UnsupportedAudioTypeServiceError as error: + raise UnsupportedAudioTypeError() from error + except ProviderNotSupportSpeechToTextServiceError as error: + raise ProviderNotSupportSpeechToTextError() from error + except SpeechToTextDisabledServiceError as error: + raise SpeechToTextDisabledError() from error + except ProviderTokenNotInitError as error: + raise ProviderNotInitializeError(error.description) from error + except QuotaExceededError as error: + raise ProviderQuotaExceededError() from error + except ModelCurrentlyNotSupportError as error: + raise ProviderModelCurrentlyNotSupportError() from error + except InvokeError as error: + raise CompletionRequestError(error.description) from error + except ValueError: + raise + except Exception as error: + raise_unexpected_error(error, message="Trial audio operation failed") @console_ns.route( @@ -735,16 +736,18 @@ class TrialChatAudioApi(Resource): @console_ns.response(HTTPStatus.OK, "Success", console_ns.models[AudioTranscriptResponse.__name__]) @console_account_admission() @get_trial_app - @_trial_audio_errors def post(self, request_context: RequestContext, trial_app: TrialAppRef) -> dict[str, object]: file = request.files.get("file") audio = AudioUpload(stream=file.stream, mime_type=file.mimetype) if file is not None else None - transcript = application_services().app_audio.transcript_asr( - app=AudioAppRef(app_id=trial_app.app_id, tenant_id=trial_app.tenant_id, app_mode=trial_app.app_mode), - audio=audio, - ) - application_services().trial_apps.usage.record(app_id=trial_app.app_id, account_id=request_context.account_id) - return dump_response(AudioTranscriptResponse, transcript) + with _trial_audio_errors(): + transcript = application_services().app_audio.transcript_asr( + app=AudioAppRef(app_id=trial_app.app_id, tenant_id=trial_app.tenant_id, app_mode=trial_app.app_mode), + audio=audio, + ) + application_services().trial_apps.usage.record( + app_id=trial_app.app_id, account_id=request_context.account_id + ) + return dump_response(AudioTranscriptResponse, transcript) @console_ns.route( @@ -757,30 +760,30 @@ class TrialChatTextApi(Resource): @console_account_admission() @get_trial_app @model_validate(TextToSpeechRequest) - @_trial_audio_errors def post( self, req_data: TextToSpeechRequest, request_context: RequestContext, trial_app: TrialAppRef ) -> Response | None: - output = application_services().app_audio.transcript_tts( - app=AudioAppRef(app_id=trial_app.app_id, tenant_id=trial_app.tenant_id, app_mode=trial_app.app_mode), - account_id=request_context.account_id, - text=req_data.text, - voice=req_data.voice, - message_id=req_data.message_id, - ) - response = audio_binary_response(output) - try: - # Preserve usage after MIME inspection, including a missing message's - # null response. Early provider/MIME failures do not consume a trial. - application_services().trial_apps.usage.record( - app_id=trial_app.app_id, account_id=request_context.account_id + with _trial_audio_errors(): + output = application_services().app_audio.transcript_tts( + app=AudioAppRef(app_id=trial_app.app_id, tenant_id=trial_app.tenant_id, app_mode=trial_app.app_mode), + account_id=request_context.account_id, + text=req_data.text, + voice=req_data.voice, + message_id=req_data.message_id, ) - except BaseException: - if response is not None: - close_stream(response) - raise - # response-contract:ignore audio_binary_response - return response + response = audio_binary_response(output) + try: + # Preserve usage after MIME inspection, including a missing message's + # null response. Early provider/MIME failures do not consume a trial. + application_services().trial_apps.usage.record( + app_id=trial_app.app_id, account_id=request_context.account_id + ) + except BaseException: + if response is not None: + close_stream(response) + raise + # response-contract:ignore audio_binary_response + return response @console_ns.route( @@ -807,9 +810,9 @@ def post(self, req_data: CompletionRequest, request_context: RequestContext, tri except AppDefinitionUnavailableError as error: raise AppUnavailableError() from error except AccountNotFoundError as error: - raise Unauthorized("Account no longer exists.") from error + raise AuthenticationRequiredError("Account no longer exists.") from error except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") except services.errors.conversation.ConversationCompletedError: raise ConversationCompletedError() except services.errors.app_model_config.AppModelConfigBrokenError: @@ -890,7 +893,7 @@ def get(self, app: AppPreviewRef, request_context: RequestContext) -> dict[str, except AppDefinitionUnavailableError as error: raise AppUnavailableError() from error except AccountNotFoundError as error: - raise Unauthorized("Account no longer exists.") from error + raise AuthenticationRequiredError("Account no longer exists.") from error except AppPreviewSiteUnavailableError as error: raise AppPreviewSiteUnavailableHttpError(str(error)) from error diff --git a/api/controllers/console/snippets/snippet_workflow.py b/api/controllers/console/snippets/snippet_workflow.py index a0952eefa0035b..50a37b7403339f 100644 --- a/api/controllers/console/snippets/snippet_workflow.py +++ b/api/controllers/console/snippets/snippet_workflow.py @@ -6,24 +6,16 @@ from flask_restx import Resource from pydantic import BaseModel, Field from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, InternalServerError, NotFound from controllers.common.controller_schemas import WorkflowUpdatePayload +from controllers.common.errors import InternalServerError, InvalidRequestError, NotFoundError from controllers.common.fields import GeneratedAppResponse, SimpleResultResponse from controllers.common.rbac import RBACCheck, Workspace from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models from controllers.common.session import with_session from controllers.console import console_ns from controllers.console.app.error import DraftWorkflowNotExist, DraftWorkflowNotSync -from controllers.console.app.workflow import ( - RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE, - DefaultBlockConfigsResponse, - WorkflowPaginationResponse, - WorkflowPublishResponse, - WorkflowResponse, - WorkflowResponseSource, - WorkflowRestoreResponse, -) +from controllers.console.flask_admission import console_account_admission from controllers.console.snippets.payloads import ( PublishWorkflowPayload, SnippetDraftNodeRunPayload, @@ -41,12 +33,23 @@ model_validate, rbac_permission_required, setup_required, + validate_request, with_current_user, ) from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.entities.app_invoke_entities import InvokeFrom from core.db.session_factory import session_factory +from enums.account import TenantAccountRole +from extensions.application_services.snippets import build_snippet_service +from extensions.ext_application_services import application_services from extensions.ext_redis import redis_client +from fields.workflow_fields import ( + DefaultBlockConfigsResponse, + WorkflowPaginationResponse, + WorkflowPublishResponse, + WorkflowResponse, + WorkflowRestoreResponse, +) from fields.workflow_run_fields import ( WorkflowRunDetailResponse, WorkflowRunNodeExecutionListResponse, @@ -60,13 +63,18 @@ from libs import helper from libs.helper import TimestampField from libs.login import current_account_with_tenant, login_required +from machinery.context import RequestContext from models import Account from models.snippet import CustomizedSnippet -from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError from services.snippet_generate_service import SnippetGenerateService from services.snippet_service import SnippetService +from services.workflow.contracts import ( + RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE, + DraftSyncCommand, + WorkflowOwner, +) logger = logging.getLogger(__name__) @@ -74,7 +82,7 @@ def _snippet_service() -> SnippetService: - return SnippetService(session_factory.get_session_maker()) + return build_snippet_service(session_factory.get_session_maker()) class SnippetWorkflowResponse(WorkflowResponse): @@ -148,7 +156,7 @@ def decorated_view(*args: P.args, **kwargs: P.kwargs) -> R: ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") kwargs["snippet"] = snippet @@ -175,19 +183,12 @@ class SnippetDraftWorkflowApi(Resource): def get(self, session: Session, snippet: CustomizedSnippet): """Get draft workflow for snippet.""" snippet_service = _snippet_service() - workflow = snippet_service.get_draft_workflow(snippet=snippet) + workflow = snippet_service.get_draft_workflow_record(snippet=snippet) if not workflow: raise DraftWorkflowNotExist() - workflow.conversation_variables = [] - response = SnippetWorkflowResponse.model_validate( - WorkflowResponseSource(workflow, session=session), from_attributes=True - ).model_dump(mode="json") - response["graph"] = WorkflowAgentPublishService.project_draft_bindings_to_graph( - session=session, - draft_workflow=workflow, - ) + response = SnippetWorkflowResponse.model_validate(workflow, from_attributes=True).model_dump(mode="json") response["input_fields"] = snippet.input_fields_list return response @@ -199,26 +200,31 @@ def get(self, session: Session, snippet: CustomizedSnippet): console_ns.models[WorkflowRestoreResponse.__name__], ) @console_ns.response(400, "Hash mismatch") - @setup_required - @login_required - @account_initialization_required - @with_current_user - @get_snippet - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())) - @model_validate(SnippetDraftSyncPayload) - def post(self, req_data: SnippetDraftSyncPayload, current_user: Account, snippet: CustomizedSnippet): + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())], + ) + def post(self, context: RequestContext, snippet_id: str): """Sync draft workflow for snippet.""" - + req_data = validate_request(SnippetDraftSyncPayload) try: - snippet_service = _snippet_service() - workflow = snippet_service.sync_draft_workflow( - snippet=snippet, - graph=req_data.graph, - unique_hash=req_data.hash, - account=current_user, - input_fields=req_data.input_fields, + workflow = application_services().workflow_drafts.sync( + context, + WorkflowOwner(str(snippet_id), "snippet"), + DraftSyncCommand( + graph=req_data.graph, + features={}, + unique_hash=req_data.hash, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=[], + conversation_variables=[], + input_fields=req_data.input_fields, + ), ) + except WorkflowNotFoundError as exc: + raise NotFoundError(str(exc)) from exc except WorkflowHashNotEqualError: raise DraftWorkflowNotSync() except ValueError as e: @@ -226,7 +232,7 @@ def post(self, req_data: SnippetDraftSyncPayload, current_user: Account, snippet return { "result": "success", - "hash": workflow.unique_hash, + "hash": workflow.hash, "updated_at": TimestampField().format(workflow.updated_at or workflow.created_at), } @@ -272,14 +278,12 @@ def get(self, session: Session, snippet: CustomizedSnippet): return None snippet_service = _snippet_service() - workflow = snippet_service.get_published_workflow(snippet=snippet) + workflow = snippet_service.get_published_workflow_record(snippet=snippet) if not workflow: return None - response = SnippetWorkflowResponse.model_validate( - WorkflowResponseSource(workflow, session=session), from_attributes=True - ).model_dump(mode="json") + response = SnippetWorkflowResponse.model_validate(workflow, from_attributes=True).model_dump(mode="json") response["input_fields"] = snippet.input_fields_list return response @@ -291,26 +295,22 @@ def get(self, session: Session, snippet: CustomizedSnippet): @login_required @account_initialization_required @with_current_user - @with_session @get_snippet @edit_permission_required @rbac_permission_required(RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())) - def post(self, session: Session, current_user: Account, snippet: CustomizedSnippet): + def post(self, current_user: Account, snippet: CustomizedSnippet): """Publish snippet workflow.""" snippet_service = _snippet_service() - snippet = session.merge(snippet) try: workflow = snippet_service.publish_workflow( - session=session, snippet=snippet, account=current_user, ) workflow_created_at = TimestampField().format(workflow.created_at) + except WorkflowHashNotEqualError as e: + raise DraftWorkflowNotSync() from e except ValueError as e: - # This domain validation remains a 400 response, so it cannot escape to - # the request-session decorator that normally owns rollback-on-error. - session.rollback() # guard-ignore: no-new-controller-sqlalchemy -- translated validation response return {"message": str(e)}, 400 return { @@ -375,7 +375,7 @@ def get( response = SnippetWorkflowPaginationResponse.model_validate( { - "items": [WorkflowResponseSource(workflow, session=session) for workflow in workflows], + "items": workflows, "page": req_data.page, "limit": req_data.limit, "has_more": has_more, @@ -395,33 +395,31 @@ class SnippetDraftWorkflowRestoreApi(Resource): @console_ns.response(200, "Workflow restored successfully", console_ns.models[WorkflowRestoreResponse.__name__]) @console_ns.response(400, "Source workflow must be published") @console_ns.response(404, "Workflow not found") - @setup_required - @login_required - @account_initialization_required - @with_current_user - @get_snippet - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())) - def post(self, current_user: Account, snippet: CustomizedSnippet, workflow_id: str): + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())], + ) + def post(self, context: RequestContext, snippet_id: str, workflow_id: str): """Restore a published snippet workflow version into the draft workflow.""" - snippet_service = _snippet_service() try: - workflow = snippet_service.restore_published_workflow_to_draft( - snippet=snippet, - workflow_id=workflow_id, - account=current_user, + workflow = application_services().workflow_drafts.restore( + context, + WorkflowOwner(str(snippet_id), "snippet"), + workflow_id, ) + except WorkflowHashNotEqualError as exc: + raise DraftWorkflowNotSync() from exc except IsDraftWorkflowError as exc: - raise BadRequest(RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE) from exc + raise InvalidRequestError(RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE) from exc except WorkflowNotFoundError as exc: - raise NotFound(str(exc)) from exc + raise NotFoundError(str(exc)) from exc except ValueError as exc: - raise BadRequest(str(exc)) from exc + raise InvalidRequestError(str(exc)) from exc return { "result": "success", - "hash": workflow.unique_hash, + "hash": workflow.hash, "updated_at": TimestampField().format(workflow.updated_at or workflow.created_at), } @@ -467,11 +465,9 @@ def patch( data=update_data, ) if not workflow: - raise NotFound("Workflow not found") + raise NotFoundError("Workflow not found") - response = SnippetWorkflowResponse.model_validate( - WorkflowResponseSource(workflow, session=session), from_attributes=True - ).model_dump(mode="json") + response = SnippetWorkflowResponse.model_validate(workflow, from_attributes=True).model_dump(mode="json") response["input_fields"] = snippet.input_fields_list return response @@ -481,29 +477,20 @@ def patch( @console_ns.response(204, "Workflow deleted successfully") @console_ns.response(400, "Workflow is in use") @console_ns.response(404, "Workflow not found") - @setup_required - @login_required - @account_initialization_required - @get_snippet - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())) - def delete(self, snippet: CustomizedSnippet, workflow_id: str): - """Delete a published snippet workflow version.""" - snippet_service = _snippet_service() - with session_factory.get_session_maker().begin() as session: - try: - snippet_service.delete_workflow( - session=session, - snippet=snippet, - workflow_id=workflow_id, - ) - except WorkflowInUseError as e: - raise BadRequest(str(e)) - except DraftWorkflowDeletionError as e: - raise BadRequest(str(e)) - except ValueError as e: - raise NotFound(str(e)) - + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())], + ) + def delete(self, context: RequestContext, snippet_id: str, workflow_id: str): + """Delete a version and retire its unowned Agents through the shared use case.""" + try: + application_services().console_workflows.delete( + context, WorkflowOwner(str(snippet_id), "snippet"), str(workflow_id) + ) + except (WorkflowInUseError, DraftWorkflowDeletionError) as error: + raise InvalidRequestError(str(error)) from error + except WorkflowNotFoundError as error: + raise NotFoundError(str(error)) from error return None, 204 @@ -564,7 +551,7 @@ def get(self, session: Session, snippet: CustomizedSnippet, run_id): workflow_run = snippet_service.get_snippet_workflow_run(snippet=snippet, run_id=run_id) if not workflow_run: - raise NotFound("Workflow run not found") + raise NotFoundError("Workflow run not found") return WorkflowRunDetailResponse.model_validate( workflow_run_response_source(workflow_run, session=session), from_attributes=True @@ -589,8 +576,7 @@ def get(self, snippet: CustomizedSnippet, run_id): snippet_service = _snippet_service() node_executions = snippet_service.get_snippet_workflow_run_node_executions( - snippet=snippet, - run_id=run_id, + snippet=snippet, run_id=run_id, tool_providers=application_services().tools.tool_providers ) return WorkflowRunNodeExecutionListResponse.model_validate( @@ -635,23 +621,25 @@ def post( snippet_service = _snippet_service() draft_workflow = snippet_service.get_draft_workflow(snippet=snippet) if not draft_workflow: - raise NotFound("Draft workflow not found") + raise NotFoundError("Draft workflow not found") files = SnippetGenerateService.parse_files(draft_workflow, req_data.files) - workflow_node_execution = SnippetGenerateService.run_draft_node( + workflow_node_execution = application_services().snippet_generation.run_draft_node( snippet=snippet, node_id=node_id, user_inputs=user_inputs, account=current_user, query=req_data.query, files=files, - session_maker=session_factory.get_session_maker(), ) with session_factory.create_session() as session: return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(workflow_node_execution, session=session), from_attributes=True + node_execution_response_source( + workflow_node_execution, session=session, tool_providers=application_services().tools.tool_providers + ), + from_attributes=True, ).model_dump(mode="json") @@ -678,7 +666,7 @@ def get(self, snippet: CustomizedSnippet, node_id: str): snippet_service = _snippet_service() draft_workflow = snippet_service.get_draft_workflow(snippet=snippet) if not draft_workflow: - raise NotFound("Draft workflow not found") + raise NotFoundError("Draft workflow not found") node_exec = snippet_service.get_snippet_node_last_run( snippet=snippet, @@ -686,11 +674,14 @@ def get(self, snippet: CustomizedSnippet, node_id: str): node_id=node_id, ) if node_exec is None: - raise NotFound("Node last run not found") + raise NotFoundError("Node last run not found") with session_factory.create_session() as session: return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(node_exec, session=session), from_attributes=True + node_execution_response_source( + node_exec, session=session, tool_providers=application_services().tools.tool_providers + ), + from_attributes=True, ).model_dump(mode="json") @@ -729,13 +720,12 @@ def post( args = req_data.model_dump(exclude_none=True) try: - response = SnippetGenerateService.generate_single_iteration( + response = application_services().snippet_generation.generate_single_iteration( snippet=snippet, user=current_user, node_id=node_id, args=args, streaming=True, - session_maker=session_factory.get_session_maker(), ) return helper.compact_generate_response(response) @@ -780,13 +770,12 @@ def post( """ try: - response = SnippetGenerateService.generate_single_loop( + response = application_services().snippet_generation.generate_single_loop( snippet=snippet, user=current_user, node_id=node_id, - args=req_data, + args=req_data.model_dump(), streaming=True, - session_maker=session_factory.get_session_maker(), ) return helper.compact_generate_response(response) @@ -824,13 +813,12 @@ def post(self, req_data: SnippetDraftRunPayload, current_user: Account, snippet: args = req_data.model_dump(exclude_none=True) try: - response = SnippetGenerateService.generate( + response = application_services().snippet_generation.generate( snippet=snippet, user=current_user, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=True, - session_maker=session_factory.get_session_maker(), ) return helper.compact_generate_response(response) diff --git a/api/controllers/console/workspace/snippets.py b/api/controllers/console/workspace/snippets.py index 310199482ffc3c..92babf651de891 100644 --- a/api/controllers/console/workspace/snippets.py +++ b/api/controllers/console/workspace/snippets.py @@ -6,8 +6,8 @@ from flask import Response, request from flask_restx import Resource from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, NotFound +from controllers.common.errors import InvalidRequestError, NotFoundError from controllers.common.fields import TextFileResponse from controllers.common.rbac import RBACCheck, Workspace from controllers.common.schema import ( @@ -36,6 +36,7 @@ ) from core.db.session_factory import session_factory from core.plugin.entities.plugin import PluginDependency +from extensions.application_services.snippets import build_snippet_dsl_service, build_snippet_service from extensions.ext_application_services import application_services from fields.base import ResponseModel from fields.snippet_fields import ( @@ -50,7 +51,7 @@ from models import Account from models.snippet import SnippetType from services.entities.dsl_entities import DslImportWarning -from services.snippet_dsl_service import ImportStatus, SnippetDslService +from services.snippet_dsl_service import ImportStatus from services.snippet_service import SnippetService logger = logging.getLogger(__name__) @@ -77,8 +78,8 @@ class SnippetUseCountResponse(ResponseModel): def _snippet_service(session: Session | None = None) -> SnippetService: if session is not None: - return SnippetService(session=session) - return SnippetService(session_factory.get_session_maker()) + return build_snippet_service(session=session) + return build_snippet_service(session_factory.get_session_maker()) def _snippet_list_query_from_request() -> SnippetListQuery: @@ -180,9 +181,6 @@ def post( snippet_type = SnippetType.NODE try: - if req_data.graph is not None: - SnippetService.validate_snippet_graph_forbidden_nodes(req_data.graph) - snippet_service = _snippet_service(session) snippet = snippet_service.create_snippet( tenant_id=current_tenant_id, @@ -191,6 +189,7 @@ def post( snippet_type=snippet_type, icon_info=req_data.icon_info.model_dump() if req_data.icon_info else None, input_fields=[f.model_dump() for f in req_data.input_fields] if req_data.input_fields else None, + graph=req_data.graph, account=current_user, ) except ValueError as e: @@ -218,7 +217,7 @@ def get(self, session: Session, current_tenant_id: str, snippet_id: UUID): ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") return dump_response(SnippetResponse, snippet_response(snippet, session=session)), 200 @@ -252,7 +251,7 @@ def patch( ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") update_data = req_data.model_dump(exclude_unset=True) @@ -275,8 +274,8 @@ def patch( # Raise rather than return: `with_session` commits on a normal return, so returning here # would persist whatever the update wrote before it rejected the payload. Raising routes # through the decorator's rollback. Status stays 400 and `message` is unchanged; the body - # picks up the standard error envelope, as on every other BadRequest in the console API. - raise BadRequest(str(e)) from e + # picks up the standard error envelope, as on every other InvalidRequestError in the console API. + raise InvalidRequestError(str(e)) from e return dump_response(SnippetResponse, snippet_response(snippet, session=session)), 200 @@ -300,7 +299,7 @@ def delete(self, session: Session, current_tenant_id: str, current_user: Account ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") SnippetService.delete_snippet( session=session, @@ -335,12 +334,12 @@ def get(self, session: Session, current_tenant_id: str, snippet_id: str): ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") # Get include_secret parameter query = SnippetExportQuery.model_validate(request.args.to_dict()) - export_service = SnippetDslService(session) + export_service = build_snippet_dsl_service(session) try: result = export_service.export_snippet_dsl( snippet=snippet, @@ -348,7 +347,7 @@ def get(self, session: Session, current_tenant_id: str, snippet_id: str): workflow_id=query.workflow_id, ) except ValueError as exc: - raise NotFound(str(exc)) from exc + raise NotFoundError(str(exc)) from exc # Set filename with .snippet extension filename = f"{snippet.name}.snippet" @@ -382,7 +381,7 @@ class CustomizedSnippetImportApi(Resource): @model_validate(SnippetImportPayload) def post(self, req_data: SnippetImportPayload, session: Session, current_user: Account): """Import snippet from DSL.""" - import_service = SnippetDslService(session) + import_service = build_snippet_dsl_service(session) result = import_service.import_snippet( account=current_user, import_mode=req_data.mode, @@ -418,7 +417,7 @@ class CustomizedSnippetImportConfirmApi(Resource): @with_session def post(self, session: Session, current_user: Account, import_id: str): """Confirm a pending snippet import.""" - import_service = SnippetDslService(session) + import_service = build_snippet_dsl_service(session) result = import_service.confirm_import(import_id=import_id, account=current_user) if result.status == ImportStatus.FAILED: @@ -453,9 +452,9 @@ def get(self, session: Session, current_tenant_id: str, snippet_id: str): ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") - import_service = SnippetDslService(session) + import_service = build_snippet_dsl_service(session) result = import_service.check_dependencies(snippet=snippet) return result.model_dump(mode="json"), 200 @@ -483,7 +482,7 @@ def post(self, session: Session, current_tenant_id: str, snippet_id: str): ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") SnippetService.increment_use_count(session=session, snippet=snippet) session.flush() diff --git a/api/controllers/inner_api/agent/tools.py b/api/controllers/inner_api/agent/tools.py index be11c47486c0c7..19e42cd12504d4 100644 --- a/api/controllers/inner_api/agent/tools.py +++ b/api/controllers/inner_api/agent/tools.py @@ -2,14 +2,12 @@ from flask_restx import Resource from pydantic import ValidationError -from sqlalchemy.orm import Session from controllers.common.schema import register_response_schema_models, register_schema_models -from controllers.console.app.wraps import with_session from controllers.inner_api import inner_api_ns from controllers.inner_api.wraps import agent_inner_api_only +from extensions.ext_application_services import application_services from libs.exception import BaseHTTPException -from services.agent_tool_inner_service import AgentToolInnerService from services.entities.agent_tool_inner import AgentToolInvokeRequest, AgentToolInvokeResponse from services.errors.agent_tool_inner import AgentToolInnerServiceError @@ -47,8 +45,7 @@ class AgentToolInvokeApi(Resource): @inner_api_ns.doc("inner_agent_tool_invoke") @inner_api_ns.expect(inner_api_ns.models[AgentToolInvokeRequest.__name__]) @inner_api_ns.response(200, "Tool invoked successfully", inner_api_ns.models[AgentToolInvokeResponse.__name__]) - @with_session - def post(self, session: Session) -> dict[str, object]: + def post(self) -> dict[str, object]: try: payload = AgentToolInvokeRequest.model_validate(inner_api_ns.payload or {}) except ValidationError as exc: @@ -59,7 +56,7 @@ def post(self, session: Session) -> dict[str, object]: ) from exc try: - response = AgentToolInnerService().invoke(session=session, request=payload) + response = application_services().agent_apps.tools.invoke(request=payload) except AgentToolInnerServiceError as exc: raise AgentToolInvokeHttpError( error_code=exc.error_code, diff --git a/api/controllers/inner_api/plugin/plugin.py b/api/controllers/inner_api/plugin/plugin.py index 65103cee9bd073..976710008e1bad 100644 --- a/api/controllers/inner_api/plugin/plugin.py +++ b/api/controllers/inner_api/plugin/plugin.py @@ -7,12 +7,10 @@ from controllers.inner_api import inner_api_ns from controllers.inner_api.plugin.wraps import get_user_tenant, plugin_data from controllers.inner_api.wraps import plugin_inner_api_only -from core.plugin.backwards_invocation.app import PluginAppBackwardsInvocation from core.plugin.backwards_invocation.base import BaseBackwardsInvocationResponse from core.plugin.backwards_invocation.encrypt import PluginEncrypter from core.plugin.backwards_invocation.model import PluginModelBackwardsInvocation from core.plugin.backwards_invocation.node import PluginNodeBackwardsInvocation -from core.plugin.backwards_invocation.tool import PluginToolBackwardsInvocation from core.plugin.entities.request import ( RequestFetchAppInfo, RequestInvokeApp, @@ -40,6 +38,8 @@ from models import Account, Tenant from models.model import EndUser from services.file_request_service import FileRequestService +from services.plugin.app_invocation import PluginAppBackwardsInvocation +from services.tools.plugin_invocation import PluginToolBackwardsInvocation @inner_api_ns.route("/invoke/llm") @@ -240,12 +240,10 @@ class PluginInvokeToolApi(Resource): 404: "Service not available", } ) - @with_session - def post(self, session: Session, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeTool): + def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeTool): def generator(): return PluginToolBackwardsInvocation.convert_to_event_stream( PluginToolBackwardsInvocation.invoke_tool( - session=session, tenant_id=tenant_model.id, user_id=user_model.id, tool_type=ToolProviderType.value_of(payload.tool_type), @@ -253,6 +251,8 @@ def generator(): tool_name=payload.tool, tool_parameters=payload.tool_parameters, credential_id=payload.credential_id, + workflow_runtime=application_services().workflow_runtime, + draft_variable_saver=application_services().workflow_variables.saver_factory, ), ) @@ -343,6 +343,7 @@ class PluginInvokeAppApi(Resource): @with_session def post(self, session: Session, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeApp): response = PluginAppBackwardsInvocation.invoke_app( + runtime=application_services().workflow_runtime, session=session, app_id=payload.app_id, user_id=user_model.id, diff --git a/api/controllers/inner_api/runtime_credentials.py b/api/controllers/inner_api/runtime_credentials.py index c828ab07822b07..1e30c7f2a96438 100644 --- a/api/controllers/inner_api/runtime_credentials.py +++ b/api/controllers/inner_api/runtime_credentials.py @@ -22,10 +22,10 @@ from core.helper.provider_cache import ToolProviderCredentialsCache from core.helper.provider_encryption import create_provider_encrypter from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager -from core.tools.tool_manager import ToolManager from extensions.ext_database import db from models.provider import ProviderCredential from models.tools import BuiltinToolProvider +from services.tools.tool_manager import ToolManager logger = logging.getLogger(__name__) diff --git a/api/controllers/mcp/mcp.py b/api/controllers/mcp/mcp.py index 98ac0acce18216..83d772e6b572e7 100644 --- a/api/controllers/mcp/mcp.py +++ b/api/controllers/mcp/mcp.py @@ -1,3 +1,4 @@ +from functools import partial from typing import Any, Union from flask import Response, request @@ -10,12 +11,14 @@ from controllers.mcp import mcp_ns from core.mcp import types as mcp_types from core.mcp.server.streamable_http import handle_mcp_request, negotiate_protocol_version +from extensions.ext_application_services import application_services from extensions.ext_database import db from graphon.variables.input_entities import VariableEntity, VariableEntityType from libs import helper from models.enums import EndUserType from models.model import App, AppMCPServer, AppMode, EndUser from services.app.mcp_server_service import AppMCPServerStatus +from services.app_generate_service import AppGenerateService class MCPRequestError(Exception): @@ -284,5 +287,17 @@ def _handle_mcp_request( end_user = self._create_end_user(client_name, app.tenant_id, app.id, mcp_server.id, session) return handle_mcp_request( - session, app, mcp_request, user_input_form, mcp_server, end_user, request_id, protocol_version + session, + app, + mcp_request, + user_input_form, + mcp_server, + end_user, + request_id, + protocol_version, + generate=partial( + AppGenerateService.generate, + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, + ), ) diff --git a/api/controllers/openapi/app_run.py b/api/controllers/openapi/app_run.py index 6a32b062c64714..10e035f9272b84 100644 --- a/api/controllers/openapi/app_run.py +++ b/api/controllers/openapi/app_run.py @@ -10,17 +10,16 @@ from flask_restx import Resource from sqlalchemy.orm import Session -from werkzeug.exceptions import ( - BadRequest, - HTTPException, - InternalServerError, - NotFound, - TooManyRequests, - UnprocessableEntity, -) import services from constants.oauth_bearer import Scope +from controllers.common.errors import ( + InvalidRequestError, + NotFoundError, + TooManyRequestsError, + UnprocessableEntityError, + raise_unexpected_error, +) from controllers.common.fields import EventStreamResponse from controllers.common.rbac import PlainApp, RBACCheck, RBACPermission from controllers.openapi import openapi_ns @@ -67,6 +66,7 @@ ProviderTokenNotInitError, QuotaExceededError, ) +from extensions.ext_application_services import application_services from extensions.ext_redis import redis_client from graphon.graph_engine.manager import GraphEngineManager from graphon.model_runtime.errors.invoke import InvokeError @@ -95,13 +95,13 @@ def _translate_service_errors() -> Generator[None, None, None]: try: yield except WorkflowNotFoundError as ex: - raise NotFound(str(ex)) + raise NotFoundError(str(ex)) except (IsDraftWorkflowError, WorkflowIdFormatError) as ex: - raise BadRequest(str(ex)) + raise InvalidRequestError(str(ex)) except TriggerWorkflowServiceModeUnavailableServiceError: raise TriggerWorkflowServiceModeUnavailableError() except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") except services.errors.conversation.ConversationCompletedError: raise ConversationCompletedError() except services.errors.app_model_config.AppModelConfigBrokenError: @@ -117,19 +117,16 @@ def _translate_service_errors() -> Generator[None, None, None]: # App concurrency limit. Without this it falls through to the bare `except Exception` # below and surfaces as a 500. Render as the canonical 429 (code "too_many_requests"); # the source message is dropped since it carries internal detail (client_id / limits). - raise TooManyRequests() + raise TooManyRequestsError() except InvokeRateLimitError as ex: raise InvokeRateLimitHttpError(ex.description) except InvokeError as e: raise CompletionRequestError(e.description) except ValueError: logger.warning("run input refused by the service.", exc_info=True) - raise BadRequest(_INVALID_RUN_INPUT) - except HTTPException: - raise - except Exception: - logger.exception("internal server error.") - raise InternalServerError() + raise InvalidRequestError(_INVALID_RUN_INPUT) + except Exception as error: + raise_unexpected_error(error, message="App execution failed") _RUN_GUARDS: Final = ( @@ -145,6 +142,8 @@ def _translate_service_errors() -> Generator[None, None, None]: def _generate(app: App, caller: Any, args: dict[str, Any], session: Session): return AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app, user=caller, @@ -171,7 +170,7 @@ def _stream(ctx: Context, args: dict[str, Any]): def _require_mode(app: App, *modes: AppMode) -> None: if app.mode not in modes: - raise UnprocessableEntity("app_mode_mismatch") + raise UnprocessableEntityError("app_mode_mismatch") def _respond(ctx: Context, stream: Any): diff --git a/api/controllers/service_api/app/completion.py b/api/controllers/service_api/app/completion.py index be587079f28dd1..2236d1fc2e40d7 100644 --- a/api/controllers/service_api/app/completion.py +++ b/api/controllers/service_api/app/completion.py @@ -7,10 +7,10 @@ from pydantic import BaseModel, Field, field_validator from pydantic.json_schema import SkipJsonSchema from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, InternalServerError, NotFound import services from configs import dify_config +from controllers.common.errors import InternalServerError, InvalidRequestError, NotFoundError from controllers.common.fields import ChatBlockingResponse, CompletionBlockingResponse, SimpleResultResponse from controllers.common.schema import register_response_schema_models, register_schema_models from controllers.console.app.wraps import with_session @@ -35,7 +35,6 @@ ) from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError -from core.app.apps.agent_app.errors import AgentAppNotPublishedError from core.app.entities.app_invoke_entities import InvokeFrom from core.errors.error import ( ModelCurrentlyNotSupportError, @@ -44,10 +43,12 @@ ) from core.helper.trace_id_helper import get_external_trace_id, get_trace_session_id, omit_trace_session_id_from_payload from enums import CloudPlan, DeploymentEdition +from extensions.ext_application_services import application_services from graphon.model_runtime.errors.invoke import InvokeError from libs import helper from libs.helper import UUIDStrOrEmpty from models.model import App, AppMode, EndUser +from services.app.generation.errors import AgentAppNotPublishedError from services.app_generate_service import AppGenerateService from services.app_task_service import AppTaskService from services.billing_service import BillingService @@ -63,7 +64,7 @@ def _resolve_agent_app_streaming(*, app_mode: AppMode, response_mode: str | None if app_mode != AppMode.AGENT: return response_mode == "streaming" if response_mode == "blocking": - raise BadRequest("Agent App only supports streaming response mode.") + raise InvalidRequestError("Agent App only supports streaming response mode.") return True @@ -244,6 +245,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser): try: response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, @@ -255,7 +258,7 @@ def post(self, session: Session, app_model: App, end_user: EndUser): # response-contract:ignore compact_generate_response return helper.compact_generate_response(response) except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") except services.errors.conversation.ConversationCompletedError: raise ConversationCompletedError() except services.errors.app_model_config.AppModelConfigBrokenError: @@ -422,6 +425,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser): ) response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, @@ -433,13 +438,13 @@ def post(self, session: Session, app_model: App, end_user: EndUser): # response-contract:ignore compact_generate_response return helper.compact_generate_response(response) except WorkflowNotFoundError as ex: - raise NotFound(str(ex)) + raise NotFoundError(str(ex)) except IsDraftWorkflowError as ex: - raise BadRequest(str(ex)) + raise InvalidRequestError(str(ex)) except WorkflowIdFormatError as ex: - raise BadRequest(str(ex)) + raise InvalidRequestError(str(ex)) except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") except services.errors.conversation.ConversationCompletedError: raise ConversationCompletedError() except services.errors.app_model_config.AppModelConfigBrokenError: diff --git a/api/controllers/service_api/app/workflow.py b/api/controllers/service_api/app/workflow.py index ef6d2a35a57c8a..d42c0282b1cf61 100644 --- a/api/controllers/service_api/app/workflow.py +++ b/api/controllers/service_api/app/workflow.py @@ -9,10 +9,10 @@ from pydantic import BaseModel, Field, WithJsonSchema, field_validator, model_validator from pydantic.json_schema import SkipJsonSchema from sqlalchemy.orm import Session, sessionmaker -from werkzeug.exceptions import BadRequest, InternalServerError, NotFound from configs import dify_config from controllers.common.controller_schemas import WorkflowRunPayload as WorkflowRunPayloadBase +from controllers.common.errors import InternalServerError, InvalidRequestError, NotFoundError from controllers.common.fields import SimpleResultResponse, WorkflowBlockingResponse from controllers.common.schema import ( query_params_from_model, @@ -280,7 +280,7 @@ def get(self, app_model: App, workflow_run_id: str): run_id=workflow_run_id, ) if not workflow_run: - raise NotFound("Workflow run not found.") + raise NotFoundError("Workflow run not found.") return dump_response(WorkflowRunResponse, workflow_run) @@ -362,6 +362,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser): try: response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, @@ -494,6 +496,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser, workflow_id: try: response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, @@ -507,11 +511,11 @@ def post(self, session: Session, app_model: App, end_user: EndUser, workflow_id: except TriggerWorkflowServiceModeUnavailableServiceError: raise TriggerWorkflowServiceModeUnavailableError() except WorkflowNotFoundError as ex: - raise NotFound(str(ex)) + raise NotFoundError(str(ex)) except IsDraftWorkflowError as ex: - raise BadRequest(str(ex)) + raise InvalidRequestError(str(ex)) except WorkflowIdFormatError as ex: - raise BadRequest(str(ex)) + raise InvalidRequestError(str(ex)) except ProviderTokenNotInitError as ex: raise ProviderNotInitializeError(ex.description) except QuotaExceededError: diff --git a/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py b/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py index 64c5ab76006e5e..bae807e713a37f 100644 --- a/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py +++ b/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py @@ -7,13 +7,14 @@ from pydantic import BaseModel, Field, RootModel, field_validator from sqlalchemy import select from sqlalchemy.orm import Session -from werkzeug.exceptions import Forbidden, NotFound import services from controllers.common.errors import ( + AccessDeniedError, FilenameNotExistsError, FileTooLargeError, NoFileUploadedError, + NotFoundError, TooManyFilesError, ) from controllers.common.fields import WorkflowBlockingResponse @@ -29,7 +30,6 @@ from controllers.service_api.dataset.error import PipelineRunError from controllers.service_api.schema import event_stream_response, json_or_event_stream_response, multipart_file_params from controllers.service_api.wraps import DatasetApiResource -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom from core.entities.knowledge_entities import PipelineDataset, PipelineDocument from extensions.ext_application_services import application_services @@ -40,6 +40,7 @@ from models import Account from models.dataset import Dataset, Pipeline from models.engine import db +from services.app.generation.response import convert_to_event_stream from services.errors.file import UnsupportedFileTypeError from services.feature_service import FeatureService from services.file_service import FileService @@ -178,7 +179,7 @@ def get(self, tenant_id: str, dataset_id: UUID): stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str) dataset = db.session.scalar(stmt) if not dataset: - raise NotFound("Dataset not found.") + raise NotFoundError("Dataset not found.") query = query_params_from_request(DatasourcePluginsQuery) @@ -229,7 +230,7 @@ def post(self, payload: DatasourceNodeRunPayload, tenant_id: str, dataset_id: UU stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str) dataset = db.session.scalar(stmt) if not dataset: - raise NotFound("Dataset not found.") + raise NotFoundError("Dataset not found.") assert isinstance(current_user, Account) rag_pipeline_service: RagPipelineService = RagPipelineService(db.session()) @@ -243,8 +244,8 @@ def post(self, payload: DatasourceNodeRunPayload, tenant_id: str, dataset_id: UU ) # response-contract:ignore compact_generate_response return helper.compact_generate_response( - PipelineGenerator.convert_to_event_stream( - rag_pipeline_service.run_datasource_workflow_node( + convert_to_event_stream( + application_services().knowledge.pipeline_execution.run_datasource_workflow_node( pipeline=pipeline, node_id=node_id, user_inputs=datasource_node_run_api_entity.inputs, @@ -275,7 +276,7 @@ class PipelineRunApi(DatasetApiResource): "`documents`. Draft runs return `text/event-stream` for streaming mode or a workflow result JSON " "object for blocking mode." ), - 403: "`forbidden` : Forbidden.", + 403: "`forbidden` : AccessDeniedError.", 404: "`not_found` : Dataset not found.", 500: "`pipeline_run_error` : Pipeline execution failed.", }, @@ -305,19 +306,18 @@ def post(self, session: Session, tenant_id: str, dataset_id: UUID): stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str) dataset = session.scalar(stmt) if not dataset: - raise NotFound("Dataset not found.") + raise NotFoundError("Dataset not found.") payload = PipelineRunApiEntity.model_validate(service_api_ns.payload or {}) if not isinstance(current_user, Account): - raise Forbidden() + raise AccessDeniedError() rag_pipeline_service = RagPipelineService(session) pipeline = rag_pipeline_service.get_pipeline(tenant_id=tenant_id, dataset_id=dataset_id_str) try: response: dict[Any, Any] | Generator[str, Any, None] = PipelineGenerateService.generate( generator=application_services().knowledge.pipeline_generator, - session=session, pipeline=pipeline, user=current_user, args=payload.model_dump(), diff --git a/api/controllers/trigger/__init__.py b/api/controllers/trigger/__init__.py index 4f584dc4f67ea6..dbb3c511cb9254 100644 --- a/api/controllers/trigger/__init__.py +++ b/api/controllers/trigger/__init__.py @@ -1,7 +1,10 @@ from flask import Blueprint +from controllers.trigger.errors import register_error_handlers + # Create trigger blueprint bp = Blueprint("trigger", __name__, url_prefix="/triggers") +register_error_handlers(bp) # Import routes after blueprint creation to avoid circular imports from . import trigger, webhook diff --git a/api/controllers/trigger/errors.py b/api/controllers/trigger/errors.py new file mode 100644 index 00000000000000..9a88fd51215105 --- /dev/null +++ b/api/controllers/trigger/errors.py @@ -0,0 +1,25 @@ +"""Flask error responses for trigger endpoints. + +Request parsing failures retain their HTTP status and response. Route handlers +translate domain failures; unexpected failures are rendered here after unwinding +the handler, so they cannot swallow a framework-generated 400 or 413. +""" + +import logging + +from flask import Blueprint, Response, jsonify, request +from werkzeug.exceptions import HTTPException + +logger = logging.getLogger(__name__) + + +def register_error_handlers(blueprint: Blueprint) -> None: + @blueprint.errorhandler(HTTPException) + def handle_http_error(error: HTTPException) -> HTTPException: + return error + + @blueprint.errorhandler(Exception) + def handle_unexpected_error(error: Exception) -> tuple[Response, int]: + logger.exception("Trigger request failed for %s", request.path) + message = str(error) if request.endpoint == "trigger.handle_webhook" else "An internal error has occurred." + return jsonify({"error": "Internal server error", "message": message}), 500 diff --git a/api/controllers/trigger/trigger.py b/api/controllers/trigger/trigger.py index 203bfdf895d5de..bf1646dee22526 100644 --- a/api/controllers/trigger/trigger.py +++ b/api/controllers/trigger/trigger.py @@ -2,8 +2,8 @@ import re from flask import jsonify, request -from werkzeug.exceptions import NotFound +from controllers.common.errors import NotFoundError from controllers.trigger import bp from services.trigger.trigger_service import TriggerService from services.trigger.trigger_subscription_builder_service import TriggerSubscriptionBuilderService @@ -21,7 +21,7 @@ def trigger_endpoint(endpoint_id: str): """ # endpoint_id must be UUID if not UUID_MATCHER.match(endpoint_id): - raise NotFound("Invalid endpoint ID") + raise NotFoundError("Invalid endpoint ID") handling_chain = [ TriggerService.process_endpoint, TriggerSubscriptionBuilderService.process_builder_validation_endpoint, diff --git a/api/controllers/trigger/webhook.py b/api/controllers/trigger/webhook.py index 4b5e4cfc9b8806..bc0aa235518d12 100644 --- a/api/controllers/trigger/webhook.py +++ b/api/controllers/trigger/webhook.py @@ -2,14 +2,15 @@ import time from flask import jsonify, request -from werkzeug.exceptions import NotFound, RequestEntityTooLarge +from controllers.common.errors import NotFoundError, RequestBodyTooLargeError from controllers.trigger import bp from core.trigger.debug.event_bus import TriggerDebugEventBus from core.trigger.debug.events import WebhookDebugEvent, build_webhook_pool_key from enums import QuotaType from extensions.ext_application_services import application_services from services.errors.app import QuotaExceededError +from services.trigger.errors import WebhookBodyTooLargeError from services.trigger.webhook_service import RawWebhookDataDict, WebhookService logger = logging.getLogger(__name__) @@ -90,12 +91,9 @@ def handle_webhook(webhook_id: str): } ), 429 except ValueError as error: - raise NotFound(str(error)) - except RequestEntityTooLarge: - raise - except Exception as e: - logger.exception("Webhook processing failed for %s", webhook_id) - return jsonify({"error": "Internal server error", "message": str(e)}), 500 + raise NotFoundError(str(error)) from error + except WebhookBodyTooLargeError as error: + raise RequestBodyTooLargeError(str(error)) from error @bp.route("/webhook-debug/", methods=["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]) @@ -161,9 +159,6 @@ def handle_webhook_debug(webhook_id: str): return jsonify(response_data), status_code except ValueError as e: - raise NotFound(str(e)) - except RequestEntityTooLarge: - raise - except Exception as e: - logger.exception("Webhook debug processing failed for %s", webhook_id) - return jsonify({"error": "Internal server error", "message": "An internal error has occurred."}), 500 + raise NotFoundError(str(e)) from e + except WebhookBodyTooLargeError as error: + raise RequestBodyTooLargeError(str(error)) from error diff --git a/api/controllers/web/completion.py b/api/controllers/web/completion.py index 58083ed48a67bb..1e3554809c2295 100644 --- a/api/controllers/web/completion.py +++ b/api/controllers/web/completion.py @@ -3,9 +3,9 @@ from pydantic import BaseModel, Field, field_validator from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, InternalServerError, NotFound import services +from controllers.common.errors import InternalServerError, InvalidRequestError, NotFoundError from controllers.common.fields import GeneratedAppResponse, SimpleResultResponse from controllers.common.schema import register_response_schema_models, register_schema_models from controllers.console.app.wraps import with_session @@ -23,17 +23,18 @@ ) from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError from controllers.web.wraps import WebApiResource -from core.app.apps.agent_app.errors import AgentAppNotPublishedError from core.app.entities.app_invoke_entities import InvokeFrom from core.errors.error import ( ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError, ) +from extensions.ext_application_services import application_services from graphon.model_runtime.errors.invoke import InvokeError from libs import helper from libs.helper import uuid_value from models.model import App, AppMode, EndUser +from services.app.generation.errors import AgentAppNotPublishedError from services.app_generate_service import AppGenerateService from services.app_task_service import AppTaskService from services.conversation_service import ConversationService @@ -47,7 +48,7 @@ def _resolve_agent_app_streaming(*, app_mode: AppMode, response_mode: str | None if app_mode != AppMode.AGENT: return response_mode == "streaming" if response_mode == "blocking": - raise BadRequest("Agent App only supports streaming response mode.") + raise InvalidRequestError("Agent App only supports streaming response mode.") return True @@ -124,6 +125,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser): try: response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, @@ -135,7 +138,7 @@ def post(self, session: Session, app_model: App, end_user: EndUser): # response-contract:ignore compact_generate_response return helper.compact_generate_response(response) except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") except services.errors.conversation.ConversationCompletedError: raise ConversationCompletedError() except services.errors.app_model_config.AppModelConfigBrokenError: @@ -227,6 +230,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser): ) response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, @@ -238,7 +243,7 @@ def post(self, session: Session, app_model: App, end_user: EndUser): # response-contract:ignore compact_generate_response return helper.compact_generate_response(response) except services.errors.conversation.ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") except services.errors.conversation.ConversationCompletedError: raise ConversationCompletedError() except services.errors.app_model_config.AppModelConfigBrokenError: diff --git a/api/controllers/web/message.py b/api/controllers/web/message.py index 021474c52cba64..417d74bf316a26 100644 --- a/api/controllers/web/message.py +++ b/api/controllers/web/message.py @@ -4,9 +4,9 @@ from pydantic import BaseModel, Field, TypeAdapter from sqlalchemy.orm import Session -from werkzeug.exceptions import InternalServerError, NotFound from controllers.common.controller_schemas import MessageFeedbackPayload, MessageListQuery +from controllers.common.errors import InternalServerError, NotFoundError from controllers.common.fields import GeneratedAppResponse from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models from controllers.console.app.wraps import with_session @@ -15,6 +15,7 @@ from controllers.web.error import ( AppMoreLikeThisDisabledError, AppSuggestedQuestionsAfterAnswerDisabledError, + AppUnavailableError, CompletionRequestError, NotChatAppError, NotCompletionAppError, @@ -25,6 +26,7 @@ from controllers.web.wraps import WebApiResource from core.app.entities.app_invoke_entities import InvokeFrom from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError +from extensions.ext_application_services import application_services from extensions.ext_database import db from fields.conversation_fields import MessageResponseSource, ResultResponse from fields.message_fields import SuggestedQuestionsResponse, WebMessageInfiniteScrollPagination, WebMessageListItem @@ -34,6 +36,7 @@ from models.model import App, AppMode, EndUser from services.app_generate_service import AppGenerateService from services.errors.app import MoreLikeThisDisabledError +from services.errors.app_model_config import AppModelConfigBrokenError from services.errors.conversation import ConversationNotExistsError from services.errors.message import ( FirstMessageNotExistsError, @@ -99,9 +102,9 @@ def get(self, query: MessageListQuery, app_model: App, end_user: EndUser): data=items, ).model_dump(mode="json") except ConversationNotExistsError: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") except FirstMessageNotExistsError: - raise NotFound("First Message Not Exists.") + raise NotFoundError("First Message Not Exists.") @web_ns.route("/messages//feedbacks") @@ -146,7 +149,7 @@ def post(self, payload: MessageFeedbackPayload, app_model: App, end_user: EndUse session=db.session(), ) except MessageNotExistsError: - raise NotFound("Message Not Exists.") + raise NotFoundError("Message Not Exists.") return ResultResponse(result="success").model_dump(mode="json") @@ -186,6 +189,9 @@ def get( try: response = AppGenerateService.generate_more_like_this( + retrieval=application_services().workflow_runtime.retrieval, + annotations=application_services().workflow_runtime.annotation_replies, + records=application_services().workflow_runtime.chat_records, session=session, app_model=app_model, user=end_user, @@ -197,7 +203,9 @@ def get( # response-contract:ignore compact_generate_response return helper.compact_generate_response(response) except MessageNotExistsError: - raise NotFound("Message Not Exists.") + raise NotFoundError("Message Not Exists.") + except AppModelConfigBrokenError as error: + raise AppUnavailableError() from error except MoreLikeThisDisabledError: raise AppMoreLikeThisDisabledError() except ProviderTokenNotInitError as ex: @@ -248,9 +256,9 @@ def get(self, app_model: App, end_user: EndUser, message_id: UUID): ) # questions is a list of strings, not a list of Message objects except MessageNotExistsError: - raise NotFound("Message not found") + raise NotFoundError("Message not found") except ConversationNotExistsError: - raise NotFound("Conversation not found") + raise NotFoundError("Conversation not found") except SuggestedQuestionsAfterAnswerDisabledError: raise AppSuggestedQuestionsAfterAnswerDisabledError() except ProviderTokenNotInitError as ex: diff --git a/api/controllers/web/workflow.py b/api/controllers/web/workflow.py index 9dc728e2efde7b..ee47b8d861f863 100644 --- a/api/controllers/web/workflow.py +++ b/api/controllers/web/workflow.py @@ -1,9 +1,9 @@ import logging from sqlalchemy.orm import Session -from werkzeug.exceptions import InternalServerError from controllers.common.controller_schemas import WorkflowRunPayload +from controllers.common.errors import InternalServerError from controllers.common.fields import GeneratedAppResponse, SimpleResultResponse from controllers.common.schema import register_response_schema_models, register_schema_models from controllers.console.app.wraps import with_session @@ -25,6 +25,7 @@ ProviderTokenNotInitError, QuotaExceededError, ) +from extensions.ext_application_services import application_services from extensions.ext_redis import redis_client from graphon.graph_engine.manager import GraphEngineManager from graphon.model_runtime.errors.invoke import InvokeError @@ -72,6 +73,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser): try: response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, diff --git a/api/core/app/apps/agent_app/session_store.py b/api/core/app/apps/agent_app/session_store.py index 07478da10f1754..87da6e7ff009f6 100644 --- a/api/core/app/apps/agent_app/session_store.py +++ b/api/core/app/apps/agent_app/session_store.py @@ -17,6 +17,7 @@ AgentWorkspaceOwnerType, ) from models.model import App, Conversation +from repositories.agent_workspace_repository import AgentWorkspaceRepository from services.agent.workspace_service import ( AgentWorkspaceNotFoundError, AgentWorkspaceService, @@ -128,7 +129,7 @@ def _get_binding( ) if binding is None or binding.agent_id != scope.agent_id: raise AgentWorkspaceNotFoundError("Caller participant Binding is unavailable") - AgentWorkspaceService.validate_binding_generation( + AgentWorkspaceRepository.validate_binding_generation( binding, base_home_snapshot_id=scope.home_snapshot_id, agent_config_version_id=scope.agent_config_snapshot_id, diff --git a/api/core/app/layers/pause_state_persist_layer.py b/api/core/app/layers/pause_state_persist_layer.py index 5d327adb06561a..02c665623fbfc4 100644 --- a/api/core/app/layers/pause_state_persist_layer.py +++ b/api/core/app/layers/pause_state_persist_layer.py @@ -6,8 +6,6 @@ from sqlalchemy.orm import Session, sessionmaker from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity -from core.repositories.human_input_repository import HumanInputFormSubmissionRepository -from core.workflow.nodes.human_input.boundary import enrich_graph_pause_reasons from core.workflow.system_variables import SystemVariableKey, get_system_text from graphon.filters import ResponseStreamFilter from graphon.graph_engine.layers import GraphEngineLayer @@ -15,6 +13,8 @@ from models.model import AppMode from repositories.api_workflow_run_repository import APIWorkflowRunRepository from repositories.factory import DifyAPIRepositoryFactory +from repositories.human_input.form_repository import HumanInputFormSubmissionRepository +from services.workflow.execution.adapters.human_input_events import enrich_graph_pause_reasons # Wrapper types for `WorkflowAppGenerateEntity` and diff --git a/api/core/app/task_pipeline/based_generate_task_pipeline.py b/api/core/app/task_pipeline/based_generate_task_pipeline.py index 925e13f274c334..347bd46b03e829 100644 --- a/api/core/app/task_pipeline/based_generate_task_pipeline.py +++ b/api/core/app/task_pipeline/based_generate_task_pipeline.py @@ -1,9 +1,6 @@ import logging import time -from sqlalchemy import select -from sqlalchemy.orm import Session - from clients.agent_backend.errors import AgentBackendError from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.entities.app_invoke_entities import ( @@ -19,8 +16,6 @@ from core.errors.error import QuotaExceededError from core.moderation.output_moderation import ModerationRule, OutputModeration from graphon.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError -from models.enums import MessageStatus -from models.model import Message logger = logging.getLogger(__name__) @@ -47,7 +42,7 @@ def __init__( self.output_moderation_handler = self._init_output_moderation() self.stream = stream - def handle_error(self, *, event: QueueErrorEvent, session: Session | None = None, message_id: str = ""): + def handle_error(self, *, event: QueueErrorEvent): logger.debug("error: %s", event.error) e = event.error err: Exception @@ -61,17 +56,6 @@ def handle_error(self, *, event: QueueErrorEvent, session: Session | None = None description = getattr(e, "description", None) err = Exception(description if description is not None else str(e)) - if not message_id or not session: - return err - - stmt = select(Message).where(Message.id == message_id) - message = session.scalar(stmt) - if not message: - return err - - err_desc = self._error_to_desc(err) - message.status = MessageStatus.ERROR - message.error = err_desc return err def _error_to_desc(self, e: Exception) -> str: diff --git a/api/core/app/workflow/__init__.py b/api/core/app/workflow/__init__.py index 3bca7f5c34abf6..e69de29bb2d1d6 100644 --- a/api/core/app/workflow/__init__.py +++ b/api/core/app/workflow/__init__.py @@ -1,3 +0,0 @@ -from core.workflow.node_factory import DifyNodeFactory - -__all__ = ["DifyNodeFactory"] diff --git a/api/core/app/workflow/file_runtime.py b/api/core/app/workflow/file_runtime.py index 7bdabd937171aa..c4e0dc83fb3fb8 100644 --- a/api/core/app/workflow/file_runtime.py +++ b/api/core/app/workflow/file_runtime.py @@ -3,7 +3,6 @@ import base64 import hashlib import hmac -import os import time import urllib.parse from collections.abc import Generator @@ -20,6 +19,7 @@ from graphon.file.protocols import WorkflowFileRuntimeProtocol from graphon.file.runtime import set_workflow_file_runtime from graphon.http.protocols import HttpResponseProtocol +from libs.signed_query import sign_query if TYPE_CHECKING: from graphon.file import File @@ -121,11 +121,10 @@ def resolve_upload_file_uri( """Resolve a signed UploadFile URI without selecting an origin.""" self._assert_upload_file_access(upload_file_id=upload_file_id) - uri = f"/files/{upload_file_id}/file-preview" - query = self._sign_query(payload=f"file-preview|{upload_file_id}") + query = sign_query(payload=f"file-preview|{upload_file_id}", key=self._secret_key()) if as_attachment: query["as_attachment"] = "true" - return f"{uri}?{urllib.parse.urlencode(query)}" + return f"/files/{upload_file_id}/file-preview?{urllib.parse.urlencode(query)}" @override def resolve_tool_file_url(self, *, tool_file_id: str, extension: str, for_external: bool = True) -> str: @@ -164,16 +163,6 @@ def _base_url(*, for_external: bool) -> str: def _secret_key() -> bytes: return dify_config.SECRET_KEY.encode() - def _sign_query(self, *, payload: str) -> dict[str, str]: - timestamp = str(int(time.time())) - nonce = os.urandom(16).hex() - sign = hmac.new(self._secret_key(), f"{payload}|{timestamp}|{nonce}".encode(), hashlib.sha256).digest() - return { - "timestamp": timestamp, - "nonce": nonce, - "sign": base64.urlsafe_b64encode(sign).decode(), - } - def _resolve_storage_key(self, *, file: File) -> str: parsed_reference = parse_file_reference(file.reference) if parsed_reference is None: diff --git a/api/core/app/workflow/layers/__init__.py b/api/core/app/workflow/layers/__init__.py index 945f75303c7c31..8b3c2a023ac14e 100644 --- a/api/core/app/workflow/layers/__init__.py +++ b/api/core/app/workflow/layers/__init__.py @@ -1,10 +1,3 @@ -"""Workflow-level GraphEngine layers that depend on outer infrastructure.""" - from .observability import ObservabilityLayer -from .persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer -__all__ = [ - "ObservabilityLayer", - "PersistenceWorkflowInfo", - "WorkflowPersistenceLayer", -] +__all__ = ["ObservabilityLayer"] diff --git a/api/core/mcp/server/streamable_http.py b/api/core/mcp/server/streamable_http.py index 3b96c5da43eb3e..c0372ee45fdc8f 100644 --- a/api/core/mcp/server/streamable_http.py +++ b/api/core/mcp/server/streamable_http.py @@ -1,6 +1,6 @@ import json import logging -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import Any, NotRequired, TypedDict, cast from sqlalchemy.orm import Session @@ -11,7 +11,6 @@ from core.mcp import types as mcp_types from graphon.variables.input_entities import VariableEntity, VariableEntityType from models.model import App, AppMCPServer, AppMode, EndUser -from services.app_generate_service import AppGenerateService from services.errors.app import TriggerWorkflowServiceModeUnavailableError logger = logging.getLogger(__name__) @@ -67,6 +66,8 @@ def handle_mcp_request( end_user: EndUser | None = None, request_id: int | str = 1, protocol_version: str = mcp_types.DEFAULT_NEGOTIATED_VERSION, + *, + generate: Callable[..., Any], ) -> mcp_types.JSONRPCResponse | mcp_types.JSONRPCError: """ Handle MCP request and return JSON-RPC response @@ -130,7 +131,9 @@ def create_error_response( ) case mcp_types.CallToolRequest(): return create_success_response( - handle_call_tool(session, app, request, user_input_form, end_user, protocol_version) + handle_call_tool( + session, app, request, user_input_form, end_user, protocol_version, generate=generate + ) ) case mcp_types.PingRequest(): return create_success_response(handle_ping()) @@ -210,6 +213,8 @@ def handle_call_tool( user_input_form: list[VariableEntity], end_user: EndUser | None, protocol_version: str = mcp_types.DEFAULT_NEGOTIATED_VERSION, + *, + generate: Callable[..., Any], ) -> mcp_types.CallToolResult: """Handle call tool request""" request_obj = cast(mcp_types.CallToolRequest, request.root) @@ -218,7 +223,7 @@ def handle_call_tool( if not end_user: raise ValueError("End user not found") - response = AppGenerateService.generate( + response = generate( session=session, app_model=app, user=end_user, diff --git a/api/core/plugin/backwards_invocation/node.py b/api/core/plugin/backwards_invocation/node.py index 9550e499922cfb..86458869aba07f 100644 --- a/api/core/plugin/backwards_invocation/node.py +++ b/api/core/plugin/backwards_invocation/node.py @@ -34,7 +34,6 @@ def invoke_parameter_extractor( :param query: str :return: dict """ - # FIXME(-LAN-): Avoid import service into core workflow_service = WorkflowService() node_id = "1919810" node_data = ParameterExtractorNodeData( @@ -85,7 +84,6 @@ def invoke_question_classifier( :param query: str :return: dict """ - # FIXME(-LAN-): Avoid import service into core workflow_service = WorkflowService() node_id = "1919810" node_data = QuestionClassifierNodeData( diff --git a/api/core/tools/__base/tool.py b/api/core/tools/__base/tool.py index e023ce117b1379..151733bd2cd4c7 100644 --- a/api/core/tools/__base/tool.py +++ b/api/core/tools/__base/tool.py @@ -5,8 +5,6 @@ from copy import deepcopy from typing import TYPE_CHECKING, Any -from sqlalchemy.orm import Session - if TYPE_CHECKING: # pragma: no cover from models.model import File @@ -48,20 +46,19 @@ def tool_provider_type(self) -> ToolProviderType: def invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, app_id: str | None = None, message_id: str | None = None, ) -> Generator[ToolInvokeMessage]: + """Invoke a configured provider without carrying persistence resources.""" if self.runtime and self.runtime.runtime_parameters: tool_parameters.update(self.runtime.runtime_parameters) tool_parameters = self._transform_tool_parameters_type(tool_parameters) result = self._invoke( - session=session, user_id=user_id, tool_parameters=tool_parameters, conversation_id=conversation_id, @@ -100,7 +97,6 @@ def _transform_tool_parameters_type(self, tool_parameters: dict[str, Any]) -> di @abstractmethod def _invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/audio/tools/asr.py b/api/core/tools/builtin_tool/providers/audio/tools/asr.py index 44543667815b50..79a345cfbe52f7 100644 --- a/api/core/tools/builtin_tool/providers/audio/tools/asr.py +++ b/api/core/tools/builtin_tool/providers/audio/tools/asr.py @@ -2,8 +2,6 @@ from collections.abc import Generator from typing import Any, override -from sqlalchemy.orm import Session - from core.credit_usage import CreditUsageCreatedBy from core.model_context import with_credit_usage_created_by from core.model_manager import ModelManager @@ -22,7 +20,6 @@ class ASRTool(BuiltinTool): @with_credit_usage_created_by(CreditUsageCreatedBy.AUDIO) def _invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/audio/tools/tts.py b/api/core/tools/builtin_tool/providers/audio/tools/tts.py index 8350af1e040ead..55a575a332de15 100644 --- a/api/core/tools/builtin_tool/providers/audio/tools/tts.py +++ b/api/core/tools/builtin_tool/providers/audio/tools/tts.py @@ -1,8 +1,6 @@ from collections.abc import Generator from typing import Any, override -from sqlalchemy.orm import Session - from core.base.tts.audio_mime import get_model_audio_mime_type, inspect_audio_stream from core.credit_usage import CreditUsageCreatedBy from core.model_context import with_credit_usage_created_by @@ -20,7 +18,6 @@ class TTSTool(BuiltinTool): @with_credit_usage_created_by(CreditUsageCreatedBy.AUDIO) def _invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/code/tools/simple_code.py b/api/core/tools/builtin_tool/providers/code/tools/simple_code.py index 78a01115b03636..dd041ef5eb9c97 100644 --- a/api/core/tools/builtin_tool/providers/code/tools/simple_code.py +++ b/api/core/tools/builtin_tool/providers/code/tools/simple_code.py @@ -1,8 +1,6 @@ from collections.abc import Generator from typing import Any, override -from sqlalchemy.orm import Session - from core.helper.code_executor.code_executor import CodeExecutor, CodeLanguage from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage @@ -13,7 +11,6 @@ class SimpleCode(BuiltinTool): @override def _invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/time/tools/current_time.py b/api/core/tools/builtin_tool/providers/time/tools/current_time.py index e9530da9a92357..1164cc11c628c1 100644 --- a/api/core/tools/builtin_tool/providers/time/tools/current_time.py +++ b/api/core/tools/builtin_tool/providers/time/tools/current_time.py @@ -3,7 +3,6 @@ from typing import Any, override from pytz import timezone as pytz_timezone # type: ignore[import-untyped] -from sqlalchemy.orm import Session from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage @@ -13,7 +12,6 @@ class CurrentTimeTool(BuiltinTool): @override def _invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/time/tools/localtime_to_timestamp.py b/api/core/tools/builtin_tool/providers/time/tools/localtime_to_timestamp.py index b327c15e01b3b8..e88c0198c070f6 100644 --- a/api/core/tools/builtin_tool/providers/time/tools/localtime_to_timestamp.py +++ b/api/core/tools/builtin_tool/providers/time/tools/localtime_to_timestamp.py @@ -3,7 +3,6 @@ from typing import Any, cast, override import pytz # type: ignore[import-untyped] -from sqlalchemy.orm import Session from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage @@ -14,7 +13,6 @@ class LocaltimeToTimestampTool(BuiltinTool): @override def _invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/time/tools/timestamp_to_localtime.py b/api/core/tools/builtin_tool/providers/time/tools/timestamp_to_localtime.py index 489f450e6f4154..f1bf1471444a04 100644 --- a/api/core/tools/builtin_tool/providers/time/tools/timestamp_to_localtime.py +++ b/api/core/tools/builtin_tool/providers/time/tools/timestamp_to_localtime.py @@ -3,7 +3,6 @@ from typing import Any, override import pytz # type: ignore[import-untyped] -from sqlalchemy.orm import Session from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage @@ -14,7 +13,6 @@ class TimestampToLocaltimeTool(BuiltinTool): @override def _invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/time/tools/timezone_conversion.py b/api/core/tools/builtin_tool/providers/time/tools/timezone_conversion.py index 6d9eb9e57f88a4..6f00e0009e7ef1 100644 --- a/api/core/tools/builtin_tool/providers/time/tools/timezone_conversion.py +++ b/api/core/tools/builtin_tool/providers/time/tools/timezone_conversion.py @@ -3,7 +3,6 @@ from typing import Any, override import pytz # type: ignore[import-untyped] -from sqlalchemy.orm import Session from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage @@ -14,7 +13,6 @@ class TimezoneConversionTool(BuiltinTool): @override def _invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/time/tools/weekday.py b/api/core/tools/builtin_tool/providers/time/tools/weekday.py index 7cc589128ee41b..d6352fc23f4446 100644 --- a/api/core/tools/builtin_tool/providers/time/tools/weekday.py +++ b/api/core/tools/builtin_tool/providers/time/tools/weekday.py @@ -3,8 +3,6 @@ from datetime import datetime from typing import Any, override -from sqlalchemy.orm import Session - from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage @@ -13,7 +11,6 @@ class WeekdayTool(BuiltinTool): @override def _invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/webscraper/tools/webscraper.py b/api/core/tools/builtin_tool/providers/webscraper/tools/webscraper.py index 4c29ef3145b715..44c6f720d0e055 100644 --- a/api/core/tools/builtin_tool/providers/webscraper/tools/webscraper.py +++ b/api/core/tools/builtin_tool/providers/webscraper/tools/webscraper.py @@ -1,8 +1,6 @@ from collections.abc import Generator from typing import Any, override -from sqlalchemy.orm import Session - from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage from core.tools.errors import ToolInvokeError @@ -13,7 +11,6 @@ class WebscraperTool(BuiltinTool): @override def _invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/custom_tool/tool.py b/api/core/tools/custom_tool/tool.py index 0e1da00a3fc389..8a5d59656e6a1d 100644 --- a/api/core/tools/custom_tool/tool.py +++ b/api/core/tools/custom_tool/tool.py @@ -6,7 +6,6 @@ from urllib.parse import urlencode import httpx -from sqlalchemy.orm import Session from core.helper import ssrf_proxy from core.tools.__base.tool import Tool @@ -381,7 +380,6 @@ def _convert_body_property_type(self, property: dict[str, Any], value: Any): @override def _invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/mcp_tool/tool.py b/api/core/tools/mcp_tool/tool.py index 2042dec70677ed..c21c34a6f20666 100644 --- a/api/core/tools/mcp_tool/tool.py +++ b/api/core/tools/mcp_tool/tool.py @@ -68,7 +68,6 @@ def tool_provider_type(self) -> ToolProviderType: @override def _invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, @@ -290,10 +289,8 @@ def invoke_remote_mcp_tool( headers = self.headers.copy() if self.headers else {} tool_parameters = self._handle_none_parameter(tool_parameters) - from sqlalchemy.orm import Session - from extensions.ext_database import db - from services.tools.legacy_mcp_tools_manage_service import MCPToolManageService + from services.tools.mcp_tools_manage_service import MCPToolManageService # Step 1: Load provider entity and credentials in a short-lived session # This minimizes database connection hold time diff --git a/api/core/tools/plugin_tool/tool.py b/api/core/tools/plugin_tool/tool.py index 27362632c3ae09..ac17d542bc02bf 100644 --- a/api/core/tools/plugin_tool/tool.py +++ b/api/core/tools/plugin_tool/tool.py @@ -3,8 +3,6 @@ from collections.abc import Generator from typing import Any, override -from sqlalchemy.orm import Session - from core.plugin.impl.tool import PluginToolManager from core.plugin.utils.converter import convert_parameters_to_plugin_format from core.tools.__base.tool import Tool @@ -29,7 +27,6 @@ def tool_provider_type(self) -> ToolProviderType: @override def _invoke( self, - session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/workflow/nodes/agent/__init__.py b/api/core/workflow/nodes/agent/__init__.py index ba6c667194ab16..7967b3395ae830 100644 --- a/api/core/workflow/nodes/agent/__init__.py +++ b/api/core/workflow/nodes/agent/__init__.py @@ -1,4 +1,3 @@ -from .agent_node import AgentNode from .entities import AgentNodeData -__all__ = ["AgentNode", "AgentNodeData"] +__all__ = ["AgentNodeData"] diff --git a/api/core/workflow/nodes/agent/message_transformer.py b/api/core/workflow/nodes/agent/message_transformer.py index cf66c202702ab6..0352c1299b6413 100644 --- a/api/core/workflow/nodes/agent/message_transformer.py +++ b/api/core/workflow/nodes/agent/message_transformer.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Generator, Mapping +from collections.abc import Callable, Generator, Mapping from typing import Any, cast from sqlalchemy import select @@ -23,7 +23,6 @@ ) from graphon.variables.segments import ArrayFileSegment from models import ToolFile -from services.tools.legacy_builtin_tools_manage_service import BuiltinToolManageService from .events import AgentLogEvent from .exceptions import AgentNodeError, AgentVariableTypeError, ToolFileNotFoundError @@ -33,6 +32,14 @@ class AgentMessageTransformer: + def __init__( + self, + provider_icons: Callable[ + [str, str | None], tuple[str | Mapping[str, str] | None, str | Mapping[str, str] | None] + ], + ) -> None: + self._provider_icons = provider_icons + def transform( self, *, @@ -46,8 +53,6 @@ def transform( node_id: str, node_execution_id: str, ) -> Generator[NodeEventBase, None, None]: - from core.plugin.impl.plugin import PluginInstaller - message_stream = ToolFileMessageTransformer.transform_tool_invoke_messages( messages=messages, user_id=user_id, @@ -199,32 +204,7 @@ def transform( icon = tool_info.get("icon", "") dict_metadata = dict(message.message.metadata) if dict_metadata.get("provider"): - manager = PluginInstaller() - plugins = manager.list_plugins(tenant_id) - try: - current_plugin = next( - plugin - for plugin in plugins - if f"{plugin.plugin_id}/{plugin.name}" == dict_metadata["provider"] - ) - icon = current_plugin.declaration.icon - except StopIteration: - pass - icon_dark = None - try: - builtin_tool = next( - provider - for provider in BuiltinToolManageService.list_builtin_tools( - user_id, - tenant_id, - ) - if provider.name == dict_metadata["provider"] - ) - icon = builtin_tool.icon - icon_dark = builtin_tool.icon_dark - except StopIteration: - pass - + icon, icon_dark = self._provider_icons(dict_metadata["provider"], icon) dict_metadata["icon"] = icon dict_metadata["icon_dark"] = icon_dark message.message.metadata = dict_metadata diff --git a/api/core/workflow/nodes/agent_v2/__init__.py b/api/core/workflow/nodes/agent_v2/__init__.py index eee44344721542..baf5cb1d1e779b 100644 --- a/api/core/workflow/nodes/agent_v2/__init__.py +++ b/api/core/workflow/nodes/agent_v2/__init__.py @@ -1,4 +1,3 @@ -from .agent_node import DifyAgentNode from .entities import DifyAgentNodeData -__all__ = ["DifyAgentNode", "DifyAgentNodeData"] +__all__ = ["DifyAgentNodeData"] diff --git a/api/core/workflow/nodes/agent_v2/ask_human_hitl.py b/api/core/workflow/nodes/agent_v2/ask_human_hitl.py index 12981a021bc31f..629273ae65aa3c 100644 --- a/api/core/workflow/nodes/agent_v2/ask_human_hitl.py +++ b/api/core/workflow/nodes/agent_v2/ask_human_hitl.py @@ -40,10 +40,10 @@ from dify_agent.protocol import DeferredToolCallPayload from pydantic import ValidationError -from core.repositories.human_input_repository import FormCreateParams, HumanInputFormRepository from core.workflow.nodes.human_input.pause_reason import HumanInputRequired from enums.human_input import ButtonStyle, TimeoutUnit, ValueSourceType from models.agent_config_entities import AgentHumanContactConfig +from models.human_input_contracts import FormCreateParams, HumanInputFormRepository from models.human_input_delivery import ( DeliveryChannelConfig, EmailDeliveryConfig, diff --git a/api/core/workflow/nodes/agent_v2/session_store.py b/api/core/workflow/nodes/agent_v2/session_store.py index dd5f5117efe3ec..1e63fdba4dc7b1 100644 --- a/api/core/workflow/nodes/agent_v2/session_store.py +++ b/api/core/workflow/nodes/agent_v2/session_store.py @@ -19,6 +19,7 @@ AgentWorkspaceOwnerType, ) from models.workflow import WorkflowNodeExecutionModel +from repositories.agent_workspace_repository import AgentWorkspaceRepository from services.agent.workspace_service import ( AgentWorkspaceNotFoundError, AgentWorkspaceService, @@ -223,7 +224,7 @@ def load_or_create_node_execution_session( if resolved_binding is None or resolved_binding.agent_id != scope.agent_id: raise AgentWorkspaceNotFoundError("Workflow node participant Binding is unavailable") binding = resolved_binding - AgentWorkspaceService.validate_binding_generation( + AgentWorkspaceRepository.validate_binding_generation( binding, base_home_snapshot_id=home_snapshot_id, agent_config_version_id=scope.agent_config_snapshot_id, @@ -336,7 +337,7 @@ def _resolve_or_create_binding( agent_id=scope.agent_id, ) if existing_binding is not None: - AgentWorkspaceService.validate_binding_generation( + AgentWorkspaceRepository.validate_binding_generation( existing_binding, base_home_snapshot_id=home_snapshot_id, agent_config_version_id=scope.agent_config_snapshot_id, diff --git a/api/core/workflow/nodes/human_input/__init__.py b/api/core/workflow/nodes/human_input/__init__.py index 2e8f3a7036fe47..f262d24c8f6fa9 100644 --- a/api/core/workflow/nodes/human_input/__init__.py +++ b/api/core/workflow/nodes/human_input/__init__.py @@ -1,51 +1 @@ -from enums.human_input import ( - ButtonStyle, - FormInputType, - HumanInputFormKind, - HumanInputFormStatus, - TimeoutUnit, - ValueSourceType, -) -from models.human_input_entities import ( - FileInputConfig, - FileListInputConfig, - FormDefinition, - FormInputConfig, - HumanInputNodeData, - HumanInputSubmissionValidationError, - ParagraphInputConfig, - SelectInputConfig, - StringListSource, - StringSource, - UserActionConfig, - validate_human_input_submission, -) - -from .pause_reason import DifyHITLEventType, HumanInputRequired, PauseReason -from .session_binding import SessionBinding, default_session_binding - -__all__ = [ - "ButtonStyle", - "DifyHITLEventType", - "FileInputConfig", - "FileListInputConfig", - "FormDefinition", - "FormInputConfig", - "FormInputType", - "HumanInputFormKind", - "HumanInputFormStatus", - "HumanInputNodeData", - "HumanInputRequired", - "HumanInputSubmissionValidationError", - "ParagraphInputConfig", - "PauseReason", - "SelectInputConfig", - "SessionBinding", - "StringListSource", - "StringSource", - "TimeoutUnit", - "UserActionConfig", - "ValueSourceType", - "default_session_binding", - "validate_human_input_submission", -] +"""Workflow runtime adapters for the shared Human Input domain.""" diff --git a/api/core/workflow/nodes/trigger_webhook/__init__.py b/api/core/workflow/nodes/trigger_webhook/__init__.py index e41d290f6d8b58..e69de29bb2d1d6 100644 --- a/api/core/workflow/nodes/trigger_webhook/__init__.py +++ b/api/core/workflow/nodes/trigger_webhook/__init__.py @@ -1,3 +0,0 @@ -from .node import TriggerWebhookNode - -__all__ = ["TriggerWebhookNode"] diff --git a/api/events/app_event.py b/api/events/app_event.py index 2fba0028f91878..d684ce9b4d5996 100644 --- a/api/events/app_event.py +++ b/api/events/app_event.py @@ -6,9 +6,6 @@ # sender: app, kwargs: app_model_config app_model_config_was_updated = signal("app-model-config-was-updated") -# sender: app, kwargs: published_workflow -app_published_workflow_was_updated = signal("app-published-workflow-was-updated") - # sender: app, kwargs: synced_draft_workflow app_draft_workflow_was_synced = signal("app-draft-workflow-was-synced") diff --git a/api/events/event_handlers/__init__.py b/api/events/event_handlers/__init__.py index b60d86b2b54b30..eb8b68f46c9331 100644 --- a/api/events/event_handlers/__init__.py +++ b/api/events/event_handlers/__init__.py @@ -10,18 +10,10 @@ from .queue_default_plugin_install_when_tenant_created import ( handle as handle_queue_default_plugin_install_when_tenant_created, ) -from .sync_plugin_trigger_when_app_created import handle as handle_sync_plugin_trigger_when_app_created from .sync_webhook_when_app_created import handle as handle_sync_webhook_when_app_created -from .sync_workflow_schedule_when_app_published import handle as handle_sync_workflow_schedule_when_app_published from .update_app_dataset_join_when_app_model_config_updated import ( handle as handle_update_app_dataset_join_when_app_model_config_updated, ) -from .update_app_dataset_join_when_app_published_workflow_updated import ( - handle as handle_update_app_dataset_join_when_app_published_workflow_updated, -) -from .update_app_triggers_when_app_published_workflow_updated import ( - handle as handle_update_app_triggers_when_app_published_workflow_updated, -) # Consolidated handler replaces both deduct_quota_when_message_created and # update_provider_last_used_at_when_message_created @@ -36,11 +28,7 @@ "handle_delete_tool_parameters_cache_when_sync_draft_workflow", "handle_queue_credential_sync_when_tenant_created", "handle_queue_default_plugin_install_when_tenant_created", - "handle_sync_plugin_trigger_when_app_created", "handle_sync_webhook_when_app_created", - "handle_sync_workflow_schedule_when_app_published", "handle_update_app_dataset_join_when_app_model_config_updated", - "handle_update_app_dataset_join_when_app_published_workflow_updated", - "handle_update_app_triggers_when_app_published_workflow_updated", "handle_update_provider_when_message_created", ] diff --git a/api/events/event_handlers/create_installed_app_when_app_created.py b/api/events/event_handlers/create_installed_app_when_app_created.py index 7b0358364d7f1f..2dce23593ad727 100644 --- a/api/events/event_handlers/create_installed_app_when_app_created.py +++ b/api/events/event_handlers/create_installed_app_when_app_created.py @@ -1,7 +1,7 @@ from sqlalchemy.orm import Session from events.app_event import app_was_created -from services.app_creation_records import create_installed_app_record +from repositories.app.creation_records import create_installed_app_record @app_was_created.connect diff --git a/api/events/event_handlers/create_site_record_when_app_created.py b/api/events/event_handlers/create_site_record_when_app_created.py index 45eff122321b57..aa009718e1d2bf 100644 --- a/api/events/event_handlers/create_site_record_when_app_created.py +++ b/api/events/event_handlers/create_site_record_when_app_created.py @@ -1,7 +1,7 @@ from sqlalchemy.orm import Session from events.app_event import app_was_created -from services.app_creation_records import create_site_record +from repositories.app.creation_records import create_site_record @app_was_created.connect diff --git a/api/events/event_handlers/delete_tool_parameters_cache_when_sync_draft_workflow.py b/api/events/event_handlers/delete_tool_parameters_cache_when_sync_draft_workflow.py index 59c30ad0427d62..ccce4357542b72 100644 --- a/api/events/event_handlers/delete_tool_parameters_cache_when_sync_draft_workflow.py +++ b/api/events/event_handlers/delete_tool_parameters_cache_when_sync_draft_workflow.py @@ -2,12 +2,13 @@ from core.tools.entities.tool_entities import ToolProviderType from core.tools.errors import ToolProviderNotFoundError -from core.tools.tool_manager import ToolManager from core.tools.utils.configuration import ToolParameterConfigurationManager from core.workflow.human_input_adapter import adapt_node_config_for_graph from events.app_event import app_draft_workflow_was_synced +from extensions.ext_application_services import application_services from graphon.nodes import BuiltinNodeTypes from graphon.nodes.tool.entities import ToolEntity +from services.tools.tool_manager import ToolManager logger = logging.getLogger(__name__) @@ -25,6 +26,8 @@ def handle(sender, **kwargs): tool_entity = ToolEntity.model_validate(adapted_node_data["data"]) provider_type = ToolProviderType(tool_entity.provider_type.value) tool_runtime = ToolManager.get_tool_runtime( + tool_providers=application_services().tools.tool_providers, + workflow_queries=application_services().tools.workflow_queries, provider_type=provider_type, provider_id=tool_entity.provider_id, tool_name=tool_entity.tool_name, diff --git a/api/events/event_handlers/sync_webhook_when_app_created.py b/api/events/event_handlers/sync_webhook_when_app_created.py index 7fd23a8dfa4176..e34a7bc14bed5d 100644 --- a/api/events/event_handlers/sync_webhook_when_app_created.py +++ b/api/events/event_handlers/sync_webhook_when_app_created.py @@ -1,6 +1,8 @@ import logging -from events.app_event import app_draft_workflow_was_synced, app_published_workflow_was_updated +from sqlalchemy.orm import Session, sessionmaker + +from events.app_event import app_draft_workflow_was_synced from models.model import App, AppMode from models.workflow import Workflow from services.trigger.webhook_service import WebhookService @@ -9,7 +11,13 @@ @app_draft_workflow_was_synced.connect -def handle(sender, synced_draft_workflow: Workflow, **kwargs): +def handle( + sender, + synced_draft_workflow: Workflow, + *, + session_factory: sessionmaker[Session] | None = None, + **kwargs, +): """ While creating a workflow or updating a workflow, we may need to sync its webhook relationships in DB. @@ -19,14 +27,6 @@ def handle(sender, synced_draft_workflow: Workflow, **kwargs): # only handle workflow app, chatflow is not supported yet return - WebhookService.sync_webhook_relationships(app, synced_draft_workflow, remove_stale=False) - - -@app_published_workflow_was_updated.connect -def handle_published(sender, published_workflow: Workflow, **kwargs): - """Remove stale webhook relationships after their node deletion is published.""" - app: App = sender - if app.mode != AppMode.WORKFLOW.value: - return - - WebhookService.sync_webhook_relationships(app, published_workflow, remove_stale=True) + WebhookService.sync_webhook_relationships( + app, synced_draft_workflow, remove_stale=False, session_factory=session_factory + ) diff --git a/api/extensions/application_services/agent.py b/api/extensions/application_services/agent.py index d88ddff4b3a8fc..f94025df593c79 100644 --- a/api/extensions/application_services/agent.py +++ b/api/extensions/application_services/agent.py @@ -7,29 +7,36 @@ from configs import dify_config from core.app.file_access import DatabaseFileAccessController +from extensions.application_services.workflow import build_workflow_execution_dependencies from repositories.app.agent_app_repository import AgentAppRepository from repositories.app.agent_sandbox_repository import AgentSandboxRepository from repositories.app.generation_repository import AppGenerationRepository from repositories.tools.provider_repository import ToolProviderRepository from services.agent.log_file_gateway import AgentLogFileGateway from services.agent.log_service import AgentLogService +from services.agent.tool_invocation_service import AgentToolInnerService from services.app.agent_app_feature_gateway import AgentAppFeatureValidator from services.app.agent_app_sandbox_service import AgentAppSandboxService from services.app.agent_app_service import AgentAppAccessService, AgentAppFeatureConfigService from services.app.agent_sandbox_file_gateway import AgentSandboxFileGateway, create_sandbox_client from services.file_request_service import FileRequestService +from services.tools.agent_invocation_gateway import AgentToolInvocationGateway from services.tools.tool_manager import ToolManager +from services.workflow.variable_contracts import WorkflowExecutionVariables @dataclass(frozen=True, slots=True) class AgentAppServices: - logs: AgentLogService access: AgentAppAccessService features: AgentAppFeatureConfigService sandbox: AgentAppSandboxService + tools: AgentToolInnerService + logs: AgentLogService -def build_agent_app_services(*, database_client: sessionmaker[Session]) -> AgentAppServices: +def build_agent_app_services( + *, database_client: sessionmaker[Session], variables: WorkflowExecutionVariables +) -> AgentAppServices: apps = AgentAppRepository(session_factory=database_client) return AgentAppServices( logs=AgentLogService( @@ -37,6 +44,12 @@ def build_agent_app_services(*, database_client: sessionmaker[Session]) -> Agent icons=partial(ToolManager.get_tool_icon, tool_providers=ToolProviderRepository(database_client)), files=AgentLogFileGateway(sessions=database_client, access=DatabaseFileAccessController()), ), + tools=AgentToolInnerService( + apps=apps, + tools=AgentToolInvocationGateway( + variables=variables, runtime=build_workflow_execution_dependencies(database_client) + ), + ), access=AgentAppAccessService(references=apps), features=AgentAppFeatureConfigService(features=apps, validator=AgentAppFeatureValidator()), sandbox=AgentAppSandboxService( diff --git a/api/extensions/application_services/app.py b/api/extensions/application_services/app.py index 0ac0bc7edefcac..59525fbcd8fc50 100644 --- a/api/extensions/application_services/app.py +++ b/api/extensions/application_services/app.py @@ -1,9 +1,13 @@ +from models.tool_runtime_contracts import WorkflowToolQueries +from services.tools.provider_queries import ToolProviders + """Composition of App use cases.""" from dataclasses import dataclass from sqlalchemy.orm import Session, sessionmaker +from extensions.application_services.workflow import build_app_dsl_service from repositories.account.repository import SQLAlchemyAccountRepository from repositories.app.api_key_repository import AppApiKeyRepository from repositories.app.console_repository import ConsoleAppRepository @@ -16,7 +20,6 @@ from services.app.creators_platform_gateway import CreatorsPlatformGateway from services.app.import_service import AppImportService from services.app.query_service import AppQueryService -from services.app_dsl_service import AppDslService from services.app_package_service import AppPackageService from services.app_tracing_config_gateway import OpsTraceManagerGateway from services.oauth_server_service import OAuthServerService @@ -34,12 +37,14 @@ def build_app_services( *, database_client: sessionmaker[Session], oauth: OAuthServerService, + tool_providers: ToolProviders, + workflow_queries: WorkflowToolQueries, recommended_packages: RecommendedAppPackageService, ) -> AppServices: repository = ConsoleAppRepository(session_factory=database_client) transfers = AppTransferGateway( session_factory=database_client, - dsl_factory=AppDslService, + dsl_factory=build_app_dsl_service, packages=AppPackageService(), agent_packages=RosterAgentPackageExporter(), agent_importer=RosterAgentPackageImporter(), @@ -53,7 +58,9 @@ def build_app_services( transfers=transfers, creators=CreatorsPlatformGateway(oauth=oauth), tracing=OpsTraceManagerGateway(), - lifecycle=AppLifecycleGateway(session_factory=database_client), + lifecycle=AppLifecycleGateway( + session_factory=database_client, tool_providers=tool_providers, workflow_queries=workflow_queries + ), ), queries=AppQueryService(apps=repository), ) diff --git a/api/extensions/application_services/installed_app.py b/api/extensions/application_services/installed_app.py index 84f1e46d88a82d..f666e315ed4fe5 100644 --- a/api/extensions/application_services/installed_app.py +++ b/api/extensions/application_services/installed_app.py @@ -5,6 +5,7 @@ from flask import current_app from sqlalchemy.orm import Session, sessionmaker +from extensions.application_services.workflow import build_workflow_execution_dependencies from repositories.installed_app_conversation_repository import SQLAlchemyInstalledAppConversationRepository from repositories.installed_app_message_repository import SQLAlchemyInstalledAppMessageRepository from repositories.installed_app_repository import SQLAlchemyInstalledAppRepository @@ -17,6 +18,7 @@ from services.installed_app_message_service import InstalledAppMessageService from services.installed_app_service import InstalledAppService, WorkspaceRoleLookup from services.webapp_access_query_service import WebAppAccessQueryService +from services.workflow.variable_contracts import WorkflowExecutionVariables @dataclass(frozen=True, slots=True) @@ -46,6 +48,7 @@ def build_installed_app_services( webapp_access: WebAppAccessQueryService, get_workspace_role: WorkspaceRoleLookup, webapp_auth_enabled: bool, + variables: WorkflowExecutionVariables, ) -> InstalledAppServices: installed_apps = SQLAlchemyInstalledAppRepository(session_factory=database_client) access = InstalledAppAccessService( @@ -64,7 +67,11 @@ def build_installed_app_services( ), generation=InstalledAppGenerationService( usage=installed_apps, - runtime=AppGenerateServiceRuntime(session_factory=database_client), + runtime=AppGenerateServiceRuntime( + session_factory=database_client, + variables=variables, + runtime=build_workflow_execution_dependencies(database_client), + ), ), conversations=InstalledAppConversationService( conversations=SQLAlchemyInstalledAppConversationRepository(session_factory=database_client), diff --git a/api/extensions/application_services/knowledge.py b/api/extensions/application_services/knowledge.py index e6562cbcaa4e1b..a907ebe6a2b406 100644 --- a/api/extensions/application_services/knowledge.py +++ b/api/extensions/application_services/knowledge.py @@ -1,15 +1,18 @@ """Composition root for dataset-controller application services.""" from dataclasses import dataclass +from functools import partial from uuid import uuid4 from sqlalchemy.orm import Session, sessionmaker from configs import dify_config -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.rag.extractor.entity.datasource_type import DatasourceType +from core.repositories.factory import DifyCoreRepositoryFactory from extensions.application_services.retrieval import build_dataset_retrieval +from extensions.application_services.workflow import build_workflow_execution_dependencies from libs.helper import generate_text_hash +from repositories.factory import DifyAPIRepositoryFactory from repositories.knowledge.dataset_api_key_repository import DatasetApiKeyRepository from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository @@ -17,6 +20,8 @@ from repositories.knowledge.retrieval_repository import KnowledgeRetrievalRepository from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository +from repositories.workflow.definition_repository import WorkflowDefinitionRepository +from repositories.workflow.pipeline_publication_repository import PipelinePublicationRepository from services.api_token_service import ApiTokenCache from services.app.query_service import AppQueryService from services.data_source.credential_gateway import ( @@ -53,7 +58,12 @@ from services.knowledge.segments.application import DatasetSegmentApplicationService from services.knowledge.segments.indexing import SegmentIndexingGateway from services.knowledge_retrieval_inner_service import InnerKnowledgeRetrievalService +from services.rag_pipeline.execution_service import RagPipelineExecutionService +from services.rag_pipeline.publication_gateway import PipelineIndexUpdateGateway, PipelinePublicationModelGateway +from services.rag_pipeline.publication_service import PipelinePublicationService from services.tag_application_service import TagTargetQuery +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator +from services.workflow.variable_contracts import WorkflowExecutionVariables from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task from tasks.delete_segment_from_index_task import delete_segment_from_index_task from tasks.disable_segments_from_index_task import disable_segments_from_index_task @@ -72,6 +82,8 @@ class KnowledgeServices: indexing_estimates: IndexingEstimateApplicationService segments: DatasetSegmentApplicationService pipeline_generator: PipelineGenerator + pipeline_execution: RagPipelineExecutionService + pipeline_publication: PipelinePublicationService def build_dataset_api_key_service( @@ -97,6 +109,7 @@ def build_knowledge_services( redis: RedisSegmentClient, tags: TagTargetQuery, app_queries: AppQueryService, + variables: WorkflowExecutionVariables, ) -> KnowledgeServices: """Build the dataset-controller knowledge use cases.""" @@ -148,7 +161,28 @@ def build_knowledge_services( operations=SQLAlchemyDocumentOperations(session_factory=database_client), metadata_schema=DocumentService.DOCUMENT_METADATA_SCHEMA, ), - pipeline_generator=PipelineGenerator(documents=documents, datasource_providers=providers), + pipeline_generator=PipelineGenerator( + runtime=build_workflow_execution_dependencies(database_client), + documents=documents, + datasource_providers=providers, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ), + pipeline_publication=PipelinePublicationService( + PipelinePublicationRepository(database_client), + PipelinePublicationModelGateway(), + PipelineIndexUpdateGateway(), + ), + pipeline_execution=RagPipelineExecutionService( + runtime=build_workflow_execution_dependencies(database_client), + workflows=WorkflowDefinitionRepository(session_factory=database_client), + variables=variables, + executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(database_client), + writer_factory=partial( + DifyCoreRepositoryFactory.create_workflow_node_execution_repository, session_factory=database_client + ), + documents=documents, + ), document_sync=DocumentSyncApplicationService( dataset_access=dataset_access, documents=documents, diff --git a/api/extensions/application_services/snippets.py b/api/extensions/application_services/snippets.py new file mode 100644 index 00000000000000..b7a4021e31f138 --- /dev/null +++ b/api/extensions/application_services/snippets.py @@ -0,0 +1,65 @@ +"""Composition of Snippet use cases and their workflow Agent dependencies.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy.orm import Session, sessionmaker + +from extensions.application_services.agent_bindings import build_workflow_agent_service +from extensions.application_services.workflow import build_workflow_execution_dependencies +from services.agent.workflow_resources_gateway import WorkflowAgentSkillReader +from services.skill_management_service import SkillManagementService +from services.workflow.variable_contracts import WorkflowExecutionVariables + +if TYPE_CHECKING: + from services.snippet_dsl_service import SnippetDslService + from services.snippet_generate_service import SnippetGenerateService + from services.snippet_service import SnippetService + + +def build_snippet_generation_service( + *, database_client: sessionmaker[Session], variables: WorkflowExecutionVariables +) -> SnippetGenerateService: + from services.snippet_generate_service import SnippetGenerateService + from services.workflow_service import WorkflowService + + return SnippetGenerateService( + runtime=build_workflow_execution_dependencies(database_client), + snippets=build_snippet_service(database_client), + variables=variables, + workflows=WorkflowService(database_client, runtime=build_workflow_execution_dependencies(database_client)), + ) + + +def build_snippet_service( + session_maker: sessionmaker[Session] | Session | None = None, session: Session | None = None +) -> SnippetService: + from services.snippet_service import SnippetService + + if isinstance(session_maker, Session): + session = session_maker + session_maker = None + if session is not None: + session_maker = sessionmaker(bind=session.get_bind(), expire_on_commit=False) + if session_maker is None: + raise ValueError("SnippetService requires a session or session_maker.") + return SnippetService( + session_maker=session_maker, + session=session, + agent_bindings=build_workflow_agent_service, + skills=WorkflowAgentSkillReader(SkillManagementService(session_maker=session_maker)), + ) + + +def build_snippet_dsl_service(session: Session) -> SnippetDslService: + from extensions.application_services.workflow import build_workflow_drafts + from services.snippet_dsl_service import SnippetDslService + + return SnippetDslService( + session, + snippets=build_snippet_service(session=session), + drafts=build_workflow_drafts( + sessionmaker(bind=session.get_bind(), expire_on_commit=False), import_session=session + ), + ) diff --git a/api/extensions/application_services/trial_app.py b/api/extensions/application_services/trial_app.py index be28822cf8dbd1..12cf05036a4e5f 100644 --- a/api/extensions/application_services/trial_app.py +++ b/api/extensions/application_services/trial_app.py @@ -4,11 +4,13 @@ from sqlalchemy.orm import Session, sessionmaker +from extensions.application_services.workflow import build_workflow_execution_dependencies from repositories.trial_app_repository import TrialAppRepository from services.trial_app_access_service import TrialAppAccessService from services.trial_app_generation_adapters import AppGenerateServiceRuntime from services.trial_app_generation_service import TrialAppGenerationService from services.trial_app_usage import TrialAppUsageRecorder +from services.workflow.variable_contracts import WorkflowExecutionVariables @dataclass(frozen=True, slots=True) @@ -19,12 +21,17 @@ class TrialAppServices: def build_trial_app_services( - *, database_client: sessionmaker[Session], trial_apps: TrialAppRepository + *, database_client: sessionmaker[Session], trial_apps: TrialAppRepository, variables: WorkflowExecutionVariables ) -> TrialAppServices: return TrialAppServices( access=TrialAppAccessService(apps=trial_apps), generation=TrialAppGenerationService( - runtime=AppGenerateServiceRuntime(session_factory=database_client), usage=trial_apps + runtime=AppGenerateServiceRuntime( + session_factory=database_client, + variables=variables, + runtime=build_workflow_execution_dependencies(database_client), + ), + usage=trial_apps, ), usage=trial_apps, ) diff --git a/api/extensions/application_services/workflow.py b/api/extensions/application_services/workflow.py index 8a982e24ecf54f..64d787d1d24786 100644 --- a/api/extensions/application_services/workflow.py +++ b/api/extensions/application_services/workflow.py @@ -4,9 +4,11 @@ from collections.abc import Callable, Sequence from dataclasses import dataclass +from typing import TYPE_CHECKING from sqlalchemy.orm import Session, sessionmaker +from extensions.ext_redis import RedisClientWrapper from models.agent_runtime_contracts import WorkflowAgentRuntimeBindings from models.annotation_reply import AnnotationReplies from models.human_input_contracts import HumanInputFormFactory @@ -26,6 +28,7 @@ PauseReasonResolver, WorkflowExecutionLogs, ) +from services.workflow.variable_contracts import WorkflowExecutionVariables @dataclass(frozen=True) @@ -113,6 +116,145 @@ def build_workflow_execution_dependencies(database_client: sessionmaker[Session] ) +if TYPE_CHECKING: + from services.agent.workflow_contracts import WorkflowAgentBindingStore + from services.app_dsl_service import AppDslService + from services.workflow.console_service import ConsoleWorkflowService + from services.workflow.definition_gateway import ( + WorkflowDefinitionGateway, + WorkflowDefinitionReader, + WorkflowDraftReader, + ) + from services.workflow.draft_service import DraftDefinitions, DraftLifecycle, WorkflowDraftService + from services.workflow_service import WorkflowService + + +def build_console_workflow_service( + *, database_client: sessionmaker[Session], redis: RedisClientWrapper, variables: WorkflowExecutionVariables +) -> ConsoleWorkflowService[WorkflowAgentBindingStore]: + from core.app.file_access import DatabaseFileAccessController + from core.helper.encrypter import decrypt_token + from graphon.graph_engine.manager import GraphEngineManager + from repositories.app.console_repository import ConsoleAppRepository + from repositories.workflow.collaboration_repository import WorkflowCollaborationRepository + from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository + from repositories.workflow.definition_repository import WorkflowDefinitionRepository + from repositories.workflow.draft_repository import WorkflowDraftRepository + from repositories.workflow.node_execution_repository import WorkflowNodeExecutionRepository + from services.agent.workflow_publish_service import WorkflowAgentPublishService + from services.app.console_gateway import EnterpriseConsoleAppAccess + from services.app_generate_service import AppGenerateService + from services.app_service import AppService + from services.workflow.access_gateway import WorkflowAccessGateway + from services.workflow.console_service import ConsoleWorkflowService + from services.workflow.conversion_service import WorkflowConversionService + from services.workflow.runtime_gateway import WorkflowRuntimeGateway + from services.workflow_service import WorkflowService + + workflows = WorkflowService( + session_maker=database_client, runtime=build_workflow_execution_dependencies(database_client) + ) + + definitions = WorkflowDefinitionRepository(session_factory=database_client) + drafts = WorkflowDraftRepository(database_client) + lifecycle = build_workflow_definition_gateway(database_client, definitions, workflows, drafts=drafts) + apps = ConsoleAppRepository(session_factory=database_client) + return ConsoleWorkflowService( + conversion=WorkflowConversionService( + apps, decrypt_token=decrypt_token, notify_created=AppService.notify_created_app + ), + agent_services=WorkflowAgentPublishService, + definitions=definitions, + drafts=build_workflow_draft_service(drafts, lifecycle, database_client), + lifecycle=lifecycle, + runtime=WorkflowRuntimeGateway( + runtime=build_workflow_execution_dependencies(database_client), + variables=variables, + session_factory=database_client, + workflows=workflows, + definitions=definitions, + reservations=WorkflowDebugReservationRepository(database_client), + executions=WorkflowNodeExecutionRepository(database_client), + generator=AppGenerateService, + graph_engine=GraphEngineManager(redis), + file_access=DatabaseFileAccessController(), + ), + apps=apps, + presence=WorkflowCollaborationRepository(redis=redis), + access=WorkflowAccessGateway( + session_factory=database_client, apps=EnterpriseConsoleAppAccess(session_factory=database_client) + ), + ) + + +def build_app_dsl_service(session: Session) -> AppDslService: + from repositories.app.dsl_repository import AppDslOverwriteRepository + from services.app_dsl_service import AppDslService + + sessions = sessionmaker(bind=session.get_bind(), expire_on_commit=False) + return AppDslService( + session, + drafts=build_workflow_drafts(sessions, import_session=session), + overwrites=AppDslOverwriteRepository(sessions=sessions, import_session=session), + ) + + +def build_workflow_definition_gateway( + database_client: sessionmaker[Session], + definitions: WorkflowDefinitionReader, + workflows: WorkflowService, + *, + drafts: WorkflowDraftReader, +) -> WorkflowDefinitionGateway: + from repositories.credentials.query_repository import CredentialQueryRepository + from services.agent.workflow_resources_gateway import WorkflowAgentSkillReader + from services.skill_management_service import SkillManagementService + from services.workflow.definition_gateway import WorkflowDefinitionGateway + + return WorkflowDefinitionGateway( + session_factory=database_client, + definitions=definitions, + drafts=drafts, + workflows=workflows, + credentials=CredentialQueryRepository(session_factory=database_client), + skills=WorkflowAgentSkillReader(SkillManagementService(session_maker=database_client)), + ) + + +def build_workflow_draft_service( + definitions: DraftDefinitions[WorkflowAgentBindingStore], lifecycle: DraftLifecycle, sessions: sessionmaker[Session] +) -> WorkflowDraftService[WorkflowAgentBindingStore]: + from repositories.agent.retirement_repository import WorkflowAgentRetirementRepository + from services.agent.retirement_service import WorkflowAgentRetirementService + from services.agent.workflow_publish_service import WorkflowAgentPublishService + from services.workflow.draft_service import WorkflowDraftService + + return WorkflowDraftService( + agent_services=WorkflowAgentPublishService, + definitions=definitions, + lifecycle=lifecycle, + retirement=WorkflowAgentRetirementService(WorkflowAgentRetirementRepository(sessions)), + ) + + +def build_workflow_drafts( + database_client: sessionmaker[Session], *, import_session: Session | None = None +) -> WorkflowDraftService[WorkflowAgentBindingStore]: + from repositories.workflow.definition_repository import WorkflowDefinitionRepository + from repositories.workflow.draft_repository import WorkflowDraftRepository + from services.workflow_service import WorkflowService + + definitions = WorkflowDefinitionRepository(session_factory=database_client, import_session=import_session) + drafts = WorkflowDraftRepository(database_client, import_session=import_session) + lifecycle = build_workflow_definition_gateway( + database_client, + definitions, + WorkflowService(database_client, runtime=build_workflow_execution_dependencies(database_client)), + drafts=drafts, + ) + return build_workflow_draft_service(drafts, lifecycle, database_client) + + def build_workflow_suggestions(database_client: sessionmaker[Session]): from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository from services.workflow.generation.suggestions import WorkflowInstructionSuggestions diff --git a/api/extensions/ext_application_services.py b/api/extensions/ext_application_services.py index 016d414d04cf18..04c442885f526f 100644 --- a/api/extensions/ext_application_services.py +++ b/api/extensions/ext_application_services.py @@ -1,3 +1,5 @@ +from services.workflow.generation.suggestions import WorkflowInstructionSuggestions + """Composition root for application services used by transport adapters.""" import json @@ -29,15 +31,22 @@ build_data_source_services, ) from extensions.application_services.datasets import build_dataset_dependencies +from extensions.application_services.human_input import build_human_input_debug_service from extensions.application_services.installed_app import InstalledAppServices, build_installed_app_services from extensions.application_services.knowledge import ( KnowledgeServices, build_dataset_api_key_service, build_knowledge_services, ) +from extensions.application_services.snippets import build_snippet_generation_service from extensions.application_services.tools import ToolServices, build_tool_services from extensions.application_services.trial_app import TrialAppServices, build_trial_app_services -from extensions.application_services.workflow import build_workflow_execution_dependencies, build_workflow_suggestions +from extensions.application_services.workflow import ( + build_console_workflow_service, + build_workflow_drafts, + build_workflow_execution_dependencies, + build_workflow_suggestions, +) from extensions.application_services.workflow_variables import ( build_console_workflow_variables, build_workflow_variable_service, @@ -55,6 +64,7 @@ from repositories.account.repository import SQLAlchemyAccountRepository from repositories.account_activation_repository import SQLAlchemyAccountActivationRepository from repositories.account_integration_repository import SQLAlchemyAccountIntegrationRepository +from repositories.agent.runtime_repository import WorkflowAgentBindingResolver from repositories.app.mcp_server_repository import AppMCPServerRepository from repositories.app.site_command_repository import AppSiteCommandRepository from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository @@ -81,7 +91,7 @@ from repositories.upload_file_delivery_repository import UploadFileDeliveryQueryRepository from repositories.web_passport_repository import WebPassportRepository from repositories.webapp_access_query_repository import WebAppAccessQueryRepository -from repositories.workflow_app_log_query_repository import WorkflowAppLogQueryRepository +from repositories.workflow.app_log_repository import WorkflowAppLogRepository from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepository from repositories.workspace.workspace_repository import WorkspaceRepository from services.account.adapters import ( @@ -98,6 +108,7 @@ from services.account.service import AccountSetupProvisioner from services.account_password_hasher import DefaultAccountPasswordHasher from services.agent.roster_package_exporter import RosterAgentPackageExporter +from services.agent.workflow_contracts import WorkflowAgentBindingStore from services.app.advanced_prompt_template_service import AdvancedPromptTemplateService from services.app.api_key_service import AppApiKeyService from services.app.generation.runtime import AppGenerationRuntime @@ -128,6 +139,7 @@ from services.file_grant_gateways import FileGrantFileGateway, FileGrantRemoteFileGateway, FileGrantTokenGateway from services.file_grant_service import FileGrantService from services.file_service import FileService +from services.human_input.debug_service import HumanInputDebugService from services.human_input_file_upload_service import HumanInputFileUploadService from services.init_validation_service import InitValidationService from services.inner_mail_service import InnerMailService @@ -172,6 +184,7 @@ from services.schema_definition_service import SchemaDefinitionService from services.setup_adapters import RedisSetupLock from services.setup_service import SetupService +from services.snippet_generate_service import SnippetGenerateService from services.step_by_step_tour_service import StepByStepTourService from services.system_feature_service import SystemFeatureService from services.tag_application_service import TagApplicationService @@ -192,10 +205,11 @@ from services.web_passport_service import WebPassportService from services.webapp_access_adapters import EnterpriseWebAppAccessPolicyGateway from services.webapp_access_query_service import WebAppAccessQueryService, WebAppAccessUnavailableError +from services.workflow.app_log_query_service import WorkflowAppLogQueryService +from services.workflow.console_service import ConsoleWorkflowService from services.workflow.console_variable_service import ConsoleWorkflowVariableService -from services.workflow.generation.suggestions import WorkflowInstructionSuggestions +from services.workflow.draft_service import WorkflowDraftService from services.workflow.variable_service import WorkflowVariableService -from services.workflow_app_log_query_service import WorkflowAppLogQueryService from services.workflow_run_service import WorkflowRunService from services.workflow_statistic_query_service import WorkflowStatisticQueryService from tasks.mail_inner_task import enqueue_inner_mail @@ -248,11 +262,16 @@ class AppScopedEndUserServices: @dataclass(frozen=True, slots=True) class ApplicationServices: + human_input_debug: HumanInputDebugService + console_workflows: ConsoleWorkflowService[WorkflowAgentBindingStore] + workflow_drafts: WorkflowDraftService[WorkflowAgentBindingStore] + snippet_generation: SnippetGenerateService tools: ToolServices + workflow_suggestions: WorkflowInstructionSuggestions workflow_runtime: AppGenerationRuntime workflow_variables: WorkflowVariableService console_workflow_variables: ConsoleWorkflowVariableService - workflow_suggestions: WorkflowInstructionSuggestions + workflow_agent_bindings: WorkflowAgentBindingResolver agent_apps: AgentAppServices advanced_prompt_templates: AdvancedPromptTemplateService credential_queries: CredentialQuery @@ -441,13 +460,18 @@ def build_application_services( providers=datasource_credentials.providers, ) oauth_server = _build_oauth_server_service(database_client=database_client, redis=redis) + tools = build_tool_services(database_client) apps = build_app_services( database_client=database_client, oauth=oauth_server, recommended_packages=recommended_app_packages, + tool_providers=tools.tool_providers, + workflow_queries=tools.workflow_queries, ) tags = TagApplicationService(tags=TagRepository(session_factory=database_client)) + workflow_variables = build_workflow_variable_service(database_client=database_client) knowledge = build_knowledge_services( + variables=workflow_variables, database_client=database_client, dataset_access=dataset_dependencies.access, datasets=dataset_dependencies.datasets, @@ -505,25 +529,35 @@ def build_application_services( registration=account_services.lifecycle, invitation_tokens=invitation_tokens, ) - workflow_variables = build_workflow_variable_service(database_client=database_client) return ApplicationServices( - tools=build_tool_services(database_client), - workflow_runtime=build_workflow_execution_dependencies(database_client), + tools=tools, + human_input_debug=build_human_input_debug_service(database_client=database_client), + console_workflows=build_console_workflow_service( + database_client=database_client, redis=redis, variables=workflow_variables + ), + workflow_drafts=build_workflow_drafts(database_client), workflow_variables=workflow_variables, console_workflow_variables=build_console_workflow_variables( database_client=database_client, variables=workflow_variables ), + workflow_runtime=build_workflow_execution_dependencies(database_client), workflow_suggestions=build_workflow_suggestions(database_client), + workflow_agent_bindings=WorkflowAgentBindingResolver(database_client), + snippet_generation=build_snippet_generation_service( + database_client=database_client, variables=workflow_variables + ), accounts=account_services, apps=apps, credential_queries=CredentialQueryRepository(session_factory=database_client), - agent_apps=build_agent_app_services(database_client=database_client), + agent_apps=build_agent_app_services(database_client=database_client, variables=workflow_variables), advanced_prompt_templates=AdvancedPromptTemplateService(), app_definitions=app_definitions, app_mcp_servers=AppMCPServerService( servers=AppMCPServerRepository(session_factory=database_client), ), - app_preview_details=AppPreviewDetailsRuntime(details=app_preview_repository), + app_preview_details=AppPreviewDetailsRuntime( + details=app_preview_repository, tool_providers=tools.tool_providers, workflow_queries=tools.workflow_queries + ), app_previews=AppPreviewQueryService( apps=app_preview_repository, is_previewable=recommended_app_queries.is_previewable, @@ -583,6 +617,7 @@ def build_application_services( private_app_access_enabled=deployment_edition == DeploymentEdition.ENTERPRISE, ), installed_apps=build_installed_app_services( + variables=workflow_variables, database_client=database_client, webapp_access=webapp_access, get_workspace_role=workspace_repository.get_account_role, @@ -671,7 +706,9 @@ def build_application_services( ), app_tasks=AppTaskControlService(redis_client=redis), app_audio=AppAudioRuntime(session_factory=database_client), - trial_apps=build_trial_app_services(database_client=database_client, trial_apps=trial_apps), + trial_apps=build_trial_app_services( + database_client=database_client, trial_apps=trial_apps, variables=workflow_variables + ), workflow_run_archives=WorkflowRunArchiveService( bundles=WorkflowRunArchiveBundleQueryRepository(session_factory=database_client), tasks=WorkflowRunArchiveDownloadTaskCache(redis=redis), @@ -681,10 +718,11 @@ def build_application_services( workflow_runs=WorkflowRunService( workflow_runs=workflow_run_repository, node_executions=workflow_node_execution_repository, + tool_providers=tools.tool_providers, ), workspaces=workspace_services, workflow_app_logs=WorkflowAppLogQueryService( - logs=WorkflowAppLogQueryRepository(session_factory=database_client), + logs=WorkflowAppLogRepository(session_factory=database_client), ), inner_mail=InnerMailService(dispatch=enqueue_inner_mail), web_passport=WebPassportService( diff --git a/api/extensions/workflow_warm_shutdown.py b/api/extensions/workflow_warm_shutdown.py index e6579ccdb5d898..c0722f352e81f7 100644 --- a/api/extensions/workflow_warm_shutdown.py +++ b/api/extensions/workflow_warm_shutdown.py @@ -6,7 +6,7 @@ from celery.signals import worker_shutdown, worker_shutting_down -from core.app.apps.workflow.active_workflow_tasks import ( +from services.workflow.execution.adapters.workflow.active_workflow_tasks import ( get_active_workflow_task_count, reset_active_workflow_tasks, ) diff --git a/api/fields/agent_fields.py b/api/fields/agent_fields.py index 0f1542bc2567b7..84668682efeee7 100644 --- a/api/fields/agent_fields.py +++ b/api/fields/agent_fields.py @@ -3,6 +3,7 @@ from pydantic import Field, field_validator +from enums.agent import WorkflowAgentBindingType from fields.base import ResponseModel from libs.helper import to_timestamp from models.agent import ( @@ -13,7 +14,6 @@ AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, ) from models.agent_config_entities import ( AgentCliToolConfig, diff --git a/api/fields/workflow_run_fields.py b/api/fields/workflow_run_fields.py index 4b69c1a107a33f..05efe9f709ccc0 100644 --- a/api/fields/workflow_run_fields.py +++ b/api/fields/workflow_run_fields.py @@ -17,6 +17,9 @@ from fields.end_user_fields import SimpleEndUser from fields.member_fields import SimpleAccount from libs.helper import to_timestamp +from models.workflow import WorkflowNodeExecutionModel +from services.tools.provider_queries import ToolProviderIcons +from services.workflow.node_execution_extras import node_execution_extras class WorkflowRunForLogResponse(ResponseModel): @@ -212,6 +215,16 @@ def workflow_run_pagination_response_source(pagination: Any, *, session: Session class WorkflowNodeExecutionResponseSource(SessionResponseSource[Any]): """Expose session-backed node-execution accessors during response validation.""" + def __init__(self, source: Any, *, session: Session, tool_providers: ToolProviderIcons): + super().__init__(source, session=session) + self._tool_providers = tool_providers + + @property + def extras(self) -> Any: + if isinstance(self._source, WorkflowNodeExecutionModel): + return node_execution_extras(self._source, tool_providers=self._tool_providers) + return self._source.extras + @property def created_by_account(self) -> Any: return self._source.created_by_account(self._session) @@ -221,8 +234,10 @@ def created_by_end_user(self) -> Any: return self._source.created_by_end_user(self._session) -def node_execution_response_source(node_execution: Any, *, session: Session) -> WorkflowNodeExecutionResponseSource: - return WorkflowNodeExecutionResponseSource(node_execution, session=session) +def node_execution_response_source( + node_execution: Any, *, session: Session, tool_providers: ToolProviderIcons +) -> WorkflowNodeExecutionResponseSource: + return WorkflowNodeExecutionResponseSource(node_execution, session=session, tool_providers=tool_providers) class WorkflowRunNodeExecutionListResponse(ResponseModel): diff --git a/api/libs/helper.py b/api/libs/helper.py index e5a62dd97470d4..2cccaa436c43e2 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -9,6 +9,7 @@ import uuid from collections.abc import Callable, Generator, Iterable, Mapping from datetime import datetime +from functools import partial from hashlib import sha256 from typing import TYPE_CHECKING, Annotated, Any, Protocol, cast, overload, override from uuid import UUID @@ -25,6 +26,7 @@ from extensions.ext_redis import redis_client from graphon.file import helpers as file_helpers from graphon.model_runtime.utils.encoders import jsonable_encoder +from libs.stream import close_stream if TYPE_CHECKING: from models import Account @@ -428,11 +430,15 @@ def compact_generate_response( def generate() -> Generator[str, None, None]: yield from stream_response - return Response( + result = Response( _stream_with_request_context(generate()), status=200, mimetype="text/event-stream", ) + # Closing an unconsumed wrapper does not enter its yield-from scope. + # Explicitly forward response closure to streams that own resources. + result.call_on_close(partial(close_stream, stream_response)) + return result def length_prefixed_response( diff --git a/api/models/model.py b/api/models/model.py index bf8c3d14347f46..5e042d6d9266a0 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -51,7 +51,6 @@ ProviderQuotaType, TagType, ) -from .provider_ids import GenericProviderID from .types import EnumText, LongText, StringUUID if TYPE_CHECKING: @@ -170,12 +169,6 @@ class FileUploadConfig(TypedDict): image: ImageUploadConfig -class DeletedToolInfo(TypedDict): - type: str - tool_name: str - provider_id: str - - class ExternalDataToolConfig(TypedDict): enabled: bool variable: str @@ -567,118 +560,6 @@ def mode_compatible_with_agent_with_session(self, *, session: Session) -> str: return str(self.mode) - def deleted_tools_with_session(self, *, session: Session) -> list[DeletedToolInfo]: - from core.plugin.plugin_service import PluginService - from core.tools.tool_manager import ToolManager, ToolProviderType - - # get agent mode tools - app_model_config = self.app_model_config_with_session(session=session) - if not app_model_config: - return [] - - if not app_model_config.agent_mode: - return [] - - agent_mode = app_model_config.agent_mode_dict - tools = agent_mode.get("tools", []) - - api_provider_ids: list[str] = [] - - builtin_provider_ids: list[GenericProviderID] = [] - - for tool in tools: - keys = list(tool.keys()) - if len(keys) >= 4: - provider_type = tool.get("provider_type", "") - provider_id = tool.get("provider_id", "") - if provider_type == ToolProviderType.API: - try: - uuid.UUID(provider_id) - except Exception: - continue - api_provider_ids.append(provider_id) - if provider_type == ToolProviderType.BUILT_IN: - try: - # check if it's hardcoded - try: - ToolManager.get_hardcoded_provider(provider_id) - is_hardcoded = True - except Exception: - is_hardcoded = False - - provider_id = GenericProviderID(provider_id, is_hardcoded) - except Exception: - continue - - builtin_provider_ids.append(provider_id) - - if not api_provider_ids and not builtin_provider_ids: - return [] - - if api_provider_ids: - existing_api_providers = [ - str(api_provider.id) - for api_provider in session.execute( - text("SELECT id FROM tool_api_providers WHERE id IN :provider_ids"), - {"provider_ids": tuple(api_provider_ids)}, - ).fetchall() - ] - else: - existing_api_providers = [] - - if builtin_provider_ids: - # get the non-hardcoded builtin providers - non_hardcoded_builtin_providers = [ - provider_id for provider_id in builtin_provider_ids if not provider_id.is_hardcoded - ] - if non_hardcoded_builtin_providers: - existence = list(PluginService.check_tools_existence(self.tenant_id, non_hardcoded_builtin_providers)) - else: - existence = [] - # add the hardcoded builtin providers - existence.extend([True] * (len(builtin_provider_ids) - len(non_hardcoded_builtin_providers))) - builtin_provider_ids = non_hardcoded_builtin_providers + [ - provider_id for provider_id in builtin_provider_ids if provider_id.is_hardcoded - ] - else: - existence = [] - - existing_builtin_providers = { - provider_id.provider_name: existence[i] for i, provider_id in enumerate(builtin_provider_ids) - } - - deleted_tools: list[DeletedToolInfo] = [] - - for tool in tools: - keys = list(tool.keys()) - if len(keys) >= 4: - provider_type = tool.get("provider_type", "") - provider_id = tool.get("provider_id", "") - - if provider_type == ToolProviderType.API: - if provider_id not in existing_api_providers: - deleted_tools.append( - { - "type": ToolProviderType.API, - "tool_name": tool["tool_name"], - "provider_id": provider_id, - } - ) - - if provider_type == ToolProviderType.BUILT_IN: - generic_provider_id = GenericProviderID(provider_id) - - if not existing_builtin_providers[generic_provider_id.provider_name]: - deleted_tools.append( - { - "type": ToolProviderType.BUILT_IN, - "tool_name": tool["tool_name"], - "provider_id": provider_id, # use the original one - } - ) - - return deleted_tools - def tags_with_session(self, *, session: Session) -> Sequence[Tag]: tags = session.scalars( select(Tag) diff --git a/api/models/workflow.py b/api/models/workflow.py index 963ddb163a2f92..b35796664d0ec4 100644 --- a/api/models/workflow.py +++ b/api/models/workflow.py @@ -23,7 +23,6 @@ from sqlalchemy.orm import Mapped, mapped_column from typing_extensions import deprecated -from core.trigger.constants import TRIGGER_PLUGIN_NODE_TYPE from core.workflow.environment_variables import load_environment_variables from core.workflow.human_input_adapter import adapt_node_config_for_graph from core.workflow.llm_environment_variable import LLMEnvironmentVariable, dump_environment_variable @@ -47,7 +46,6 @@ BuiltinNodeTypes, NodeType, WorkflowExecutionStatus, - WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus, ) from graphon.file import File @@ -1108,37 +1106,6 @@ def execution_metadata_dict(self) -> dict[str, Any]: # cases where metadata is absent. return json.loads(self.execution_metadata) if self.execution_metadata else {} - @property - def extras(self) -> dict[str, Any]: - from core.tools.tool_manager import ToolManager - from core.trigger.trigger_manager import TriggerManager - - extras: dict[str, Any] = {} - execution_metadata = self.execution_metadata_dict - if execution_metadata: - if self.node_type == BuiltinNodeTypes.TOOL and "tool_info" in execution_metadata: - tool_info: dict[str, Any] = execution_metadata["tool_info"] - extras["icon"] = ToolManager.get_tool_icon( - tenant_id=self.tenant_id, - provider_type=tool_info["provider_type"], - provider_id=tool_info["provider_id"], - ) - elif self.node_type == BuiltinNodeTypes.DATASOURCE and "datasource_info" in execution_metadata: - datasource_info = execution_metadata["datasource_info"] - extras["icon"] = datasource_info.get("icon") - elif ( - self.node_type == TRIGGER_PLUGIN_NODE_TYPE - and WorkflowNodeExecutionMetadataKey.TRIGGER_INFO in execution_metadata - ): - trigger_info = execution_metadata[WorkflowNodeExecutionMetadataKey.TRIGGER_INFO] or {} - provider_id = trigger_info.get("provider_id") - if provider_id: - extras["icon"] = TriggerManager.get_trigger_plugin_icon( - tenant_id=self.tenant_id, - provider_id=provider_id, - ) - return extras - def _get_offload_by_type(self, type_: ExecutionOffLoadType) -> "WorkflowNodeExecutionOffload | None": return next(iter([i for i in self.offload_data if i.type_ == type_]), None) @@ -1172,13 +1139,6 @@ def load_full_inputs(self, session: orm.Session, storage: Storage) -> Mapping[st return self._load_full_content(session, offload.file_id, storage) - def load_full_outputs(self, session: orm.Session, storage: Storage) -> Mapping[str, Any] | None: - offload: WorkflowNodeExecutionOffload | None = self._get_offload_by_type(ExecutionOffLoadType.OUTPUTS) - if offload is None: - return self.outputs_dict - - return self._load_full_content(session, offload.file_id, storage) - def load_full_process_data(self, session: orm.Session, storage: Storage) -> Mapping[str, Any] | None: offload: WorkflowNodeExecutionOffload | None = self._get_offload_by_type(ExecutionOffLoadType.PROCESS_DATA) if offload is None: diff --git a/api/repositories/app_definition_query_repository.py b/api/repositories/app_definition_query_repository.py index 03114a08d46e53..2d831820142c4a 100644 --- a/api/repositories/app_definition_query_repository.py +++ b/api/repositories/app_definition_query_repository.py @@ -8,13 +8,13 @@ from core.agent.publish_visibility import agent_has_workflow_callable_active_snapshot from core.app.apps.agent_app.app_feature_projection import merge_agent_app_features from core.app.apps.agent_app.app_variable_projection import agent_app_variables_to_user_input_form -from core.app.apps.agent_app.errors import AgentAppGeneratorError, AgentAppNotPublishedError from models.account import Tenant, TenantStatus from models.agent import AgentConfigSnapshot from models.agent_config_entities import AgentSoulConfig from models.model import App, AppMode, AppModelConfig, Site, load_annotation_reply_config from models.tools import ApiToolProvider from models.workflow import Workflow +from services.app.generation.errors import AgentAppGeneratorError, AgentAppNotPublishedError from services.app_definition_query_service import ( AppDefinitionQuery, AppDefinitionSummary, diff --git a/api/repositories/sqlalchemy_execution_extra_content_repository.py b/api/repositories/sqlalchemy_execution_extra_content_repository.py index b74bf637fd9a47..f900c3fe1af8f5 100644 --- a/api/repositories/sqlalchemy_execution_extra_content_repository.py +++ b/api/repositories/sqlalchemy_execution_extra_content_repository.py @@ -18,7 +18,6 @@ from core.entities.execution_extra_content import ( HumanInputContent as HumanInputContentDomainModel, ) -from core.workflow.nodes.human_input.callback import DifyHITLCallback from enums.human_input import HumanInputFormStatus, RecipientType from models.execution_extra_content import ( ExecutionExtraContent as ExecutionExtraContentModel, @@ -29,6 +28,7 @@ from models.human_input import HumanInputFormRecipient from models.human_input_entities import FormDefinition from repositories.execution_extra_content_repository import ExecutionExtraContentRepository +from services.workflow.execution.adapters.human_input import DifyHITLCallback logger = logging.getLogger(__name__) diff --git a/api/services/agent/__init__.py b/api/services/agent/__init__.py index c3a5ebaec4a1ae..795367c0d4ebce 100644 --- a/api/services/agent/__init__.py +++ b/api/services/agent/__init__.py @@ -1,4 +1 @@ -from .composer_service import AgentComposerService -from .roster_service import AgentRosterService - -__all__ = ["AgentComposerService", "AgentRosterService"] +"""Agent use cases; import each service explicitly from its owning module.""" diff --git a/api/services/agent/composer_service.py b/api/services/agent/composer_service.py index 62ac5176170478..61ea9edfd3acac 100644 --- a/api/services/agent/composer_service.py +++ b/api/services/agent/composer_service.py @@ -4,13 +4,15 @@ from sqlalchemy import func, or_, select from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from core.agent.publish_visibility import agent_has_workflow_callable_active_snapshot +from enums.agent import WorkflowAgentBindingType from libs.helper import to_timestamp from models import Account, App, Conversation from models.agent import ( APP_BACKED_AGENT_SOURCES, + WORKFLOW_EXECUTION_BINDING_VERSION, Agent, AgentConfigDraft, AgentConfigDraftType, @@ -25,15 +27,17 @@ AgentSource, AgentStatus, AgentWorkspaceOwnerType, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) -from models.agent_config_entities import DeclaredOutputConfig +from models.agent_config_entities import DeclaredOutputConfig, agent_soul_has_model from models.agent_config_entities import ( effective_declared_outputs as _effective_declared_outputs, ) from models.workflow import Workflow -from services.agent.agent_soul_state import agent_soul_has_model +from repositories.agent.config_repository import AgentConfigRepository +from repositories.agent.retirement_repository import WorkflowAgentRetirementRepository +from repositories.agent.workflow_binding_repository import WorkflowAgentBindingRepository, workflow_binding_scope +from repositories.agent_workspace_repository import AgentWorkspaceRepository from services.agent.composer_validator import ComposerConfigValidator from services.agent.dsl_entities import make_portable_agent_soul from services.agent.errors import ( @@ -53,7 +57,7 @@ get_tenant_knowledge_dataset_rows, list_missing_tenant_knowledge_dataset_ids, ) -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService +from services.agent.retirement_service import WorkflowAgentRetirementService from services.agent.roster_service import AgentRosterService from services.agent.workspace_service import AgentWorkspaceNotFoundError, AgentWorkspaceService, WorkspaceOwnerScope from services.app_service import AppService, CreateAppParams @@ -66,6 +70,8 @@ WorkflowNodeJobConfig, ) from services.skill_management_service import SkillManagementService +from services.tools.provider_queries import ToolProviders +from services.workflow.variable_service import WorkflowVariableService from tasks.collect_agent_resources_task import enqueue_agent_resource_collection from tasks.new_agent_beta_task import register_new_agent_beta_publish_after_commit @@ -136,7 +142,7 @@ def load_workflow_composer( raise AgentVersionNotFoundError() return cls._empty_workflow_state(app_id=app_id, workflow_id=workflow.id, node_id=node_id) - agent = cls._get_agent_if_present(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) + agent = AgentConfigRepository.get_agent(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) version = cls._workflow_composer_version( session=session, tenant_id=tenant_id, @@ -174,14 +180,16 @@ def _workflow_composer_version( raise AgentVersionNotFoundError() else: raise AgentVersionNotFoundError() - return cls._require_version(session=session, tenant_id=tenant_id, agent_id=agent.id, version_id=snapshot_id) + return AgentConfigRepository.require_snapshot( + session=session, tenant_id=tenant_id, agent_id=agent.id, version_id=snapshot_id + ) version_id = ( agent.active_config_snapshot_id if agent and binding.binding_type == WorkflowAgentBindingType.ROSTER_AGENT else binding.current_snapshot_id ) - return cls._get_version_if_present( + return AgentConfigRepository.get_snapshot( session=session, tenant_id=tenant_id, agent_id=agent.id if agent else None, @@ -207,9 +215,10 @@ def save_workflow_composer( if payload.save_strategy in _PUBLISH_SAVE_STRATEGIES: cls.validate_knowledge_datasets(session=session, tenant_id=tenant_id, agent_soul=payload.agent_soul) workflow = cls._get_draft_workflow(session=session, tenant_id=tenant_id, app_id=app_id) - binding = cls._get_workflow_binding( - session=session, tenant_id=tenant_id, workflow_id=workflow.id, node_id=node_id + bindings = WorkflowAgentBindingRepository(session).lock_bindings( + workflow_binding_scope(workflow), node_ids={node_id} ) + binding = bindings[0] if bindings else None retirement_candidates = ( {binding.agent_id} if binding is not None @@ -264,13 +273,13 @@ def save_workflow_composer( ) session.flush() - agent = cls._get_agent_if_present(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) + agent = AgentConfigRepository.get_agent(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) version_id = ( agent.active_config_snapshot_id if agent and binding.binding_type == WorkflowAgentBindingType.ROSTER_AGENT else binding.current_snapshot_id ) - version = cls._get_version_if_present( + version = AgentConfigRepository.get_snapshot( session=session, tenant_id=tenant_id, agent_id=agent.id if agent else None, @@ -281,7 +290,9 @@ def save_workflow_composer( ) state["validation"] = cls.collect_validation_findings(payload=payload) session.commit() - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService( + WorkflowAgentRetirementRepository(sessionmaker(bind=session.get_bind(), expire_on_commit=False)) + ).retire_unowned( tenant_id=tenant_id, agent_ids=retirement_candidates, account_id=account_id, @@ -309,8 +320,8 @@ def copy_workflow_composer_from_roster( ) if binding.binding_type == WorkflowAgentBindingType.INLINE_AGENT and idempotency_key: - agent = cls._get_agent_if_present(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) - version = cls._get_version_if_present( + agent = AgentConfigRepository.get_agent(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) + version = AgentConfigRepository.get_snapshot( session=session, tenant_id=tenant_id, agent_id=agent.id if agent else None, @@ -330,7 +341,7 @@ def copy_workflow_composer_from_roster( raise InvalidComposerConfigError("Source agent must be an active roster agent.") if not agent_has_workflow_callable_active_snapshot(session=session, agent=source_agent): raise InvalidComposerConfigError("Source agent must have a published config snapshot.") - source_version = cls._require_version( + source_version = AgentConfigRepository.require_snapshot( session=session, tenant_id=tenant_id, agent_id=source_agent.id, @@ -369,7 +380,7 @@ def copy_workflow_composer_from_roster( binding.updated_by = account_id session.flush() - version = cls._require_version( + version = AgentConfigRepository.require_snapshot( session=session, tenant_id=tenant_id, agent_id=inline_agent.id, @@ -421,7 +432,7 @@ def prepare_agent_composer_draft( ) -> AgentConfigDraft: """Prepare a persisted normal draft for an explicit configuration write.""" agent = cls._require_agent(session=session, tenant_id=tenant_id, agent_id=agent_id) - return cls.get_or_create_normal_agent_draft( + return AgentConfigRepository.normal_draft( session=session, tenant_id=tenant_id, agent=agent, created_by=account_id ) @@ -464,7 +475,7 @@ def _load_agent_composer_for_agent( draft = ( None if published_only - else cls._get_agent_draft( + else AgentConfigRepository.get_draft( session=session, tenant_id=tenant_id, agent_id=agent.id, @@ -472,7 +483,7 @@ def _load_agent_composer_for_agent( account_id=None, ) ) - version = cls._get_version_if_present( + version = AgentConfigRepository.get_snapshot( session=session, tenant_id=tenant_id, agent_id=agent.id, version_id=agent.active_config_snapshot_id ) if draft is None or ( @@ -645,7 +656,7 @@ def _agent_soul_matches_active_config( if not agent.active_config_snapshot_id: return False - active_version = cls._get_version_if_present( + active_version = AgentConfigRepository.get_snapshot( session=session, tenant_id=tenant_id, agent_id=agent.id, @@ -668,7 +679,7 @@ def publish_agent_app_draft( if agent.scope != AgentScope.ROSTER or agent.source not in APP_BACKED_AGENT_SOURCES: raise AgentNotFoundError() access_was_ready = agent_has_workflow_callable_active_snapshot(session=session, agent=agent) - draft = cls._get_or_create_agent_draft( + draft = AgentConfigRepository.get_or_create_draft( session=session, tenant_id=tenant_id, agent=agent, @@ -776,7 +787,7 @@ def _checkout_agent_app_build_draft_in_transaction( cls, *, session: Session, tenant_id: str, agent_id: str, account_id: str, force: bool ) -> tuple[dict[str, Any], str | None]: agent = cls._require_agent(session=session, tenant_id=tenant_id, agent_id=agent_id) - normal_draft = cls._get_or_create_agent_draft( + normal_draft = AgentConfigRepository.get_or_create_draft( session=session, tenant_id=tenant_id, agent=agent, @@ -784,7 +795,7 @@ def _checkout_agent_app_build_draft_in_transaction( account_id=None, created_by=account_id, ) - build_draft = cls._get_agent_draft( + build_draft = AgentConfigRepository.get_draft( session=session, tenant_id=tenant_id, agent_id=agent.id, @@ -852,7 +863,7 @@ def _validate_active_build_draft_binding( ) if binding is None or binding.agent_id != agent.id: raise AgentBuildSandboxNotFoundError() - AgentWorkspaceService.validate_binding_generation( + AgentWorkspaceRepository.validate_binding_generation( binding, base_home_snapshot_id=build_draft.home_snapshot_id, agent_config_version_id=build_draft.id, @@ -863,7 +874,7 @@ def _validate_active_build_draft_binding( def load_agent_app_build_draft( cls, *, session: Session, tenant_id: str, agent_id: str, account_id: str ) -> dict[str, Any]: - build_draft = cls._get_agent_draft( + build_draft = AgentConfigRepository.get_draft( session=session, tenant_id=tenant_id, agent_id=agent_id, @@ -923,7 +934,7 @@ def _apply_agent_app_build_draft_in_transaction( account_id: str, ) -> tuple[dict[str, Any], list[str]]: agent = cls._require_agent(session=session, tenant_id=tenant_id, agent_id=agent_id) - build_draft = cls._get_agent_draft( + build_draft = AgentConfigRepository.get_draft( session=session, tenant_id=tenant_id, agent_id=agent.id, @@ -1027,7 +1038,7 @@ def _retire_normal_preview_bindings( ) if binding is None or binding.agent_id != agent.id: raise AgentWorkspaceNotFoundError("Agent Preview participant Binding is unavailable") - AgentWorkspaceService.validate_binding_generation( + AgentWorkspaceRepository.validate_binding_generation( binding, base_home_snapshot_id=normal_draft.home_snapshot_id, agent_config_version_id=normal_draft.id, @@ -1071,7 +1082,7 @@ def _discard_agent_app_build_draft_in_transaction( cls, *, session: Session, tenant_id: str, agent_id: str, account_id: str ) -> tuple[dict[str, Any], str | None]: agent = cls._require_agent(session=session, tenant_id=tenant_id, agent_id=agent_id) - build_draft = cls._get_agent_draft( + build_draft = AgentConfigRepository.get_draft( session=session, tenant_id=tenant_id, agent_id=agent.id, @@ -1154,7 +1165,15 @@ def resolve_workflow_node_agent_id( @classmethod def get_workflow_candidates( - cls, *, session: Session, tenant_id: str, app_id: str, node_id: str, user_id: str + cls, + *, + session: Session, + tenant_id: str, + app_id: str, + node_id: str, + user_id: str, + variables: WorkflowVariableService, + tool_providers: ToolProviders, ) -> dict[str, Any]: """Slash-menu data source for the workflow Agent node composer (ENG-615).""" from services.agent.composer_candidates import previous_node_output_candidates, soul_candidates @@ -1184,10 +1203,10 @@ def get_workflow_candidates( session=session, tenant_id=tenant_id, workflow_id=workflow.id, node_id=nid ), draft_variables_loader=lambda nid: cls._draft_node_variables( - session=session, app_id=app_id, node_id=nid, user_id=user_id + variables=variables, app_id=app_id, node_id=nid, user_id=user_id ), system_variables_loader=lambda: cls._draft_system_variables( - session=session, app_id=app_id, user_id=user_id + variables=variables, app_id=app_id, user_id=user_id ), ) truncated = truncated or outputs_truncated @@ -1197,7 +1216,9 @@ def get_workflow_candidates( dataset_lookup=lambda ids: get_tenant_knowledge_dataset_rows( session=session, tenant_id=tenant_id, dataset_ids=ids ), - workspace_tools_loader=lambda: cls._workspace_dify_tools(tenant_id=tenant_id, user_id=user_id), + workspace_tools_loader=lambda: cls._workspace_dify_tools( + tenant_id=tenant_id, user_id=user_id, tool_providers=tool_providers + ), ) truncated = truncated or soul_truncated @@ -1217,7 +1238,7 @@ def get_workflow_candidates( @classmethod def get_agent_app_candidates( - cls, *, session: Session, tenant_id: str, agent_id: str, user_id: str + cls, *, session: Session, tenant_id: str, agent_id: str, user_id: str, tool_providers: ToolProviders ) -> dict[str, Any]: """Slash-menu data source for the Agent App (Console) composer (ENG-615).""" from services.agent.composer_candidates import soul_candidates @@ -1228,7 +1249,9 @@ def get_agent_app_candidates( dataset_lookup=lambda ids: get_tenant_knowledge_dataset_rows( session=session, tenant_id=tenant_id, dataset_ids=ids ), - workspace_tools_loader=lambda: cls._workspace_dify_tools(tenant_id=tenant_id, user_id=user_id), + workspace_tools_loader=lambda: cls._workspace_dify_tools( + tenant_id=tenant_id, user_id=user_id, tool_providers=tool_providers + ), ) response = ComposerCandidatesResponse( variant=ComposerVariant.AGENT_APP, @@ -1252,8 +1275,8 @@ def _parse_node_job(binding: WorkflowAgentNodeBinding) -> WorkflowNodeJobConfig def _load_binding_soul( cls, *, session: Session, tenant_id: str, binding: WorkflowAgentNodeBinding ) -> AgentSoulConfig | None: - agent = cls._get_agent_if_present(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) - version = cls._get_version_if_present( + agent = AgentConfigRepository.get_agent(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) + version = AgentConfigRepository.get_snapshot( session=session, tenant_id=tenant_id, agent_id=agent.id if agent else None, @@ -1263,10 +1286,10 @@ def _load_binding_soul( @classmethod def _load_agent_soul(cls, *, session: Session, tenant_id: str, agent_id: str) -> AgentSoulConfig | None: - agent = cls._get_agent_if_present(session=session, tenant_id=tenant_id, agent_id=agent_id) + agent = AgentConfigRepository.get_agent(session=session, tenant_id=tenant_id, agent_id=agent_id) if agent is None: return None - draft = cls._get_or_create_agent_draft( + draft = AgentConfigRepository.get_or_create_draft( session=session, tenant_id=tenant_id, agent=agent, @@ -1301,30 +1324,30 @@ def _binding_declared_outputs( return list(_effective_declared_outputs(node_job.declared_outputs, node_job.output_routes)) @staticmethod - def _draft_node_variables(*, session: Any, app_id: str, node_id: str, user_id: str) -> list[tuple[str, str | None]]: - from services.workflow_draft_variable_service import WorkflowDraftVariableService - - variables = WorkflowDraftVariableService(session=session).list_node_variables(app_id, node_id, user_id) - return [(variable.name, variable.value_type.value) for variable in variables.variables] + def _draft_node_variables( + *, variables: WorkflowVariableService, app_id: str, node_id: str, user_id: str + ) -> list[tuple[str, str | None]]: + result = variables.list_node_variables(app_id, node_id, user_id) + return [(variable.name, variable.value_type.value) for variable in result.variables] @staticmethod - def _draft_system_variables(*, session: Any, app_id: str, user_id: str) -> list[tuple[str, str | None]]: - from services.workflow_draft_variable_service import WorkflowDraftVariableService - - variables = WorkflowDraftVariableService(session=session).list_system_variables(app_id, user_id) - return [(variable.name, variable.value_type.value) for variable in variables.variables] + def _draft_system_variables( + *, variables: WorkflowVariableService, app_id: str, user_id: str + ) -> list[tuple[str, str | None]]: + result = variables.list_system_variables(app_id, user_id) + return [(variable.name, variable.value_type.value) for variable in result.variables] @staticmethod - def _workspace_dify_tools(*, tenant_id: str, user_id: str) -> list[dict[str, Any]]: + def _workspace_dify_tools(*, tenant_id: str, user_id: str, tool_providers: ToolProviders) -> list[dict[str, Any]]: """Workspace Dify Plugin tools, same source as the tool selector. A plugin-daemon outage must degrade the slash menu to an empty tools tab, not break the whole candidates endpoint. """ - from services.tools.legacy_builtin_tools_manage_service import BuiltinToolManageService + from services.tools.builtin_tools_manage_service import BuiltinToolManageService try: - providers = BuiltinToolManageService.list_builtin_tools(user_id, tenant_id) + providers = BuiltinToolManageService.list_builtin_tools(user_id, tenant_id, tool_providers=tool_providers) except Exception: logger.warning("candidates: failed to list workspace tools for tenant %s", tenant_id, exc_info=True) return [] @@ -1382,6 +1405,7 @@ def calculate_impact(cls, *, session: Session, tenant_id: str, current_snapshot_ select(WorkflowAgentNodeBinding).where( WorkflowAgentNodeBinding.tenant_id == tenant_id, or_(*predicates), + WorkflowAgentNodeBinding.workflow_version != WORKFLOW_EXECUTION_BINDING_VERSION, ) ).all() ) @@ -1426,7 +1450,7 @@ def _save_node_job_only( if payload.node_job is not None: binding.node_job_config = payload.node_job if payload.agent_soul is not None and binding.binding_type == WorkflowAgentBindingType.INLINE_AGENT: - current_snapshot = cls._require_version( + current_snapshot = AgentConfigRepository.require_snapshot( session=session, tenant_id=tenant_id, agent_id=binding.agent_id, @@ -1448,14 +1472,14 @@ def _save_node_job_only( agent.active_config_is_published = True agent.updated_by = account_id binding.current_snapshot_id = version.id - normal_draft = cls._get_agent_draft( + normal_draft = AgentConfigRepository.get_draft( session=session, tenant_id=tenant_id, agent_id=agent.id, draft_type=AgentConfigDraftType.DRAFT, account_id=None, ) - if normal_draft is not None and cls._rebase_workflow_only_normal_draft( + if normal_draft is not None and AgentConfigRepository.rebase_workflow_draft( agent=agent, draft=normal_draft, snapshot=version, @@ -1547,7 +1571,7 @@ def _save_to_current_version( binding = cls._require_binding(binding) if payload.agent_soul is None: raise ValueError("agent_soul is required") - current_snapshot = cls._require_version( + current_snapshot = AgentConfigRepository.require_snapshot( session=session, tenant_id=tenant_id, agent_id=binding.agent_id, @@ -1591,7 +1615,7 @@ def _save_as_new_version( binding = cls._require_binding(binding) if not binding.agent_id or payload.agent_soul is None: raise ValueError("agent_id and agent_soul are required") - current_snapshot = cls._require_version( + current_snapshot = AgentConfigRepository.require_snapshot( session=session, tenant_id=tenant_id, agent_id=binding.agent_id, @@ -1697,7 +1721,7 @@ def _save_to_roster( ) -> WorkflowAgentNodeBinding: binding = cls._require_binding(binding) source_agent = cls._require_agent(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) - source_version = cls._require_version( + source_version = AgentConfigRepository.require_snapshot( session=session, tenant_id=tenant_id, agent_id=source_agent.id, @@ -1841,7 +1865,7 @@ def _create_roster_agent_for_composer( if agent is None: raise AgentNotFoundError() - current_snapshot = cls._require_version( + current_snapshot = AgentConfigRepository.require_snapshot( session=session, tenant_id=tenant_id, agent_id=agent.id, @@ -1966,144 +1990,6 @@ def _require_agent_app_agent(cls, *, session: Session, tenant_id: str, app_id: s raise AgentNotFoundError() return agent - @classmethod - def _get_agent_draft( - cls, - *, - session: Session, - tenant_id: str, - agent_id: str, - draft_type: AgentConfigDraftType, - account_id: str | None, - ) -> AgentConfigDraft | None: - stmt = select(AgentConfigDraft).where( - AgentConfigDraft.tenant_id == tenant_id, - AgentConfigDraft.agent_id == agent_id, - AgentConfigDraft.draft_type == draft_type, - ) - if draft_type == AgentConfigDraftType.DEBUG_BUILD: - stmt = stmt.where(AgentConfigDraft.account_id == account_id) - else: - stmt = stmt.where(AgentConfigDraft.account_id.is_(None)) - return session.scalar(stmt.order_by(AgentConfigDraft.updated_at.desc()).limit(1)) - - @classmethod - def get_or_create_normal_agent_draft( - cls, - *, - session: Session, - tenant_id: str, - agent: Agent, - created_by: str | None, - ) -> AgentConfigDraft: - """Resolve the normal Draft, rebasing only stale WORKFLOW_ONLY DRAFT rows whose account_id is None. - - Roster and DEBUG_BUILD Drafts are never rebased. - """ - return cls._get_or_create_agent_draft( - session=session, - tenant_id=tenant_id, - agent=agent, - draft_type=AgentConfigDraftType.DRAFT, - account_id=None, - created_by=created_by, - ) - - @staticmethod - def _rebase_workflow_only_normal_draft( - *, - agent: Agent, - draft: AgentConfigDraft, - snapshot: AgentConfigSnapshot, - updated_by: str | None, - ) -> bool: - """Sync a stale normal Draft's base_snapshot_id, home_snapshot_id, config_snapshot, and updated_by.""" - - if ( - agent.scope != AgentScope.WORKFLOW_ONLY - or draft.draft_type != AgentConfigDraftType.DRAFT - or draft.account_id is not None - or not agent.active_config_snapshot_id - or draft.base_snapshot_id == agent.active_config_snapshot_id - or snapshot.id != agent.active_config_snapshot_id - ): - return False - draft.base_snapshot_id = snapshot.id - draft.home_snapshot_id = snapshot.home_snapshot_id - draft.config_snapshot = AgentSoulConfig.model_validate(snapshot.config_snapshot_dict) - draft.updated_by = updated_by - return True - - @classmethod - def _get_or_create_agent_draft( - cls, - *, - session: Session, - tenant_id: str, - agent: Agent, - draft_type: AgentConfigDraftType, - account_id: str | None, - created_by: str | None, - ) -> AgentConfigDraft: - # All draft writers lock the parent before checking for an absent draft. - # Locking a draft query alone cannot serialize concurrent first writes. - session.scalar(select(Agent.id).where(Agent.tenant_id == tenant_id, Agent.id == agent.id).with_for_update()) - draft = cls._get_agent_draft( - session=session, - tenant_id=tenant_id, - agent_id=agent.id, - draft_type=draft_type, - account_id=account_id, - ) - if draft is not None: - if ( - agent.scope == AgentScope.WORKFLOW_ONLY - and draft_type == AgentConfigDraftType.DRAFT - and draft.account_id is None - and agent.active_config_snapshot_id - and draft.base_snapshot_id != agent.active_config_snapshot_id - ): - active_snapshot = cls._get_version_if_present( - session=session, - tenant_id=tenant_id, - agent_id=agent.id, - version_id=agent.active_config_snapshot_id, - ) - if active_snapshot is None: - raise AgentVersionNotFoundError() - if cls._rebase_workflow_only_normal_draft( - agent=agent, - draft=draft, - snapshot=active_snapshot, - updated_by=agent.updated_by or agent.created_by, - ): - session.flush() - return draft - base_snapshot = cls._get_version_if_present( - session=session, - tenant_id=tenant_id, - agent_id=agent.id, - version_id=agent.active_config_snapshot_id, - ) - if base_snapshot is None: - raise AgentVersionNotFoundError() - agent_soul = AgentSoulConfig.model_validate(base_snapshot.config_snapshot_dict) - draft = AgentConfigDraft( - tenant_id=tenant_id, - agent_id=agent.id, - draft_type=draft_type, - account_id=account_id if draft_type == AgentConfigDraftType.DEBUG_BUILD else None, - draft_owner_key=account_id if draft_type == AgentConfigDraftType.DEBUG_BUILD and account_id else "", - base_snapshot_id=base_snapshot.id, - home_snapshot_id=base_snapshot.home_snapshot_id, - config_snapshot=agent_soul, - created_by=created_by, - updated_by=created_by, - ) - session.add(draft) - session.flush() - return draft - @classmethod def _save_agent_draft( cls, @@ -2117,7 +2003,7 @@ def _save_agent_draft( account_id_for_audit: str, base_snapshot_id: str | None = None, ) -> AgentConfigDraft: - draft = cls._get_or_create_agent_draft( + draft = AgentConfigRepository.get_or_create_draft( session=session, tenant_id=tenant_id, agent=agent, @@ -2216,47 +2102,6 @@ def _require_account(cls, *, session: Session, account_id: str) -> Account: raise ValueError("Account not found") return account - @classmethod - def _get_agent_if_present(cls, *, session: Session, tenant_id: str, agent_id: str | None) -> Agent | None: - if not agent_id: - return None - return session.scalar(select(Agent).where(Agent.tenant_id == tenant_id, Agent.id == agent_id).limit(1)) - - @classmethod - def _require_version( - cls, *, session: Session, tenant_id: str, agent_id: str | None, version_id: str | None - ) -> AgentConfigSnapshot: - if not agent_id or not version_id: - raise AgentVersionNotFoundError() - version = session.scalar( - select(AgentConfigSnapshot) - .where( - AgentConfigSnapshot.tenant_id == tenant_id, - AgentConfigSnapshot.agent_id == agent_id, - AgentConfigSnapshot.id == version_id, - ) - .limit(1) - ) - if not version: - raise AgentVersionNotFoundError() - return version - - @classmethod - def _get_version_if_present( - cls, *, session: Session, tenant_id: str, agent_id: str | None, version_id: str | None - ) -> AgentConfigSnapshot | None: - if not agent_id or not version_id: - return None - return session.scalar( - select(AgentConfigSnapshot) - .where( - AgentConfigSnapshot.tenant_id == tenant_id, - AgentConfigSnapshot.agent_id == agent_id, - AgentConfigSnapshot.id == version_id, - ) - .limit(1) - ) - @staticmethod def _serialize_effective_outputs(node_job: WorkflowNodeJobConfig) -> list[dict[str, Any]]: """Serialize custom declarations and the system outputs enabled for this job.""" diff --git a/api/services/agent/dsl_entities.py b/api/services/agent/dsl_entities.py index 6fa9392c911caf..b6b6adb9e77664 100644 --- a/api/services/agent/dsl_entities.py +++ b/api/services/agent/dsl_entities.py @@ -9,7 +9,7 @@ from core.plugin.entities.plugin import PluginDependency from models.agent import Agent -from models.agent_config_entities import AgentSoulConfig +from models.agent_config_entities import AgentPackageMetadata, AgentSoulConfig from models.model import App from services.entities.dsl_entities import make_app_dsl from services.entities.site_dsl import SiteDsl @@ -19,17 +19,6 @@ AGENT_NODE_JOB_DSL_KEY = "agent_job" -class AgentPackageMetadata(BaseModel): - model_config = ConfigDict(extra="forbid") - - name: str = Field(min_length=1, max_length=255) - description: str = "" - role: str = "" - icon_type: str | None = None - icon: str | None = None - icon_background: str | None = None - - class AgentPackageOmittedAsset(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/api/services/agent/dsl_service.py b/api/services/agent/dsl_service.py index b54b6ea37da5fe..fc8749b3cc30d2 100644 --- a/api/services/agent/dsl_service.py +++ b/api/services/agent/dsl_service.py @@ -9,17 +9,15 @@ from __future__ import annotations import copy -import json from collections.abc import Mapping from typing import Any, cast from uuid import UUID from pydantic import BaseModel, ConfigDict, Field -from sqlalchemy import func, select, update +from sqlalchemy import select, update from sqlalchemy.orm import Session -from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data -from core.workflow.nodes.agent_v2.validators import WorkflowAgentNodeValidator +from enums.agent import WorkflowAgentBindingType from libs.datetime_utils import naive_utc_now from models import Account from models.agent import ( @@ -27,29 +25,25 @@ Agent, AgentConfigDraft, AgentConfigDraftType, - AgentConfigRevision, AgentConfigRevisionOperation, AgentConfigSnapshot, AgentIconType, - AgentKind, AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) -from models.agent_config_entities import AgentSoulConfig, WorkflowNodeJobConfig +from models.agent_config_entities import AgentPackageMetadata, AgentSoulConfig, WorkflowNodeJobConfig from models.model import App, AppModelConfig, UploadFile from models.skill import AgentSkillBinding, AgentSkillBindingSnapshot, Skill from models.tools import ToolFile from models.workflow import Workflow -from services.agent.agent_soul_state import agent_soul_has_model +from repositories.agent.creation_repository import WorkflowAgentCreationRepository from services.agent.dependency_service import extract_agent_soul_dependencies from services.agent.dsl_entities import ( AGENT_NODE_JOB_DSL_KEY, AGENT_PACKAGE_REF_KEY, AgentPackage, - AgentPackageMetadata, AgentPackageWorkspaceSkill, make_portable_agent_package, portable_ref, @@ -57,7 +51,9 @@ from services.agent.knowledge_datasets import get_tenant_knowledge_dataset_rows from services.agent.package_resource_exporter import AgentPackageResourceExporter from services.agent.roster_service import AgentRosterService +from services.agent.workflow_validator import WorkflowAgentNodeValidator from services.entities.dsl_entities import DslImportWarning +from services.workflow.contracts import WorkflowBindingScope class AgentPackageImportResult(BaseModel): @@ -288,12 +284,16 @@ def import_agent_app_package( def import_workflow_packages( self, *, - workflow: Workflow, + workflow: WorkflowBindingScope, portable_graph: Mapping[str, Any], raw_packages: Mapping[str, Any], account: Account, ) -> tuple[dict[str, Any], list[DslImportWarning], set[str]]: - """Materialize every packaged Agent as a node-owned inline Agent.""" + """Stage node-owned Agent packages and return the resolved graph. + + The draft writer persists the returned graph in this same transaction, + keeping the workflow revision and imported bindings atomic. + """ graph = copy.deepcopy(dict(portable_graph)) packages = {key: AgentPackage.model_validate(value) for key, value in raw_packages.items()} @@ -364,41 +364,9 @@ def import_workflow_packages( node_data.pop(AGENT_NODE_JOB_DSL_KEY, None) warnings.extend(imported.warnings) - workflow.graph = json.dumps(graph) self.session.flush() return graph, warnings, retirement_candidates - def clone_inline_binding_for_node( - self, - *, - workflow: Workflow, - node_id: str, - source_agent: Agent, - source_snapshot: AgentConfigSnapshot, - account_id: str, - ) -> tuple[Agent, AgentConfigSnapshot]: - """Clone a same-workspace Inline Agent for a pasted target node.""" - - soul = AgentSoulConfig.model_validate(source_snapshot.config_snapshot_dict) - metadata = AgentPackageMetadata( - name=source_agent.name, - description=source_agent.description, - role=source_agent.role, - icon_type=source_agent.icon_type.value if source_agent.icon_type else None, - icon=source_agent.icon, - icon_background=source_agent.icon_background, - ) - agent, snapshot = self._create_workflow_only_agent( - workflow=workflow, - node_id=node_id, - account_id=account_id, - metadata=metadata, - soul=soul, - source=AgentSource.WORKFLOW, - operation=AgentConfigRevisionOperation.CREATE_VERSION, - ) - return agent, snapshot - def extract_package_dependencies(self, packages: Mapping[str, AgentPackage]) -> list[str]: dependencies: list[str] = [] for package in packages.values(): @@ -504,7 +472,7 @@ def _restore_workspace_skill_bindings( def _create_imported_inline_agent( self, *, - workflow: Workflow, + workflow: WorkflowBindingScope, node_id: str, account: Account, package: AgentPackage, @@ -546,7 +514,7 @@ def _create_imported_inline_agent( def _create_workflow_only_agent( self, *, - workflow: Workflow, + workflow: WorkflowBindingScope, node_id: str, account_id: str, metadata: AgentPackageMetadata, @@ -554,48 +522,15 @@ def _create_workflow_only_agent( source: AgentSource, operation: AgentConfigRevisionOperation, ) -> tuple[Agent, AgentConfigSnapshot]: - backing_app = AgentRosterService(self.session).create_hidden_backing_app_for_workflow_agent( - tenant_id=workflow.tenant_id, - account_id=account_id, - name=metadata.name, - description=metadata.description, - icon_type=metadata.icon_type, - icon=metadata.icon, - icon_background=metadata.icon_background, - ) - agent = Agent( - tenant_id=workflow.tenant_id, - name=metadata.name, - description=metadata.description, - role=metadata.role, - icon_type=self._agent_icon_type(metadata.icon_type), - icon=metadata.icon, - icon_background=metadata.icon_background, - agent_kind=AgentKind.DIFY_AGENT, - scope=AgentScope.WORKFLOW_ONLY, - source=source, - app_id=workflow.app_id, - backing_app_id=backing_app.id, - workflow_id=workflow.id, - workflow_node_id=node_id, - status=AgentStatus.ACTIVE, - created_by=account_id, - updated_by=account_id, - ) - self.session.add(agent) - self.session.flush() - snapshot = self._create_snapshot( - tenant_id=workflow.tenant_id, - agent=agent, + return WorkflowAgentCreationRepository(self.session).create_workflow_agent( + workflow=workflow, + node_id=node_id, account_id=account_id, + metadata=metadata, soul=soul, + source=source, operation=operation, ) - agent.active_config_snapshot_id = snapshot.id - agent.active_config_has_model = agent_soul_has_model(soul) - agent.active_config_is_published = True - self.session.flush() - return agent, snapshot def resolve_package_soul( self, @@ -725,46 +660,6 @@ def _mark_missing_package_assets( ) return warnings - def _create_snapshot( - self, - *, - tenant_id: str, - agent: Agent, - account_id: str, - soul: AgentSoulConfig, - operation: AgentConfigRevisionOperation, - ) -> AgentConfigSnapshot: - next_version = ( - self.session.scalar( - select(func.max(AgentConfigSnapshot.version)).where( - AgentConfigSnapshot.tenant_id == tenant_id, - AgentConfigSnapshot.agent_id == agent.id, - ) - ) - or 0 - ) + 1 - snapshot = AgentConfigSnapshot( - tenant_id=tenant_id, - agent_id=agent.id, - version=next_version, - config_snapshot=soul, - home_snapshot_id=None, - created_by=account_id, - ) - self.session.add(snapshot) - self.session.flush() - revision = AgentConfigRevision( - tenant_id=tenant_id, - agent_id=agent.id, - current_snapshot_id=snapshot.id, - revision=1, - operation=operation, - created_by=account_id, - ) - self.session.add(revision) - self.session.flush() - return snapshot - def unique_roster_name(self, *, tenant_id: str, requested: str) -> str: candidates = [requested] for index in range(1, 100): @@ -807,11 +702,3 @@ def _require_snapshot(self, *, tenant_id: str, agent_id: str, snapshot_id: str | @staticmethod def _agent_icon_type(value: str | None) -> AgentIconType | None: return AgentIconType(value) if value else None - - -def is_agent_v2_graph(graph: Mapping[str, Any]) -> bool: - return any( - isinstance(node.get("data"), Mapping) and is_dify_agent_node_data(node["data"]) - for node in graph.get("nodes", []) - if isinstance(node, Mapping) - ) diff --git a/api/services/agent/home_snapshot_service.py b/api/services/agent/home_snapshot_service.py index a1e4ff5d74d110..5f7b740025c790 100644 --- a/api/services/agent/home_snapshot_service.py +++ b/api/services/agent/home_snapshot_service.py @@ -14,7 +14,6 @@ from clients.agent_backend.factory import create_agent_backend_client from configs import dify_config from core.db.session_factory import session_factory -from libs.datetime_utils import naive_utc_now from libs.uuid_utils import uuidv7 from models.agent import ( Agent, @@ -25,6 +24,7 @@ AgentWorkingResourceStatus, AgentWorkspaceOwnerType, ) +from repositories.agent_workspace_repository import AgentWorkspaceRepository from services.agent.errors import ( AgentBuildSandboxNotFoundError, AgentHomeSnapshotCreateFailedError, @@ -40,7 +40,7 @@ class AgentHomeSnapshotUnavailableError(RuntimeError): class AgentHomeSnapshotService: - """Create, retire, and collect Agent-owned immutable Home Snapshots.""" + """Create and collect Agent-owned immutable Home Snapshots.""" @classmethod def create_for_build_apply( @@ -80,7 +80,7 @@ def create_for_build_apply( ) if binding is None or binding.agent_id != build_draft.agent_id: raise AgentBuildSandboxNotFoundError() - AgentWorkspaceService.validate_binding_generation( + AgentWorkspaceRepository.validate_binding_generation( binding, base_home_snapshot_id=build_draft.home_snapshot_id, agent_config_version_id=build_draft.id, @@ -121,21 +121,6 @@ def create_for_build_apply( session.add(home_snapshot) return home_snapshot - @classmethod - def retire_all_for_agent(cls, *, session: Session, tenant_id: str, agent_id: str) -> list[str]: - rows = session.scalars( - select(AgentHomeSnapshot).where( - AgentHomeSnapshot.tenant_id == tenant_id, - AgentHomeSnapshot.agent_id == agent_id, - ) - ).all() - now = naive_utc_now() - for row in rows: - if row.status == AgentWorkingResourceStatus.ACTIVE: - row.status = AgentWorkingResourceStatus.RETIRED - row.retired_at = now - return [row.id for row in rows] - @classmethod def collect_retired_home_snapshot(cls, *, tenant_id: str, home_snapshot_id: str) -> None: with session_factory.create_session() as session: diff --git a/api/services/agent/roster_package_importer.py b/api/services/agent/roster_package_importer.py index 5acde6ec80adae..40cb28f7234d94 100644 --- a/api/services/agent/roster_package_importer.py +++ b/api/services/agent/roster_package_importer.py @@ -27,9 +27,10 @@ AgentIconType, AgentSource, ) -from models.agent_config_entities import AgentSoulConfig +from models.agent_config_entities import AgentPackageMetadata, AgentSoulConfig from models.model import App, AppMode, AppModelConfig, IconType, UploadFile -from services.agent.dsl_entities import AgentAppDsl, AgentPackage, AgentPackageMetadata +from repositories.app.creation_records import create_installed_app_record, create_site_record +from services.agent.dsl_entities import AgentAppDsl, AgentPackage from services.agent.dsl_service import AgentDslService from services.agent.errors import ( AgentNameConflictError, @@ -43,7 +44,6 @@ from services.agent.roster_package_exporter import RosterAgentPackageExporter from services.agent.roster_package_reader import RosterAgentPackageReader from services.agent.roster_service import AgentRosterService -from services.app_creation_records import create_installed_app_record, create_site_record from services.app_dsl_service import AppDslService from services.app_import_source import download_app_import_source from services.app_service import AppService diff --git a/api/services/agent/roster_service.py b/api/services/agent/roster_service.py index 2f0b4e0295a5d9..ae257cad936f76 100644 --- a/api/services/agent/roster_service.py +++ b/api/services/agent/roster_service.py @@ -6,9 +6,9 @@ from sqlalchemy import and_, func, or_, select from sqlalchemy.exc import IntegrityError -from constants.model_template import default_app_templates from core.agent.publish_visibility import workflow_callable_active_snapshot_filter from core.app.entities.app_invoke_entities import InvokeFrom +from enums.agent import WorkflowAgentBindingType from libs.datetime_utils import naive_utc_now from libs.helper import to_timestamp from models.agent import ( @@ -26,14 +26,13 @@ AgentSource, AgentStatus, AgentWorkspaceOwnerType, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) -from models.agent_config_entities import AgentSoulConfig +from models.agent_config_entities import AgentSoulConfig, agent_soul_has_model from models.enums import AppStatus, ConversationFromSource, ConversationStatus -from models.model import App, AppMode, AppModelConfig, Conversation, IconType, Message +from models.model import App, AppMode, Conversation, IconType, Message from models.workflow import Workflow -from services.agent.agent_soul_state import agent_soul_has_model +from repositories.agent.creation_repository import WorkflowAgentCreationRepository from services.agent.composer_validator import ComposerConfigValidator from services.agent.errors import ( AgentArchivedError, @@ -470,33 +469,15 @@ def create_hidden_backing_app_for_workflow_agent( workspace Agent Roster until explicitly saved to roster. """ - app_template = dict(default_app_templates[AppMode.AGENT]["app"]) - app = App(**app_template) - app.name = name - app.description = description or "" - app.mode = AppMode.AGENT - normalized_icon_type = self._normalize_app_icon_type(icon_type) - app.icon_type = IconType(normalized_icon_type) if normalized_icon_type else IconType.EMOJI - app.icon = icon - app.icon_background = icon_background - app.tenant_id = tenant_id - app.enable_site = False - app.enable_api = False - app.api_rph = 0 - app.api_rpm = 0 - app.max_active_requests = None - app.created_by = account_id - app.maintainer = account_id - app.updated_by = account_id - self._session.add(app) - self._session.flush() - - app_model_config = AppModelConfig(app_id=app.id, created_by=account_id, updated_by=account_id) - self._session.add(app_model_config) - self._session.flush() - app.app_model_config_id = app_model_config.id - self._session.flush() - return app + return WorkflowAgentCreationRepository(self._session).create_hidden_backing_app( + tenant_id=tenant_id, + account_id=account_id, + name=name, + description=description, + icon_type=icon_type, + icon=icon, + icon_background=icon_background, + ) def _create_agent_app_debug_conversation(self, *, app_id: str, account_id: str) -> str: """Create one console debug conversation for an Agent App editor.""" @@ -1537,29 +1518,6 @@ def _next_revision(self, *, tenant_id: str, agent_id: str) -> int: or 0 ) + 1 - def _load_published_active_snapshot_agent_ids(self, *, tenant_id: str, agents: list[Agent]) -> set[str]: - predicates = [ - and_( - AgentConfigRevision.agent_id == agent.id, - AgentConfigRevision.current_snapshot_id == agent.active_config_snapshot_id, - AgentConfigRevision.operation.in_(self._visible_version_operations(agent)), - ) - for agent in agents - if agent.active_config_snapshot_id - ] - if not predicates: - return set() - - agent_ids = self._session.scalars( - select(AgentConfigRevision.agent_id) - .where( - AgentConfigRevision.tenant_id == tenant_id, - or_(*predicates), - ) - .distinct() - ).all() - return set(agent_ids) - def _load_published_references_by_agent_id( self, *, tenant_id: str, agent_ids: list[str] ) -> dict[str, list[AgentReferencingWorkflow]]: diff --git a/api/services/agent/workspace_service.py b/api/services/agent/workspace_service.py index 344f6b38905412..e748b4f18ed505 100644 --- a/api/services/agent/workspace_service.py +++ b/api/services/agent/workspace_service.py @@ -24,7 +24,6 @@ WorkspaceOwnerScope, ) from core.db.session_factory import session_factory -from libs.datetime_utils import naive_utc_now from libs.uuid_utils import uuidv7 from models.agent import ( AgentConfigVersionKind, @@ -209,43 +208,7 @@ def save_binding_session_snapshot( @classmethod def retire_binding(cls, *, session: Session, tenant_id: str, binding_id: str) -> str | None: - binding = session.scalar( - select(AgentWorkspaceBinding) - .where( - AgentWorkspaceBinding.id == binding_id, - AgentWorkspaceBinding.tenant_id == tenant_id, - AgentWorkspaceBinding.status == AgentWorkingResourceStatus.ACTIVE, - ) - .with_for_update() - ) - if binding is None: - return None - workspace = session.scalar( - select(AgentWorkspace) - .where( - AgentWorkspace.id == binding.workspace_id, - AgentWorkspace.tenant_id == tenant_id, - AgentWorkspace.status == AgentWorkingResourceStatus.ACTIVE, - ) - .with_for_update() - ) - now = naive_utc_now() - binding.status = AgentWorkingResourceStatus.RETIRED - binding.retired_at = now - if workspace is not None: - other_binding = session.scalar( - select(AgentWorkspaceBinding.id).where( - AgentWorkspaceBinding.tenant_id == tenant_id, - AgentWorkspaceBinding.workspace_id == workspace.id, - AgentWorkspaceBinding.status == AgentWorkingResourceStatus.ACTIVE, - AgentWorkspaceBinding.id != binding.id, - ) - ) - if other_binding is None: - workspace.status = AgentWorkingResourceStatus.RETIRED - workspace.active_guard = None - workspace.retired_at = now - return binding.id + return AgentWorkspaceRepository(session=session).retire_binding(tenant_id=tenant_id, binding_id=binding_id) @classmethod def retire_workspace(cls, *, session: Session, tenant_id: str, workspace_id: str) -> str | None: @@ -255,25 +218,7 @@ def retire_workspace(cls, *, session: Session, tenant_id: str, workspace_id: str @classmethod def retire_all_for_app(cls, *, session: Session, tenant_id: str, app_id: str) -> list[str]: - """Retire all ACTIVE Workspaces owned by an App in the caller's transaction.""" - - workspaces = session.scalars( - select(AgentWorkspace).where( - AgentWorkspace.tenant_id == tenant_id, - AgentWorkspace.app_id == app_id, - AgentWorkspace.status == AgentWorkingResourceStatus.ACTIVE, - ) - ).all() - retired: list[str] = [] - for workspace in workspaces: - workspace_id = cls.retire_workspace( - session=session, - tenant_id=tenant_id, - workspace_id=workspace.id, - ) - if workspace_id is not None: - retired.append(workspace_id) - return retired + return AgentWorkspaceRepository(session=session).retire_all_for_app(tenant_id=tenant_id, app_id=app_id) @classmethod def retire_all_for_conversation( @@ -419,23 +364,6 @@ def collect_retired_workspace(cls, *, tenant_id: str, workspace_id: str) -> None ) session.commit() - @staticmethod - def validate_binding_generation( - binding: AgentWorkspaceBinding, - *, - base_home_snapshot_id: str | None, - agent_config_version_id: str, - agent_config_version_kind: AgentConfigVersionKind, - ) -> None: - if ( - binding.base_home_snapshot_id != base_home_snapshot_id - or binding.agent_config_version_id != agent_config_version_id - or binding.agent_config_version_kind != agent_config_version_kind - ): - raise AgentWorkspaceBindingGenerationMismatchError( - "ACTIVE Binding belongs to a different Agent config/Home generation" - ) - @staticmethod def _client() -> Client: base_url = dify_config.AGENT_BACKEND_BASE_URL diff --git a/api/services/agent_service.py b/api/services/agent_service.py index bcc2539ab93457..cc7bfc3a6a809f 100644 --- a/api/services/agent_service.py +++ b/api/services/agent_service.py @@ -1,170 +1,9 @@ -import threading -from typing import Any - -import pytz -from sqlalchemy import select -from sqlalchemy.orm import Session - -import contexts -from core.app.app_config.easy_ui_based_app.agent.manager import AgentConfigManager from core.plugin.entities.plugin_daemon import PluginAgentProviderEntity from core.plugin.impl.agent import PluginAgentClient from core.plugin.impl.exc import PluginDaemonClientSideError -from core.tools.entities.tool_entities import EmojiIconDict -from core.tools.tool_manager import ToolManager -from libs.login import current_user -from models import Account -from models.model import App, Conversation, EndUser, Message, load_annotation_reply_config class AgentService: - @classmethod - def get_agent_logs(cls, app_model: App, conversation_id: str, message_id: str, session: Session): - """ - Service to get agent logs - """ - contexts.plugin_tool_providers.set({}) - contexts.plugin_tool_providers_lock.set(threading.Lock()) - - conversation: Conversation | None = session.scalar( - select(Conversation) - .where( - Conversation.id == conversation_id, - Conversation.app_id == app_model.id, - ) - .limit(1) - ) - - if not conversation: - raise ValueError(f"Conversation not found: {conversation_id}") - - message: Message | None = session.scalar( - select(Message) - .where( - Message.id == message_id, - Message.conversation_id == conversation_id, - ) - .limit(1) - ) - - if not message: - raise ValueError(f"Message not found: {message_id}") - - agent_thoughts = message.agent_thoughts_with_session(session=session) - - if conversation.from_end_user_id: - # only select name field - executor_name = session.scalar(select(EndUser.name).where(EndUser.id == conversation.from_end_user_id)) - else: - executor_name = session.scalar(select(Account.name).where(Account.id == conversation.from_account_id)) - - executor = executor_name or "Unknown" - assert isinstance(current_user, Account) - assert current_user.timezone is not None - timezone = pytz.timezone(current_user.timezone) - - app_model_config = app_model.app_model_config_with_session(session=session) - if not app_model_config: - raise ValueError("App model config not found") - - result: dict[str, Any] = { - "meta": { - "status": "success", - "executor": executor, - "start_time": message.created_at.astimezone(timezone).isoformat(), - "elapsed_time": message.provider_response_latency, - "total_tokens": message.answer_tokens + message.message_tokens, - "agent_mode": app_model_config.agent_mode_dict.get("strategy", "react"), - "iterations": len(agent_thoughts), - }, - "iterations": [], - "files": message.message_files_with_session(session=session), - } - - annotation_reply = load_annotation_reply_config(session, app_model.id) - agent_config = AgentConfigManager.convert(app_model_config.to_dict(annotation_reply=annotation_reply)) - if not agent_config: - raise ValueError("Agent config not found") - - agent_tools = agent_config.tools or [] - - def find_agent_tool(tool_name: str): - for agent_tool in agent_tools: - if agent_tool.tool_name == tool_name: - return agent_tool - - for agent_thought in agent_thoughts: - tools = agent_thought.tools - tool_labels = agent_thought.tool_labels - tool_meta = agent_thought.tool_meta - tool_inputs = agent_thought.tool_inputs_dict - tool_outputs = agent_thought.tool_outputs_dict or {} - tool_calls = [] - for tool in tools: - tool_name = tool - tool_label = tool_labels.get(tool_name, tool_name) - tool_input = tool_inputs.get(tool_name, {}) - tool_output = tool_outputs.get(tool_name, {}) - tool_meta_data = tool_meta.get(tool_name, {}) - tool_config = tool_meta_data.get("tool_config", {}) - tool_provider_type = tool_config.get("tool_provider_type", "") - tool_provider_id = tool_config.get("tool_provider", "") - - if not tool_provider_type: - tool_entity = find_agent_tool(tool_name) - if tool_entity: - tool_provider_type = tool_entity.provider_type - tool_provider_id = tool_provider_id or tool_entity.provider_id - - tool_icon: str | EmojiIconDict = "" - - if tool_provider_type and tool_provider_type != "dataset-retrieval": - tool_icon = ToolManager.get_tool_icon( - tenant_id=app_model.tenant_id, - provider_type=tool_provider_type, - provider_id=tool_provider_id, - ) - if not tool_icon: - tool_entity = find_agent_tool(tool_name) - if tool_entity: - tool_icon = ToolManager.get_tool_icon( - tenant_id=app_model.tenant_id, - provider_type=tool_entity.provider_type, - provider_id=tool_entity.provider_id, - ) - else: - tool_icon = "" - - tool_calls.append( - { - "status": "success" if not tool_meta_data.get("error") else "error", - "error": tool_meta_data.get("error"), - "time_cost": tool_meta_data.get("time_cost", 0), - "tool_name": tool_name, - "tool_label": tool_label, - "tool_input": tool_input, - "tool_output": tool_output, - "tool_parameters": tool_meta_data.get("tool_parameters", {}), - "tool_icon": tool_icon, - } - ) - - result["iterations"].append( - { - "tokens": agent_thought.tokens, - "tool_calls": tool_calls, - "tool_raw": { - "inputs": agent_thought.tool_input, - "outputs": agent_thought.observation, - }, - "thought": agent_thought.thought, - "created_at": agent_thought.created_at.isoformat(), - "files": agent_thought.files, - } - ) - - return result - @classmethod def list_agent_providers(cls, user_id: str, tenant_id: str) -> list[PluginAgentProviderEntity]: """ diff --git a/api/services/annotation_service.py b/api/services/annotation_service.py index 52c3eddc1538fe..3b66c85c345c22 100644 --- a/api/services/annotation_service.py +++ b/api/services/annotation_service.py @@ -3,10 +3,9 @@ from typing import TypedDict import pandas as pd -from sqlalchemy import delete, or_, select, update +from sqlalchemy import delete, or_, select from sqlalchemy.orm import Session from werkzeug.datastructures import FileStorage -from werkzeug.exceptions import NotFound from core.helper.csv_sanitizer import CSVSanitizer from enums import DeploymentEdition @@ -14,9 +13,9 @@ from libs.datetime_utils import naive_utc_now from libs.login import current_account_with_tenant from libs.pagination import paginate_query -from models.account import Account from models.dataset import DatasetCollectionBinding from models.model import App, AppAnnotationHitHistory, AppAnnotationSetting, Message, MessageAnnotation +from services.annotation.errors import AnnotationResourceNotFoundError from services.app_ref_service import AnnotationRef, AppRef from services.feature_service import FeatureService from tasks.annotation.add_annotation_to_index_task import add_annotation_to_index_task @@ -114,7 +113,7 @@ def up_insert_app_annotation_from_message( ) if not app: - raise NotFound("App not found") + raise AnnotationResourceNotFoundError("App not found") answer = args.get("answer") or args.get("content") if answer is None: @@ -126,7 +125,7 @@ def up_insert_app_annotation_from_message( message = session.scalar(select(Message).where(Message.id == message_id, Message.app_id == app.id).limit(1)) if not message: - raise NotFound("Message Not Exists.") + raise AnnotationResourceNotFoundError("Message Not Exists.") question = args.get("question") or message.query or "" @@ -225,7 +224,7 @@ def get_annotation_list_by_app_id(cls, app_id: str, page: int, limit: int, keywo ) if not app: - raise NotFound("App not found") + raise AnnotationResourceNotFoundError("App not found") if keyword: from libs.helper import escape_like_pattern @@ -265,7 +264,7 @@ def export_annotation_list_by_app_id(cls, app_id: str, session: Session): ) if not app: - raise NotFound("App not found") + raise AnnotationResourceNotFoundError("App not found") annotations = session.scalars( select(MessageAnnotation) .where(MessageAnnotation.app_id == app_id) @@ -294,7 +293,7 @@ def insert_app_annotation_directly( ) if not app: - raise NotFound("App not found") + raise AnnotationResourceNotFoundError("App not found") question = args.get("question") if question is None: @@ -329,7 +328,7 @@ def update_app_annotation_directly( annotation = cls._get_annotation_by_ref(annotation_ref, session) if not annotation: - raise NotFound("Annotation not found") + raise AnnotationResourceNotFoundError("Annotation not found") question = args.get("question") if question is None: @@ -364,7 +363,7 @@ def delete_app_annotation(cls, annotation_ref: AnnotationRef, session: Session): annotation = cls._get_annotation_by_ref(annotation_ref, session) if not annotation: - raise NotFound("Annotation not found") + raise AnnotationResourceNotFoundError("Annotation not found") session.delete(annotation) @@ -458,7 +457,7 @@ def batch_import_app_annotations(cls, app_id: str, file: FileStorage, session: S ) if not app: - raise NotFound("App not found") + raise AnnotationResourceNotFoundError("App not found") job_id: str | None = None # Initialize to avoid unbound variable error try: @@ -577,7 +576,7 @@ def get_annotation_hit_histories(cls, annotation_ref: AnnotationRef, page, limit annotation = cls._get_annotation_by_ref(annotation_ref, session) if not annotation: - raise NotFound("Annotation not found") + raise AnnotationResourceNotFoundError("Annotation not found") stmt = ( select(AppAnnotationHitHistory) @@ -590,61 +589,6 @@ def get_annotation_hit_histories(cls, annotation_ref: AnnotationRef, page, limit annotation_hit_histories = paginate_query(stmt, session=session, page=page, per_page=limit, max_per_page=100) return annotation_hit_histories.items, annotation_hit_histories.total or 0 - @classmethod - def get_annotation_by_id(cls, annotation_id: str, session: Session) -> MessageAnnotation | None: - annotation = session.get(MessageAnnotation, annotation_id) - - if not annotation: - return None - return annotation - - @classmethod - def get_annotation_reply_by_id( - cls, annotation_id: str, session: Session - ) -> tuple[MessageAnnotation, str | None] | None: - """Read a reply and its author together, retaining annotations whose account was deleted.""" - row = session.execute( - select(MessageAnnotation, Account.name) - .outerjoin(Account, Account.id == MessageAnnotation.account_id) - .where(MessageAnnotation.id == annotation_id) - ).one_or_none() - return (row[0], row[1]) if row is not None else None - - @classmethod - def add_annotation_history( - cls, - annotation_id: str, - app_id: str, - annotation_question: str, - annotation_content: str, - query: str, - user_id: str, - message_id: str, - from_source: str, - score: float, - session: Session, - ) -> None: - # add hit count to annotation - session.execute( - update(MessageAnnotation) - .where(MessageAnnotation.id == annotation_id) - .values(hit_count=MessageAnnotation.hit_count + 1) - ) - - annotation_hit_history = AppAnnotationHitHistory( - annotation_id=annotation_id, - app_id=app_id, - account_id=user_id, - question=query, - source=from_source, - score=score, - message_id=message_id, - annotation_question=annotation_question, - annotation_content=annotation_content, - ) - session.add(annotation_hit_history) - session.flush() - @classmethod def get_app_annotation_setting_by_app_id( cls, app_id: str, session: Session @@ -656,7 +600,7 @@ def get_app_annotation_setting_by_app_id( ) if not app: - raise NotFound("App not found") + raise AnnotationResourceNotFoundError("App not found") annotation_setting = session.scalar( select(AppAnnotationSetting).where(AppAnnotationSetting.app_id == app_id).limit(1) @@ -693,7 +637,7 @@ def update_app_annotation_setting( ) if not app: - raise NotFound("App not found") + raise AnnotationResourceNotFoundError("App not found") annotation_setting = session.scalar( select(AppAnnotationSetting) @@ -704,7 +648,7 @@ def update_app_annotation_setting( .limit(1) ) if not annotation_setting: - raise NotFound("App annotation not found") + raise AnnotationResourceNotFoundError("App annotation not found") annotation_setting.score_threshold = args["score_threshold"] annotation_setting.updated_user_id = current_user.id annotation_setting.updated_at = naive_utc_now() @@ -739,7 +683,7 @@ def clear_all_annotations(cls, app_id: str, session: Session): ) if not app: - raise NotFound("App not found") + raise AnnotationResourceNotFoundError("App not found") # if annotation reply is enabled, delete annotation index app_annotation_setting = session.scalar( diff --git a/api/services/app/console_gateway.py b/api/services/app/console_gateway.py index 519d9a7d682e25..c2076eb9efd456 100644 --- a/api/services/app/console_gateway.py +++ b/api/services/app/console_gateway.py @@ -19,10 +19,11 @@ from models.account import Account from models.agent import AgentIconType from models.model import AppMode +from models.tool_runtime_contracts import WorkflowToolQueries +from repositories.agent.home_snapshot_repository import AgentHomeSnapshotRepository from repositories.app.console_repository import ConsoleAppRepository, console_app_actor, require_console_app from repositories.app.response import app_record from services.agent.errors import AgentNameConflictError -from services.agent.home_snapshot_service import AgentHomeSnapshotService from services.agent.roster_package_entities import RosterAgentPackageExport from services.agent.roster_package_exporter import RosterAgentPackageExporter from services.agent.roster_package_importer import RosterAgentPackageImporter @@ -65,6 +66,7 @@ from services.feature_service import FeatureService from services.recommended_app_package_service import RecommendedAppPackageService from services.system_feature_service import SystemFeatureService +from services.tools.provider_queries import ToolProviders from tasks.initialize_created_app_rbac_access_task import initialize_created_app_rbac_access_task @@ -367,8 +369,15 @@ class AppLifecycleGateway(AppLifecycle): the App transaction; notification and resource cleanup run after it closes. """ - def __init__(self, *, session_factory: sessionmaker[Session]) -> None: + def __init__( + self, + *, + session_factory: sessionmaker[Session], + tool_providers: ToolProviders, + workflow_queries: WorkflowToolQueries, + ) -> None: self._session_factory = session_factory + self._responses = AppResponseGateway(tool_providers=tool_providers, workflow_queries=workflow_queries) @override def create(self, context: RequestContext, params: CreateAppParams, settings: AppCreationSettings) -> AppRecord: @@ -406,8 +415,8 @@ def delete(self, context: RequestContext, app_id: str) -> AppDeletion: session=session, tenant_id=context.active_workspace_id, binding_id=binding_id ) snapshots = ( - AgentHomeSnapshotService.retire_all_for_agent( - session=session, tenant_id=context.active_workspace_id, agent_id=deleted.backing_agent_id + AgentHomeSnapshotRepository(session).retire_all_for_agent( + tenant_id=context.active_workspace_id, agent_id=deleted.backing_agent_id ) if deleted.backing_agent_id is not None else [] @@ -435,11 +444,11 @@ def updated(self, context: RequestContext, app: AppRecord) -> None: @override def deleted(self, context: RequestContext, deleted: AppDeletion) -> None: - AppService.notify_deleted_app(deleted, account_id=context.account_id) + AppService.notify_deleted_app(deleted, account_id=context.account_id, sessions=self._session_factory) @override def present(self, context: RequestContext, app: AppRecord, *, mask_credentials: bool = False) -> AppRecord: app = replace( app, deleted_tools=AppResponseGateway.find_deleted_tools(context.active_workspace_id, app.tool_references) ) - return AppResponseGateway.mask_record(context, app) if mask_credentials else app + return self._responses.mask_record(context, app) if mask_credentials else app diff --git a/api/services/app/response_gateway.py b/api/services/app/response_gateway.py index 1eeb3b5a833133..b920db4a9ec5ed 100644 --- a/api/services/app/response_gateway.py +++ b/api/services/app/response_gateway.py @@ -1,3 +1,6 @@ +from models.tool_runtime_contracts import WorkflowToolQueries +from services.tools.provider_queries import ToolProviders + """App response enrichment through tool runtimes and plugin metadata.""" from collections.abc import Mapping @@ -7,16 +10,20 @@ from pydantic import JsonValue -from core.agent.tool_configuration import mask_agent_tool_parameters from core.plugin.plugin_service import PluginService -from core.tools.tool_manager import ToolManager from machinery.context import RequestContext from models.model import AppMode from models.provider_ids import GenericProviderID from services.entities.app_entities import AppRecord, AppToolReference +from services.tools.agent_configuration import mask_agent_tool_parameters +from services.tools.tool_manager import ToolManager class AppResponseGateway: + def __init__(self, tool_providers: ToolProviders, workflow_queries: WorkflowToolQueries) -> None: + self._tool_providers = tool_providers + self._workflow_queries = workflow_queries + @staticmethod def find_deleted_tools(tenant_id: str, references: tuple[AppToolReference, ...]) -> list[dict[str, str]]: """Resolve builtin tool availability from detached references, without a database session.""" @@ -47,12 +54,13 @@ def find_deleted_tools(tenant_id: str, references: tuple[AppToolReference, ...]) or (ref.provider_type == "builtin" and existence.get(ref.provider_id) is False) ] - @staticmethod - def mask_record(context: RequestContext, app: AppRecord) -> AppRecord: + def mask_record(self, context: RequestContext, app: AppRecord) -> AppRecord: if app.mode_compatible_with_agent != AppMode.AGENT_CHAT or app.app_model_config is None: return app config = deepcopy(app.app_model_config) config["agent_mode"] = mask_agent_tool_parameters( + tool_providers=self._tool_providers, + workflow_queries=self._workflow_queries, agent_mode=cast(Mapping[str, JsonValue], config.get("agent_mode", {})), tenant_id=context.active_workspace_id, app_id=app.id, diff --git a/api/services/app_definition_query_service.py b/api/services/app_definition_query_service.py index 327428d2518777..849ac618d92938 100644 --- a/api/services/app_definition_query_service.py +++ b/api/services/app_definition_query_service.py @@ -5,7 +5,7 @@ from typing import Any, NamedTuple, Protocol from core.app.app_config.common.parameters_mapping import AppParametersDict, get_parameters_from_feature_dict -from core.app.apps.agent_app.errors import AgentAppGeneratorError, AgentAppNotPublishedError +from services.app.generation.errors import AgentAppGeneratorError, AgentAppNotPublishedError class AppParameterConfig(NamedTuple): diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py index 1bb3036aca3623..3cde0ff3b5022c 100644 --- a/api/services/app_dsl_service.py +++ b/api/services/app_dsl_service.py @@ -12,7 +12,6 @@ from Crypto.Util.Padding import pad, unpad from packaging.version import parse as parse_version from pydantic import BaseModel, Field -from sqlalchemy import select from sqlalchemy.orm import Session from configs import dify_config @@ -47,20 +46,21 @@ from graphon.nodes.question_classifier.entities import QuestionClassifierNodeData from graphon.nodes.tool.entities import ToolNodeData from libs.datetime_utils import naive_utc_now +from machinery.context import RequestContext from models import Account, App, AppMode from models.agent import AgentScope from models.model import AppModelConfig, AppModelConfigDict, IconType, load_annotation_reply_config from models.workflow import Workflow from services.agent.dsl_entities import AgentPackage, make_agent_app_dsl from services.agent.dsl_service import AgentDslService -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService -from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService from services.agent.package_resource_exporter import AgentPackageResourceExporter +from services.agent.workflow_contracts import WorkflowAgentBindingStore from services.dsl_content import DSL_MAX_SIZE, dsl_content_size from services.dsl_version import check_version_compatibility from services.enterprise.rbac_service import RBACService from services.entities.dsl_entities import ( AppDslExportData, + AppDslOverwriteStore, AppImportPackage, CheckDependenciesResult, DslImportWarning, @@ -80,7 +80,15 @@ is_valid_image_icon, ) from services.plugin.dependencies_analysis import DependenciesAnalysisService -from services.workflow_draft_variable_service import WorkflowDraftVariableService +from services.workflow.contracts import ( + DraftImportResult, + DraftSyncCommand, + PreparedEnvironmentVariables, + WorkflowOwner, + WorkflowSnapshot, +) +from services.workflow.draft_service import WorkflowDraftService +from services.workflow.environment_variable_service import prepare_environment_variables from services.workflow_service import WorkflowService logger = logging.getLogger(__name__) @@ -127,8 +135,16 @@ class CheckDependenciesPendingData(BaseModel): class AppDslService: _warnings: list[DslImportWarning] - def __init__(self, session: Session): + def __init__( + self, + session: Session, + *, + drafts: WorkflowDraftService[WorkflowAgentBindingStore], + overwrites: AppDslOverwriteStore, + ): self._session = session + self._drafts = drafts + self._overwrites = overwrites self._warnings = [] def import_app( @@ -264,17 +280,24 @@ def import_app( # Authorize overwrites in a separate read session before storage I/O. # The normal import below reloads and rechecks the target after upload. if app_id: - with Session(self._session.get_bind()) as authorization_session: - target = AppDslService(authorization_session)._load_app_for_overwrite(account, app_id) - if target is None: - raise ValueError("App not found") - self._validate_workflow_overwrite(target, data) + target = self._overwrites.snapshot( + tenant_id=tenant_id, + account_id=account.id, + app_id=app_id, + rbac_allowed=self._check_overwrite_access(account, app_id), + ) + if target is None: + raise ValueError("App not found") + self._validate_workflow_overwrite(target.mode, data) package.materialize_icons(data=data, tenant_id=tenant_id, account_id=account.id) agents, self._warnings = package.materialize_agents(tenant_id=tenant_id, account_id=account.id) if agents: data["agent_packages"] = agents content = yaml.safe_dump(data, allow_unicode=True) + environment = ( + self._prepare_import_environment(data, account, app_id) if status != ImportStatus.PENDING else None + ) # If app_id is provided, check if it exists app = None if app_id: @@ -293,7 +316,7 @@ def import_app( error="Only workflow or advanced chat apps can be overwritten", ) - self._validate_workflow_overwrite(app, data) + self._validate_workflow_overwrite(AppMode(app.mode), data) # If major version mismatch, store import info in Redis if status == ImportStatus.PENDING: @@ -355,10 +378,9 @@ def import_app( dependencies=check_dependencies_pending_data, import_app_id=import_app_id, allow_premium_site_settings=allow_premium_site_settings, + environment=environment, ) - draft_var_srv = WorkflowDraftVariableService(session=self._session) - draft_var_srv.delete_app_workflow_variables(app_id=app.id) result_status = self._status_with_warnings(status) return Import( id=import_id, @@ -429,6 +451,7 @@ def confirm_import(self, *, import_id: str, account: Account) -> Import: raise ValueError("Current tenant is not set") allow_premium_site_settings = FeatureService.can_import_premium_site_settings(tenant_id) + environment = self._prepare_import_environment(data, account, pending_data.app_id) app = None if pending_data.app_id: app = self._load_app_for_overwrite(account, pending_data.app_id) @@ -440,7 +463,7 @@ def confirm_import(self, *, import_id: str, account: Account) -> Import: ) if app is not None: - self._validate_workflow_overwrite(app, data) + self._validate_workflow_overwrite(AppMode(app.mode), data) # Create or update app app = self._create_or_update_app( @@ -453,6 +476,7 @@ def confirm_import(self, *, import_id: str, account: Account) -> Import: icon=pending_data.icon, icon_background=pending_data.icon_background, allow_premium_site_settings=allow_premium_site_settings, + environment=environment, ) # Delete import info from Redis @@ -514,30 +538,25 @@ def cache_import_dependencies(*, app_id: str, dependencies: list[PluginDependenc CheckDependenciesPendingData(app_id=app_id, dependencies=dependencies).model_dump_json(), ) - def _load_app_for_overwrite(self, account: Account, app_id: str) -> App | None: + def _check_overwrite_access(self, account: Account, app_id: str) -> bool: if account.current_tenant_id is None: raise ValueError("Current tenant is not set") if dify_config.RBAC_ENABLED and self._session.in_transaction(): raise RuntimeError("App overwrite authorization requires a session without an active transaction") - rbac_allowed = not dify_config.RBAC_ENABLED or RBACService.CheckAccess.check( + return not dify_config.RBAC_ENABLED or RBACService.CheckAccess.check( account.current_tenant_id, account.id, scene=RBACPermission.APP_IMPORT_EXPORT_DSL, resource_type=RBACResourceScope.APP, resource_id=app_id, ) - app = self._session.scalar( - select(App) - .where( - App.id == app_id, - App.tenant_id == account.current_tenant_id, - App.status == "normal", - ) - .execution_options(populate_existing=True) + + def _load_app_for_overwrite(self, account: Account, app_id: str) -> App | None: + allowed = self._check_overwrite_access(account, app_id) + assert account.current_tenant_id is not None + return self._overwrites.load( + tenant_id=account.current_tenant_id, account_id=account.id, app_id=app_id, rbac_allowed=allowed ) - if app is not None and not rbac_allowed and app.maintainer != account.id: - raise NoPermissionError("You do not have permission to overwrite this app") - return app def _ensure_agent_import_permission(self, account: Account, *, app: App | None) -> None: if not dify_config.RBAC_ENABLED: @@ -562,10 +581,10 @@ def _ensure_agent_import_permission(self, account: Account, *, app: App | None) raise NoPermissionError("Agent DSL import permission is required to import an Agent App") @staticmethod - def _validate_workflow_overwrite(app: App, data: dict[str, Any]) -> None: + def _validate_workflow_overwrite(target_mode: AppMode, data: dict[str, Any]) -> None: """Apply editor compatibility checks to both YAML and package imports.""" app_mode = data.get("app", {}).get("mode") - if app.mode not in {AppMode.WORKFLOW, AppMode.ADVANCED_CHAT} or app_mode not in { + if target_mode not in {AppMode.WORKFLOW, AppMode.ADVANCED_CHAT} or app_mode not in { AppMode.WORKFLOW, AppMode.ADVANCED_CHAT, }: @@ -573,13 +592,46 @@ def _validate_workflow_overwrite(app: App, data: dict[str, Any]) -> None: # Package uploads cannot run the editor's YAML node checks before import. invalid_types = ( {BuiltinNodeTypes.END, "trigger-webhook", "trigger-schedule", "trigger-plugin"} - if app.mode == AppMode.ADVANCED_CHAT + if target_mode == AppMode.ADVANCED_CHAT else {BuiltinNodeTypes.ANSWER} ) nodes = data.get("workflow", {}).get("graph", {}).get("nodes", []) if any(node.get("data", {}).get("type") in invalid_types for node in nodes): raise ValueError("Workflow contains node types incompatible with the target App") + def _prepare_import_environment( + self, data: dict[str, Any], account: Account, app_id: str | None + ) -> PreparedEnvironmentVariables | None: + """Prepare secrets before the import starts writing App, Workflow and Agent rows.""" + if data.get("app", {}).get("mode") not in (AppMode.WORKFLOW, AppMode.ADVANCED_CHAT): + return None + workflow_data = data.get("workflow") + if not isinstance(workflow_data, dict): + return None + tenant_id = account.current_tenant_id + if tenant_id is None: + raise ValueError("Current tenant is not set") + source = None + if app_id is not None: + target = self._overwrites.snapshot( + tenant_id=tenant_id, + account_id=account.id, + app_id=app_id, + rbac_allowed=self._check_overwrite_access(account, app_id), + ) + if target is None: + raise ValueError("App not found") + self._validate_workflow_overwrite(target.mode, data) + source = target.workflow + return prepare_environment_variables( + tenant_id=tenant_id, + source=source, + variables=[ + variable_factory.build_environment_variable_from_mapping(value) + for value in workflow_data.get("environment_variables", []) + ], + ) + def _create_or_update_app( self, *, @@ -594,6 +646,7 @@ def _create_or_update_app( dependencies: list[PluginDependency] | None = None, import_app_id: str | None = None, allow_premium_site_settings: bool = True, + environment: PreparedEnvironmentVariables | None = None, ) -> App: """Create a new app or update an existing one.""" app_data = data.get("app", {}) @@ -680,22 +733,6 @@ def _create_or_update_app( if not workflow_data or not isinstance(workflow_data, dict): raise ValueError("Missing workflow data for workflow/advanced chat app") - environment_variables_list = workflow_data.get("environment_variables", []) - environment_variables = [ - variable_factory.build_environment_variable_from_mapping(obj) for obj in environment_variables_list - ] - conversation_variables_list = workflow_data.get("conversation_variables", []) - conversation_variables = [ - variable_factory.build_conversation_variable_from_mapping(obj) - for obj in conversation_variables_list - ] - - workflow_service = WorkflowService() - current_draft_workflow = workflow_service.get_draft_workflow(app_model=app, session=self._session) - if current_draft_workflow: - unique_hash = current_draft_workflow.unique_hash - else: - unique_hash = None graph = workflow_data.get("graph", {}) if not isinstance(graph, dict): raise ValueError("Workflow graph must be a mapping") @@ -717,37 +754,38 @@ def _create_or_update_app( raw_agent_packages = data.get("agent_packages") or {} if not isinstance(raw_agent_packages, Mapping): raise ValueError("agent_packages must be a mapping") - graph_for_sync = AgentDslService.graph_without_package_bindings(graph) if raw_agent_packages else graph - draft_workflow = workflow_service.sync_draft_workflow( - app_model=app, - graph=graph_for_sync, - features=workflow_data.get("features", {}), - unique_hash=unique_hash, - account=account, - environment_variables=environment_variables, - conversation_variables=conversation_variables, - session=self._session, - commit=not raw_agent_packages, - sync_agent_bindings=not raw_agent_packages, - ) - if raw_agent_packages: - _, warnings, retirement_candidates = AgentDslService(self._session).import_workflow_packages( - workflow=draft_workflow, + agent_packages = dict(raw_agent_packages) + graph_for_sync = AgentDslService.graph_without_package_bindings(graph) if agent_packages else graph + + def materialize(workflow: WorkflowSnapshot) -> DraftImportResult: + imported_graph, warnings, retired = AgentDslService(self._session).import_workflow_packages( + workflow=workflow, portable_graph=graph, - raw_packages=raw_agent_packages, + raw_packages=agent_packages, account=account, ) self._warnings.extend(warnings) - WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync( - session=self._session, - draft_workflow=draft_workflow, - ) - self._session.commit() - WorkflowAgentRetirementService.retire_unowned( - tenant_id=app.tenant_id, - agent_ids=retirement_candidates, - account_id=account.id, - ) + return DraftImportResult(imported_graph, retired) + + self._drafts.sync( + RequestContext(str(uuid4()), None, account.id, app.tenant_id), + WorkflowOwner(app.id), + DraftSyncCommand( + graph=graph_for_sync, + features=workflow_data.get("features", {}), + unique_hash=None, + # DSL replaces the entire definition under the write lock. + check_hash=False, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=workflow_data.get("environment_variables", []), + conversation_variables=workflow_data.get("conversation_variables", []), + clear_debug_variables=True, + ), + materialize=materialize if raw_agent_packages else None, + environment=environment, + ) case AppMode.CHAT | AppMode.AGENT_CHAT | AppMode.COMPLETION: # Initialize model config model_config = data.get("model_config") diff --git a/api/services/app_generate_service.py b/api/services/app_generate_service.py index a22a7ba5d47f0b..58b1b1916ab298 100644 --- a/api/services/app_generate_service.py +++ b/api/services/app_generate_service.py @@ -4,18 +4,11 @@ import threading import uuid from collections.abc import Callable, Generator, Mapping -from typing import TYPE_CHECKING, Any +from typing import Any from sqlalchemy.orm import Session from configs import dify_config -from core.app.apps.advanced_chat.app_generator import AdvancedChatAppGenerator -from core.app.apps.agent_app.app_generator import AgentAppGenerator -from core.app.apps.agent_chat.app_generator import AgentChatAppGenerator -from core.app.apps.chat.app_generator import ChatAppGenerator -from core.app.apps.completion.app_generator import CompletionAppGenerator -from core.app.apps.message_based_app_generator import MessageBasedAppGenerator -from core.app.apps.workflow.app_generator import WorkflowAppGenerator from core.app.entities.app_invoke_entities import InvokeFrom from core.app.features.rate_limiting import RateLimit from core.app.features.rate_limiting.rate_limit import rate_limit_context @@ -24,17 +17,30 @@ from core.trigger.constants import is_trigger_node_type from enums import DeploymentEdition, QuotaType from extensions.otel import AppGenerateHandler, trace_span +from models.annotation_reply import AnnotationReplies from models.model import Account, App, AppMode, EndUser -from models.workflow import Workflow, WorkflowRun +from models.workflow import Workflow +from repositories.workflow.definition_repository import WorkflowDefinitionStore +from services.app.generation.adapters.agent_app import AgentAppGenerator +from services.app.generation.adapters.agent_chat import AgentChatAppGenerator +from services.app.generation.adapters.chat import ChatAppGenerator +from services.app.generation.adapters.completion import CompletionAppGenerator +from services.app.generation.ports import ChatRecords +from services.app.generation.response import convert_to_event_stream +from services.app.generation.runtime import AppGenerationRuntime from services.errors.app import ( QuotaExceededError, TriggerWorkflowServiceModeUnavailableError, - WorkflowIdFormatError, - WorkflowNotFoundError, ) from services.errors.llm import InvokeRateLimitError +from services.knowledge.retrieval.ports import DatasetRetrievalFactory from services.quota_service import QuotaService, unlimited -from services.workflow_service import WorkflowService +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.execution.adapters.chatflow.app_generator import AdvancedChatAppGenerator +from services.workflow.execution.adapters.response_stream import WorkflowEventStream +from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator +from services.workflow.stream_reservation import WorkflowStreamReservation +from services.workflow.variable_contracts import WorkflowExecutionVariables from tasks.app_generate.workflow_execute_task import AppExecutionParams, workflow_based_app_execution_task logger = logging.getLogger(__name__) @@ -48,9 +54,6 @@ } ) -if TYPE_CHECKING: - from controllers.console.app.workflow import LoopNodeRunPayload - class AppGenerateService: @staticmethod @@ -106,8 +109,11 @@ def generate( invoke_from: InvokeFrom, *, session: Session, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, streaming: bool = True, root_node_id: str | None = None, + workflow_snapshot: WorkflowSnapshot | None = None, ): """ App Content Generate @@ -116,8 +122,10 @@ def generate( :param args: args :param invoke_from: invoke from :param streaming: streaming + :param workflow_snapshot: trusted trigger-debug revision; None selects the requested/current workflow :return: """ + cls._validate_workflow_snapshot(app_model, invoke_from, workflow_snapshot) return cls._run_with_guardrails( app_model=app_model, streaming=streaming, @@ -128,12 +136,122 @@ def generate( invoke_from=invoke_from, streaming=streaming, root_node_id=root_node_id, + workflow_snapshot=workflow_snapshot, session=session, + variables=variables, + runtime=runtime, + rate_limit=rate_limit, + request_id=request_id, + ), + ) + + @staticmethod + def _validate_workflow_snapshot( + app_model: App, invoke_from: InvokeFrom, workflow_snapshot: WorkflowSnapshot | None + ) -> None: + if workflow_snapshot is not None and ( + invoke_from != InvokeFrom.DEBUGGER + or app_model.mode != AppMode.WORKFLOW + or (workflow_snapshot.tenant_id, workflow_snapshot.app_id) != (app_model.tenant_id, app_model.id) + ): + raise ValueError("Workflow snapshot does not belong to this debugger invocation") + + @classmethod + @trace_span(AppGenerateHandler) + def generate_workflow_stream( + cls, + *, + app_model: App, + user: Account | EndUser, + workflow: Workflow, + args: Mapping[str, Any], + invoke_from: InvokeFrom, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, + root_node_id: str | None, + workflow_snapshot: WorkflowSnapshot | None, + reservation: WorkflowStreamReservation | None, + ): + """Prepare streaming execution from detached inputs, without a database transaction.""" + if app_model.mode not in {AppMode.WORKFLOW, AppMode.ADVANCED_CHAT}: + raise ValueError(f"Invalid app mode {app_model.mode}") + if (workflow.tenant_id, workflow.app_id) != (app_model.tenant_id, app_model.id): + raise ValueError("Workflow does not belong to this app") + cls._validate_workflow_snapshot(app_model, invoke_from, workflow_snapshot) + if app_model.mode == AppMode.WORKFLOW: + cls._ensure_workflow_service_mode_available(workflow=workflow, invoke_from=invoke_from) + return cls._run_with_guardrails( + app_model=app_model, + streaming=True, + action=lambda rate_limit, request_id: cls._stream_workflow( + app_model=app_model, + user=user, + workflow=workflow, + args=args, + invoke_from=invoke_from, + variables=variables, + runtime=runtime, + root_node_id=root_node_id, + workflow_snapshot=workflow_snapshot, + reservation=reservation, rate_limit=rate_limit, request_id=request_id, ), ) + @classmethod + def _stream_workflow( + cls, + *, + app_model: App, + user: Account | EndUser, + workflow: Workflow, + args: Mapping[str, Any], + invoke_from: InvokeFrom, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, + root_node_id: str | None, + workflow_snapshot: WorkflowSnapshot | None, + rate_limit: RateLimit, + request_id: str, + reservation: WorkflowStreamReservation | None = None, + ): + with rate_limit_context(rate_limit, request_id): + payload = AppExecutionParams.new( + app_model=app_model, + workflow=workflow, + user=user, + args=args, + invoke_from=invoke_from, + streaming=True, + call_depth=0, + root_node_id=root_node_id, + workflow_snapshot=workflow_snapshot, + workflow_run_id=(workflow_snapshot.execution_id if workflow_snapshot is not None else None) + or str(uuid.uuid4()), + ) + payload_json = payload.model_dump_json() + + def on_subscribe(): + workflow_based_app_execution_task.delay(payload_json) + + # Reserved debug executions start only with a subscriber. A fallback timer + # could enqueue after an unopened/failed response releases its Agent pins. + # Let dispatch errors reach the stream owner so it can release the pins. + subscribe = ( + (lambda: reservation.enqueue(on_subscribe)) + if reservation is not None + else cls._build_streaming_task_on_subscribe(on_subscribe) + ) + return rate_limit.generate( + convert_to_event_stream( + WorkflowEventStream.retrieve_events( + AppMode(app_model.mode), payload.workflow_run_id, on_subscribe=subscribe + ) + ), + request_id, + ) + @classmethod def _run_with_guardrails( cls, @@ -176,7 +294,10 @@ def _dispatch_generate( invoke_from: InvokeFrom, streaming: bool, root_node_id: str | None, + workflow_snapshot: WorkflowSnapshot | None, session: Session, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, rate_limit: RateLimit, request_id: str, ): @@ -188,8 +309,12 @@ def _dispatch_generate( match effective_mode: case AppMode.COMPLETION: return rate_limit.generate( - CompletionAppGenerator.convert_to_event_stream( - CompletionAppGenerator().generate( + convert_to_event_stream( + CompletionAppGenerator( + retrieval=runtime.retrieval, + annotations=runtime.annotation_replies, + records=runtime.chat_records, + ).generate( session=session, app_model=app_model, user=user, @@ -202,8 +327,14 @@ def _dispatch_generate( ) case AppMode.AGENT_CHAT: return rate_limit.generate( - AgentChatAppGenerator.convert_to_event_stream( - AgentChatAppGenerator().generate( + convert_to_event_stream( + AgentChatAppGenerator( + dataset_tools=runtime.dataset_tools, + annotations=runtime.annotation_replies, + records=runtime.chat_records, + draft_variable_saver=variables.saver_factory, + workflow_runtime=runtime, + ).generate( session=session, app_model=app_model, user=user, @@ -215,14 +346,22 @@ def _dispatch_generate( request_id, ) case AppMode.AGENT: + session.expunge_all() + session.close() return rate_limit.generate( - AgentAppGenerator.convert_to_event_stream( - AgentAppGenerator().generate( + convert_to_event_stream( + AgentAppGenerator( + forms=runtime.human_forms, + agent_configs=runtime.agent_configs, + tool_providers=runtime.tool_providers, + workflow_queries=runtime.tools, + records=runtime.chat_records, + annotations=runtime.annotation_replies, + ).generate( app_model=app_model, user=user, args=args, invoke_from=invoke_from, - session=session, streaming=streaming, ), ), @@ -230,8 +369,12 @@ def _dispatch_generate( ) case AppMode.CHAT: return rate_limit.generate( - ChatAppGenerator.convert_to_event_stream( - ChatAppGenerator().generate( + convert_to_event_stream( + ChatAppGenerator( + retrieval=runtime.retrieval, + annotations=runtime.annotation_replies, + records=runtime.chat_records, + ).generate( session=session, app_model=app_model, user=user, @@ -247,33 +390,17 @@ def _dispatch_generate( workflow = cls._get_workflow(app_model, invoke_from, workflow_id, session=session) if streaming: - # Streaming mode: subscribe to SSE and enqueue the execution on first subscriber - with rate_limit_context(rate_limit, request_id): - payload = AppExecutionParams.new( - app_model=app_model, - workflow=workflow, - user=user, - args=args, - invoke_from=invoke_from, - streaming=True, - call_depth=0, - workflow_run_id=str(uuid.uuid4()), - ) - payload_json = payload.model_dump_json() - - def on_subscribe(): - workflow_based_app_execution_task.delay(payload_json) - - on_subscribe = cls._build_streaming_task_on_subscribe(on_subscribe) - generator = AdvancedChatAppGenerator() - return rate_limit.generate( - generator.convert_to_event_stream( - generator.retrieve_events( - AppMode.ADVANCED_CHAT, - payload.workflow_run_id, - on_subscribe=on_subscribe, - ), - ), + return cls._stream_workflow( + app_model=app_model, + user=user, + workflow=workflow, + args=args, + invoke_from=invoke_from, + variables=variables, + runtime=runtime, + root_node_id=None, + workflow_snapshot=None, + rate_limit=rate_limit, request_id=request_id, ) @@ -283,9 +410,13 @@ def on_subscribe(): session_factory=session_factory.get_session_maker(), state_owner_user_id=workflow.created_by, ) - advanced_generator = AdvancedChatAppGenerator() + advanced_generator = AdvancedChatAppGenerator( + runtime=runtime, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ) return rate_limit.generate( - advanced_generator.convert_to_event_stream( + convert_to_event_stream( advanced_generator.generate( app_model=app_model, workflow=workflow, @@ -295,43 +426,29 @@ def on_subscribe(): workflow_run_id=str(uuid.uuid4()), streaming=False, pause_state_config=pause_config, - session=session, ) ), request_id=request_id, ) case AppMode.WORKFLOW: workflow_id = args.get("workflow_id") - workflow = cls._get_workflow(app_model, invoke_from, workflow_id, session=session) + workflow = cls._get_workflow( + app_model, invoke_from, workflow_id, session=session, snapshot=workflow_snapshot + ) cls._ensure_workflow_service_mode_available(workflow=workflow, invoke_from=invoke_from) if streaming: - with rate_limit_context(rate_limit, request_id): - payload = AppExecutionParams.new( - app_model=app_model, - workflow=workflow, - user=user, - args=args, - invoke_from=invoke_from, - streaming=True, - call_depth=0, - root_node_id=root_node_id, - workflow_run_id=str(uuid.uuid4()), - ) - payload_json = payload.model_dump_json() - - def on_subscribe(): - workflow_based_app_execution_task.delay(payload_json) - - on_subscribe = cls._build_streaming_task_on_subscribe(on_subscribe) - return rate_limit.generate( - WorkflowAppGenerator.convert_to_event_stream( - MessageBasedAppGenerator.retrieve_events( - AppMode.WORKFLOW, - payload.workflow_run_id, - on_subscribe=on_subscribe, - ), - ), - request_id, + return cls._stream_workflow( + app_model=app_model, + user=user, + workflow=workflow, + args=args, + invoke_from=invoke_from, + variables=variables, + runtime=runtime, + root_node_id=root_node_id, + workflow_snapshot=workflow_snapshot, + rate_limit=rate_limit, + request_id=request_id, ) pause_config = PauseStateLayerConfig( @@ -339,8 +456,12 @@ def on_subscribe(): state_owner_user_id=workflow.created_by, ) return rate_limit.generate( - WorkflowAppGenerator.convert_to_event_stream( - WorkflowAppGenerator().generate( + convert_to_event_stream( + WorkflowAppGenerator( + runtime=runtime, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ).generate( app_model=app_model, workflow=workflow, user=user, @@ -396,36 +517,42 @@ def generate_single_iteration( node_id: str, args: Any, *, - session: Session, + workflow: Workflow, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, streaming: bool = True, ): match app_model.mode: case AppMode.COMPLETION | AppMode.CHAT | AppMode.AGENT_CHAT: raise ValueError(f"Invalid app mode {app_model.mode}") case AppMode.ADVANCED_CHAT: - workflow = cls._get_workflow(app_model, InvokeFrom.DEBUGGER, session=session) - return AdvancedChatAppGenerator.convert_to_event_stream( - AdvancedChatAppGenerator().single_iteration_generate( + return convert_to_event_stream( + AdvancedChatAppGenerator( + runtime=runtime, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ).single_iteration_generate( app_model=app_model, workflow=workflow, node_id=node_id, user=user, args=args, streaming=streaming, - session=session, ) ) case AppMode.WORKFLOW: - workflow = cls._get_workflow(app_model, InvokeFrom.DEBUGGER, session=session) - return AdvancedChatAppGenerator.convert_to_event_stream( - WorkflowAppGenerator().single_iteration_generate( + return convert_to_event_stream( + WorkflowAppGenerator( + runtime=runtime, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ).single_iteration_generate( app_model=app_model, workflow=workflow, node_id=node_id, user=user, args=args, streaming=streaming, - session=session, ) ) case AppMode.CHANNEL | AppMode.RAG_PIPELINE: @@ -439,38 +566,44 @@ def generate_single_loop( app_model: App, user: Account, node_id: str, - args: LoopNodeRunPayload, + args: Mapping[str, Any], *, - session: Session, + workflow: Workflow, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, streaming: bool = True, ): match app_model.mode: case AppMode.COMPLETION | AppMode.CHAT | AppMode.AGENT_CHAT: raise ValueError(f"Invalid app mode {app_model.mode}") case AppMode.ADVANCED_CHAT: - workflow = cls._get_workflow(app_model, InvokeFrom.DEBUGGER, session=session) - return AdvancedChatAppGenerator.convert_to_event_stream( - AdvancedChatAppGenerator().single_loop_generate( + return convert_to_event_stream( + AdvancedChatAppGenerator( + runtime=runtime, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ).single_loop_generate( app_model=app_model, workflow=workflow, node_id=node_id, user=user, args=args, streaming=streaming, - session=session, ) ) case AppMode.WORKFLOW: - workflow = cls._get_workflow(app_model, InvokeFrom.DEBUGGER, session=session) - return AdvancedChatAppGenerator.convert_to_event_stream( - WorkflowAppGenerator().single_loop_generate( + return convert_to_event_stream( + WorkflowAppGenerator( + runtime=runtime, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ).single_loop_generate( app_model=app_model, workflow=workflow, node_id=node_id, user=user, args=args, streaming=streaming, - session=session, ) ) case AppMode.CHANNEL | AppMode.RAG_PIPELINE: @@ -487,6 +620,9 @@ def generate_more_like_this( invoke_from: InvokeFrom, *, session: Session, + records: ChatRecords, + retrieval: DatasetRetrievalFactory, + annotations: AnnotationReplies, streaming: bool = True, ) -> Mapping | Generator: """ @@ -498,7 +634,9 @@ def generate_more_like_this( :param streaming: streaming :return: """ - return CompletionAppGenerator().generate_more_like_this( + return CompletionAppGenerator( + retrieval=retrieval, annotations=annotations, records=records + ).generate_more_like_this( session=session, app_model=app_model, message_id=message_id, @@ -515,6 +653,7 @@ def _get_workflow( workflow_id: str | None = None, *, session: Session, + snapshot: WorkflowSnapshot | None = None, ) -> Workflow: """ Get workflow @@ -523,48 +662,10 @@ def _get_workflow( :param workflow_id: optional workflow id to specify a specific version :return: """ - workflow_service = WorkflowService() - - # If workflow_id is specified, get the specific workflow version - if workflow_id: - try: - _ = uuid.UUID(workflow_id) - except ValueError: - raise WorkflowIdFormatError(f"Invalid workflow_id format: '{workflow_id}'. ") - workflow = workflow_service.get_published_workflow_by_id( - app_model=app_model, workflow_id=workflow_id, session=session - ) - if not workflow: - raise WorkflowNotFoundError(f"Workflow not found with id: {workflow_id}") - return workflow - - if invoke_from == InvokeFrom.DEBUGGER: - # fetch draft workflow by app_model - workflow = workflow_service.get_draft_workflow(app_model=app_model, session=session) - - if not workflow: - raise ValueError("Workflow not initialized") - else: - # fetch published workflow by app_model - workflow = workflow_service.get_published_workflow(app_model=app_model, session=session) - - if not workflow: - raise ValueError("Workflow not published") - - return workflow - - @classmethod - def get_response_generator( - cls, - app_model: App, - workflow_run: WorkflowRun, - ): - if workflow_run.status.is_ended(): - # TODO(QuantumGhost): handled the ended scenario. - pass - - generator = AdvancedChatAppGenerator() - - return generator.convert_to_event_stream( - generator.retrieve_events(AppMode(app_model.mode), workflow_run.id), + return WorkflowDefinitionStore.prepare_execution_workflow( + app_model, + workflow_id=workflow_id, + draft=invoke_from == InvokeFrom.DEBUGGER, + session=session, + snapshot=snapshot, ) diff --git a/api/services/app_package_service.py b/api/services/app_package_service.py index 5d244723f69520..f5d40380401b66 100644 --- a/api/services/app_package_service.py +++ b/api/services/app_package_service.py @@ -15,8 +15,7 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator from core.db.session_factory import session_factory -from core.workflow.nodes.agent_v2.validators import WorkflowAgentNodeValidator -from models.agent import WorkflowAgentBindingType +from enums.agent import WorkflowAgentBindingType from models.agent_config_entities import WorkflowNodeJobConfig from models.model import App from services.agent.dsl_entities import AgentPackage @@ -35,6 +34,7 @@ validate_icon_references, ) from services.agent.roster_package_reader import RosterAgentPackageReader +from services.agent.workflow_validator import WorkflowAgentNodeValidator from services.dsl_content import DSL_MAX_SIZE from services.entities.dsl_entities import DslImportWarning from services.entities.site_dsl import SiteDsl diff --git a/api/services/app_preview_details_adapters.py b/api/services/app_preview_details_adapters.py index 2b15403c7e81a6..ff709a91545b53 100644 --- a/api/services/app_preview_details_adapters.py +++ b/api/services/app_preview_details_adapters.py @@ -1,3 +1,6 @@ +from models.tool_runtime_contracts import WorkflowToolQueries +from services.tools.provider_queries import ToolProviders + """Enrich detached app and workflow previews through legacy tool and key providers. The injected queries close their database sessions before this adapter calls tool, @@ -10,10 +13,8 @@ from dataclasses import replace from typing import cast, override -from core.agent.tool_configuration import mask_agent_tool_parameters from core.plugin.plugin_service import PluginService from core.tools.entities.tool_entities import ToolProviderType -from core.tools.tool_manager import ToolManager from core.workflow.environment_variables import load_environment_variables from core.workflow.llm_environment_variable import dump_environment_variable from models.model import AppMode @@ -27,10 +28,16 @@ AppPreviewWorkflow, ) from services.app_preview_query_service import AppPreviewRef +from services.tools.agent_configuration import mask_agent_tool_parameters +from services.tools.tool_manager import ToolManager class AppPreviewDetailsRuntime(AppPreviewDetails): - def __init__(self, *, details: AppPreviewDetailsQuery) -> None: + def __init__( + self, *, details: AppPreviewDetailsQuery, tool_providers: ToolProviders, workflow_queries: WorkflowToolQueries + ) -> None: + self._tool_providers = tool_providers + self._workflow_queries = workflow_queries self._details: AppPreviewDetailsQuery = details @override @@ -49,6 +56,8 @@ def get_detail(self, *, app: AppPreviewRef, account_id: str, active_workspace_id configuration = { **configuration, "agent_mode": mask_agent_tool_parameters( + tool_providers=self._tool_providers, + workflow_queries=self._workflow_queries, agent_mode=agent_mode, app_id=app.app_id, tenant_id=active_workspace_id, diff --git a/api/services/app_service.py b/api/services/app_service.py index b5bd0489c2fa78..ff34576db127b7 100644 --- a/api/services/app_service.py +++ b/api/services/app_service.py @@ -10,15 +10,15 @@ from pydantic import JsonValue from sqlalchemy import ColumnElement, delete, select from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from configs import dify_config from constants.model_template import default_app_templates from core.agent.publish_visibility import agent_has_workflow_callable_active_snapshot -from core.agent.tool_configuration import mask_agent_tool_parameters from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError from core.model_manager import ModelManager from enums import DeploymentEdition +from enums.agent import WorkflowAgentBindingType from events.app_event import app_was_created, app_was_deleted, app_was_updated from extensions.ext_database import db # noqa: F401 from graphon.model_runtime.entities.model_entities import ModelPropertyKey, ModelType @@ -35,18 +35,21 @@ AgentStatus, AgentWorkingResourceStatus, AgentWorkspaceBinding, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import AgentSoulConfig, AgentSoulModelConfig from models.model import App, AppMode, AppModelConfig, IconType, Site, load_annotation_reply_config from models.provider_ids import ModelProviderID from models.skill import AgentSkillBinding +from models.tool_runtime_contracts import WorkflowToolQueries from models.workflow import Workflow +from repositories.agent.home_snapshot_repository import AgentHomeSnapshotRepository +from repositories.agent.retirement_repository import WorkflowAgentRetirementRepository +from repositories.app.response import load_app_tool_references from services.agent.errors import AgentAccessNotReadyError, AgentNameConflictError -from services.agent.home_snapshot_service import AgentHomeSnapshotService -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService +from services.agent.retirement_service import WorkflowAgentRetirementService from services.agent.workspace_service import AgentWorkspaceService +from services.app.response_gateway import AppResponseGateway from services.billing_service import BillingService from services.enterprise import rbac_service as enterprise_rbac_service from services.enterprise.enterprise_service import EnterpriseService @@ -65,6 +68,8 @@ from services.rbac_agent_access_service import initialize_agent_rbac_access from services.system_feature_service import SystemFeatureService from services.tag_service import TagService +from services.tools.agent_configuration import mask_agent_tool_parameters +from services.tools.provider_queries import ToolProviders from tasks.collect_agent_resources_task import enqueue_agent_resource_collection from tasks.remove_app_and_related_data_task import remove_app_and_related_data_task @@ -122,8 +127,17 @@ class AppResponseView: """Expose App response properties through one caller-owned database session.""" def __init__( - self, app: App, *, session: Session, account: Account | None = None, access_mode: str | None = None + self, + app: App, + *, + tool_providers: ToolProviders, + workflow_queries: WorkflowToolQueries, + session: Session, + account: Account | None = None, + access_mode: str | None = None, ) -> None: + self._tool_providers = tool_providers + self._workflow_queries = workflow_queries self._app = app self._session = session self._account = account @@ -151,6 +165,8 @@ def app_model_config(self) -> AppModelConfigResponseView | None: tenant_id = self._account.current_tenant_id assert tenant_id is not None masked_agent_mode = mask_agent_tool_parameters( + tool_providers=self._tool_providers, + workflow_queries=self._workflow_queries, agent_mode=cast(Mapping[str, JsonValue], app_model_config.agent_mode_dict), app_id=self._app.id, tenant_id=tenant_id, @@ -186,7 +202,13 @@ def app_id(self) -> str | None: @property def deleted_tools(self) -> list[Any]: - return self._app.deleted_tools_with_session(session=self._session) + config = self._app.app_model_config_with_session(session=self._session) + references = load_app_tool_references( + session=self._session, + tenant_id=self._app.tenant_id, + tools=config.agent_mode_dict.get("tools", []) if config is not None else [], + ) + return AppResponseGateway.find_deleted_tools(self._app.tenant_id, references) @property def tags(self) -> Sequence[Any]: @@ -863,8 +885,7 @@ def delete_app(self, app: App, *, session: Session) -> None: binding_id=binding.id, ) retired_binding_ids.append(binding.id) - retired_snapshot_ids = AgentHomeSnapshotService.retire_all_for_agent( - session=session, + retired_snapshot_ids = AgentHomeSnapshotRepository(session).retire_all_for_agent( tenant_id=app.tenant_id, agent_id=backing_agent.id, ) @@ -886,7 +907,9 @@ def delete_app(self, app: App, *, session: Session) -> None: ) raise - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService( + WorkflowAgentRetirementRepository(sessionmaker(bind=session.get_bind(), expire_on_commit=False)) + ).retire_unowned( tenant_id=app.tenant_id, agent_ids=workflow_agent_ids, account_id=account_id, @@ -988,7 +1011,7 @@ def notify_created_app(*, event: AppEvent, account_id: str, backing_agent_id: st BillingService.clean_billing_info_cache(event.tenant_id) @staticmethod - def notify_deleted_app(deleted: AppDeletion, *, account_id: str | None) -> None: + def notify_deleted_app(deleted: AppDeletion, *, account_id: str | None, sessions: sessionmaker[Session]) -> None: app = deleted.app app_was_deleted.send(app) try: @@ -1000,7 +1023,7 @@ def notify_deleted_app(deleted: AppDeletion, *, account_id: str | None) -> None: ) raise - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService(WorkflowAgentRetirementRepository(sessions)).retire_unowned( tenant_id=app.tenant_id, agent_ids=deleted.workflow_agent_ids, account_id=account_id, diff --git a/api/services/data_migration/export_service.py b/api/services/data_migration/export_service.py index b5631744791544..6a343622e5a88d 100644 --- a/api/services/data_migration/export_service.py +++ b/api/services/data_migration/export_service.py @@ -7,7 +7,6 @@ import yaml from sqlalchemy.orm import Session -from core.tools.tool_manager import ToolManager from graphon.model_runtime.utils.encoders import jsonable_encoder from models import Account, Tenant from models.account import TenantAccountJoin @@ -26,8 +25,10 @@ ResourceType, ) from services.data_migration.package_service import MigrationPackageService -from services.tools.legacy_mcp_tools_manage_service import MCPToolManageService -from services.tools.legacy_workflow_tools_manage_service import WorkflowToolManageService +from services.tools.mcp_tools_manage_service import MCPToolManageService +from services.tools.provider_queries import ToolProviders +from services.tools.tool_manager import ToolManager +from services.tools.workflow_tools_manage_service import WorkflowToolManageService SUPPORTED_APP_MODES = {"workflow", "advanced-chat"} @@ -114,9 +115,13 @@ class MigrationExportService: def __init__( self, *, + workflow_tools: WorkflowToolManageService, + tool_providers: ToolProviders, package_service: MigrationPackageService | None = None, dependency_discovery_service: DependencyDiscoveryService | None = None, ) -> None: + self._workflow_tools = workflow_tools + self._tool_providers = tool_providers self.package_service = package_service or MigrationPackageService() self.dependency_discovery_service = dependency_discovery_service or DependencyDiscoveryService() @@ -254,6 +259,7 @@ def _export_api_tools( provider=provider_id, tenant_id=tenant_id, mask=not include_secrets, + tool_providers=self._tool_providers, ) if not include_secrets: tool_data.pop("credentials", None) @@ -297,7 +303,7 @@ def _export_workflow_tools( for provider_id in provider_ids: try: - tool_data = WorkflowToolManageService.get_workflow_tool_by_tool_id( + tool_data = self._workflow_tools.get_workflow_tool_by_tool_id( user_id=owner.id, tenant_id=tenant.id, workflow_tool_id=provider_id, diff --git a/api/services/data_migration/import_service.py b/api/services/data_migration/import_service.py index 7b29a77ff60980..3b62e795553775 100644 --- a/api/services/data_migration/import_service.py +++ b/api/services/data_migration/import_service.py @@ -8,26 +8,24 @@ from __future__ import annotations import json -from collections.abc import Iterable +from collections.abc import Callable, Iterable from dataclasses import dataclass -from typing import Any, cast +from typing import Any, Protocol, cast from uuid import UUID import sqlalchemy as sa import yaml from sqlalchemy import or_ -from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.orm import Session from configs import dify_config from core.entities.mcp_provider import IdentityMode, MCPAuthentication, MCPConfiguration from core.tools.entities.tool_entities import ApiProviderSchemaType, WorkflowToolParameterConfiguration -from extensions.ext_database import db -from libs.datetime_utils import naive_utc_now +from machinery.context import RequestContext from models import Account, ApiToken, Tenant, TenantAccountJoin, TenantAccountRole from models.enums import ApiTokenType from models.model import App from models.tools import ApiToolProvider, MCPToolProvider, WorkflowToolProvider -from services.app_dsl_service import AppDslService from services.data_migration.dependency_discovery_service import DependencyDiscoveryService from services.data_migration.entities import ( ConflictStrategy, @@ -43,11 +41,26 @@ ResourceReportItem, ResourceType, ) -from services.entities.dsl_entities import ImportStatus -from services.tools.legacy_api_tools_manage_service import ApiToolManageService -from services.tools.legacy_mcp_tools_manage_service import MCPToolManageService -from services.tools.legacy_workflow_tools_manage_service import WorkflowToolManageService -from services.workflow_service import WorkflowService +from services.entities.dsl_entities import Import, ImportStatus +from services.tools.api_tools_manage_service import ApiToolManageService +from services.tools.mcp_tools_manage_service import MCPToolManageService +from services.tools.workflow_tools_manage_service import WorkflowToolManageService + + +class WorkflowPublisher(Protocol): + def publish(self, context: RequestContext, app_id: str, *, marked_name: str, marked_comment: str) -> object: ... + + +class WorkflowDslImporter(Protocol): + def import_app( + self, + *, + account: Account, + import_mode: str, + yaml_content: str, + app_id: str | None = None, + import_app_id: str | None = None, + ) -> Import: ... @dataclass(frozen=True) @@ -147,8 +160,18 @@ class MigrationImportService: target_resolver: ImportTargetResolver - def __init__(self, *, target_resolver: ImportTargetResolver | None = None) -> None: + def __init__( + self, + *, + workflows: WorkflowPublisher, + workflow_tools: WorkflowToolManageService, + app_dsl: Callable[[Session], WorkflowDslImporter], + target_resolver: ImportTargetResolver | None = None, + ) -> None: self.target_resolver = target_resolver or ImportTargetResolver() + self._workflow_tools = workflow_tools + self._workflows = workflows + self._app_dsl = app_dsl def import_package(self, request: ImportRequest, *, session: Session) -> ImportResult: target = self.target_resolver.resolve(request, session=session) @@ -323,7 +346,7 @@ def _import_workflow_app( options: ImportOptions, session: Session, ) -> str: - import_service = AppDslService(session) + import_service = self._app_dsl(session) if existing_app is not None: existing_app_id = existing_app.id if dify_config.RBAC_ENABLED: @@ -652,7 +675,7 @@ def _import_workflow_tools( description = self._optional_string(workflow_tool_data.get("description")) or "" privacy_policy = self._optional_string(workflow_tool_data.get("privacy_policy")) or "" if existing is not None: - WorkflowToolManageService.update_workflow_tool( + self._workflow_tools.update_workflow_tool( user_id=account.id, tenant_id=target.tenant_id, workflow_tool_id=existing.id, @@ -668,7 +691,7 @@ def _import_workflow_tools( identifier = existing.id else: import_id = workflow_tool_id if options.id_strategy == IdStrategy.PRESERVE_ID else "" - WorkflowToolManageService.create_workflow_tool( + self._workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=target.tenant_id, workflow_app_id=resolved_app_id, @@ -707,24 +730,13 @@ def _ensure_workflow_app_is_published( raise MigrationDataError(f"Referenced workflow app was not found in target tenant: {app_id}") if app.workflow_id: return - workflow_service = WorkflowService() - with sessionmaker(db.engine).begin() as session: - app_in_session = session.get(App, app_id) - account_in_session = session.get(Account, account.id) - if app_in_session is None: - raise MigrationDataError(f"Referenced workflow app was not found in target tenant: {app_id}") - if account_in_session is None: - raise MigrationDataError(f"Operator account not found: {account.id}") - workflow = workflow_service.publish_workflow( - session=session, - app_model=app_in_session, - account=account_in_session, - marked_name="Migration import", - marked_comment="Published automatically for workflow tool import.", - ) - app_in_session.workflow_id = workflow.id - app_in_session.updated_by = account.id - app_in_session.updated_at = naive_utc_now() + self._workflows.publish( + RequestContext("migration-import", None, account.id, target.tenant_id), + app_id, + marked_name="Migration import", + marked_comment="Published automatically for workflow tool import.", + ) + session.refresh(app) def _import_mcp_tools( self, diff --git a/api/services/human_input_delivery_test_service.py b/api/services/human_input_delivery_test_service.py index d8bdca092018bd..aac8b47c2212d3 100644 --- a/api/services/human_input_delivery_test_service.py +++ b/api/services/human_input_delivery_test_service.py @@ -1,7 +1,5 @@ from __future__ import annotations -from dataclasses import dataclass, field -from enum import StrEnum from typing import Protocol from sqlalchemy import Engine, select @@ -10,7 +8,6 @@ from configs import dify_config from extensions.ext_database import db from extensions.ext_mail import mail -from graphon.runtime import VariablePool from libs.email_template_renderer import render_email_template from models import Account, TenantAccountJoin from models.human_input_delivery import ( @@ -21,44 +18,13 @@ MemberRecipient, ) from services.feature_service import FeatureService - - -class DeliveryTestStatus(StrEnum): - OK = "ok" - FAILED = "failed" - - -@dataclass(frozen=True) -class DeliveryTestEmailRecipient: - email: str - form_token: str - - -@dataclass(frozen=True) -class DeliveryTestContext: - tenant_id: str - app_id: str - node_id: str - node_title: str | None - rendered_content: str - template_vars: dict[str, str] = field(default_factory=dict) - recipients: list[DeliveryTestEmailRecipient] = field(default_factory=list) - variable_pool: VariablePool | None = None - - -@dataclass(frozen=True) -class DeliveryTestResult: - status: DeliveryTestStatus - delivered_to: list[str] = field(default_factory=list) - warnings: list[str] = field(default_factory=list) - - -class DeliveryTestError(Exception): - pass - - -class DeliveryTestUnsupportedError(DeliveryTestError): - pass +from services.human_input.contracts import ( + DeliveryTestContext, + DeliveryTestError, + DeliveryTestResult, + DeliveryTestStatus, + DeliveryTestUnsupportedError, +) def _build_form_link(token: str | None) -> str | None: diff --git a/api/services/installed_app_generation_adapters.py b/api/services/installed_app_generation_adapters.py index 0b2751fd6e7624..25d0d33b008e9e 100644 --- a/api/services/installed_app_generation_adapters.py +++ b/api/services/installed_app_generation_adapters.py @@ -6,10 +6,11 @@ from sqlalchemy.orm import Session, sessionmaker -from core.app.apps.completion.app_generator import CompletionAppGenerator from core.app.entities.app_invoke_entities import InvokeFrom from models import Account, App from services.account_errors import AccountNotFoundError +from services.app.generation.response import convert_to_event_stream +from services.app.generation.runtime import AppGenerationRuntime from services.app_definition_query_service import AppDefinitionUnavailableError from services.app_generate_service import AppGenerateService from services.conversation_service import ConversationService @@ -18,6 +19,7 @@ GenerationStream, InstalledAppGenerationRuntime, ) +from services.workflow.variable_contracts import WorkflowExecutionVariables logger = logging.getLogger(__name__) @@ -25,7 +27,7 @@ class _MoreLikeThisEventStream: def __init__(self, source: Generator[Mapping[str, object] | str, None, None]) -> None: self._source: Generator[Mapping[str, object] | str, None, None] = source - self._events: GenerationStream = cast(GenerationStream, CompletionAppGenerator.convert_to_event_stream(source)) + self._events: GenerationStream = cast(GenerationStream, convert_to_event_stream(source)) self._closed: bool = False def __iter__(self) -> "_MoreLikeThisEventStream": @@ -55,8 +57,16 @@ def close(self) -> None: class AppGenerateServiceRuntime(InstalledAppGenerationRuntime): - def __init__(self, *, session_factory: sessionmaker[Session]) -> None: + def __init__( + self, + *, + session_factory: sessionmaker[Session], + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, + ) -> None: self._session_factory: sessionmaker[Session] = session_factory + self._variables = variables + self._runtime = runtime @override def generate( @@ -75,6 +85,8 @@ def generate( ) return self._run_generation( lambda session: AppGenerateService.generate( + variables=self._variables, + runtime=self._runtime, session=session, app_model=app, user=account, @@ -97,6 +109,9 @@ def generate_more_like_this( def generate(session: Session) -> GenerationResponse: response = AppGenerateService.generate_more_like_this( + retrieval=self._runtime.retrieval, + annotations=self._runtime.annotation_replies, + records=self._runtime.chat_records, session=session, app_model=app, user=account, diff --git a/api/services/knowledge/dataset_service.py b/api/services/knowledge/dataset_service.py index 62b057d00323ef..0ad5652e0aed20 100644 --- a/api/services/knowledge/dataset_service.py +++ b/api/services/knowledge/dataset_service.py @@ -14,7 +14,6 @@ from redis.exceptions import LockNotOwnedError from sqlalchemy import ColumnElement, case, delete, exists, func, select, tuple_, update from sqlalchemy.orm import Session -from werkzeug.exceptions import Forbidden, NotFound from configs import dify_config from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError @@ -40,7 +39,6 @@ ChildChunk, Dataset, DatasetAutoDisableLog, - DatasetCollectionBinding, DatasetPermission, DatasetPermissionEnum, DatasetProcessRule, @@ -63,20 +61,31 @@ from models.source import DataSourceOauthBinding from models.workflow import Workflow from repositories.knowledge import dataset_api_key_bindings -from repositories.knowledge.dataset_read_repository import get_dataset_doc_form, get_latest_dataset_process_rule +from repositories.knowledge.collection_binding_repository import DatasetCollectionBindingRepository +from repositories.knowledge.dataset_read_repository import ( + get_dataset_doc_form, + get_latest_dataset_process_rule, +) +from repositories.knowledge.dataset_read_repository import get_datasets_by_ids as load_datasets_by_ids +from repositories.knowledge.document_repository import next_document_position from repositories.knowledge.segment_repository import query_child_chunks +from repositories.workflow.definition_repository import WorkflowDefinitionStore from services.document_indexing_proxy.document_indexing_task_proxy import DocumentIndexingTaskProxy from services.document_indexing_proxy.duplicate_document_indexing_task_proxy import DuplicateDocumentIndexingTaskProxy from services.enterprise import rbac_service as enterprise_rbac_service from services.entities.feature_entities import FeatureModel from services.entities.knowledge_entities.rag_pipeline_entities import ( - KnowledgeConfiguration, RagPipelineDatasetCreateEntity, ) from services.errors.base import NoPermissionError from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingError -from services.errors.dataset import DatasetNameDuplicateError -from services.errors.document import DocumentIndexingError +from services.errors.dataset import DatasetNameDuplicateError, DatasetNotFoundError +from services.errors.document import ( + DocumentAccessDeniedError, + DocumentIndexingError, + DocumentNotFoundError, + DocumentSourceNotFoundError, +) from services.errors.file import FileNotExistsError from services.feature_service import FeatureService from services.file_service import FileService @@ -99,12 +108,10 @@ SegmentMutationService, validate_segment_values, ) -from services.rag_pipeline.rag_pipeline import RagPipelineService from services.tag_application_service import TagTargetQuery from tasks.add_document_to_index_task import add_document_to_index_task from tasks.batch_clean_document_task import batch_clean_document_task from tasks.clean_notion_document_task import clean_notion_document_task -from tasks.deal_dataset_index_update_task import deal_dataset_index_update_task from tasks.deal_dataset_vector_index_task import deal_dataset_vector_index_task from tasks.document_indexing_update_task import document_indexing_update_task from tasks.recover_document_indexing_task import recover_document_indexing_task @@ -277,21 +284,14 @@ def get_datasets_by_ids( # Check if ids is not empty to avoid WHERE false condition if not ids: return [], 0 - stmt = select(Dataset).where(Dataset.id.in_(ids), Dataset.tenant_id == tenant_id) - - if dify_config.RBAC_ENABLED and accessible_dataset_ids is not None: - requested_dataset_ids = set(ids) - accessible_dataset_ids = [ - dataset_id for dataset_id in accessible_dataset_ids if dataset_id in requested_dataset_ids - ] - accessible_filter: ColumnElement[bool] = Dataset.id.in_(accessible_dataset_ids) - if include_own_datasets and user: - accessible_filter = sa.or_(Dataset.maintainer == user.id, accessible_filter) - stmt = stmt.where(accessible_filter) - - datasets = paginate_query(stmt, session=session, page=1, per_page=len(ids), max_per_page=len(ids)) - - return datasets.items, datasets.total + datasets = load_datasets_by_ids( + tenant_id, + ids, + session=session, + accessible_dataset_ids=accessible_dataset_ids if dify_config.RBAC_ENABLED else None, + maintainer_id=user.id if include_own_datasets and user else None, + ) + return datasets, len(datasets) @staticmethod def create_empty_dataset( @@ -755,9 +755,8 @@ def _update_pipeline_knowledge_base_node_data(dataset: Dataset, updata_user_id: return try: - rag_pipeline_service = RagPipelineService(session) - published_workflow = rag_pipeline_service.get_published_workflow(pipeline) - draft_workflow = rag_pipeline_service.get_draft_workflow(pipeline) + published_workflow = WorkflowDefinitionStore.get_published_workflow(pipeline, session=session) + draft_workflow = WorkflowDefinitionStore.get_draft_workflow(pipeline, session=session) # update knowledge nodes def update_knowledge_nodes(workflow_graph: str) -> str: @@ -883,7 +882,7 @@ def _configure_embedding_model_for_high_quality(data, filtered_data, session: Se embedding_model_name = embedding_model.model_name filtered_data["embedding_model"] = embedding_model_name filtered_data["embedding_model_provider"] = embedding_model.provider - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( + dataset_collection_binding = DatasetCollectionBindingRepository.get_dataset_collection_binding( embedding_model.provider, embedding_model_name, session, @@ -1022,7 +1021,7 @@ def _apply_new_embedding_settings(dataset, data, filtered_data, session: Session embedding_model_name = embedding_model.model_name filtered_data["embedding_model"] = embedding_model_name filtered_data["embedding_model_provider"] = embedding_model.provider - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( + dataset_collection_binding = DatasetCollectionBindingRepository.get_dataset_collection_binding( embedding_model.provider, embedding_model_name, session, @@ -1077,170 +1076,6 @@ def _check_summary_index_setting_model_changed(dataset: Dataset, data: dict[str, return False - @staticmethod - def update_rag_pipeline_dataset_settings( - dataset: Dataset, - knowledge_configuration: KnowledgeConfiguration, - has_published: bool = False, - *, - session: Session, - ): - if not current_user or not current_user.current_tenant_id: - raise ValueError("Current user or current tenant not found") - dataset = session.merge(dataset) - if not has_published: - dataset.chunk_structure = knowledge_configuration.chunk_structure - dataset.indexing_technique = IndexTechniqueType(knowledge_configuration.indexing_technique) - if knowledge_configuration.indexing_technique == IndexTechniqueType.HIGH_QUALITY: - model_manager = ModelManager.for_tenant(tenant_id=current_user.current_tenant_id) - embedding_model = model_manager.get_model_instance( - tenant_id=current_user.current_tenant_id, # ignore type error - provider=knowledge_configuration.embedding_model_provider or "", - model_type=ModelType.TEXT_EMBEDDING, - model=knowledge_configuration.embedding_model or "", - ) - is_multimodal = DatasetService.check_is_multimodal_model( - current_user.current_tenant_id, - knowledge_configuration.embedding_model_provider, - knowledge_configuration.embedding_model, - ) - dataset.is_multimodal = is_multimodal - embedding_model_name = embedding_model.model_name - dataset.embedding_model = embedding_model_name - dataset.embedding_model_provider = embedding_model.provider - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( - embedding_model.provider, - embedding_model_name, - session, - ) - dataset.collection_binding_id = dataset_collection_binding.id - elif knowledge_configuration.indexing_technique == IndexTechniqueType.ECONOMY: - dataset.keyword_number = knowledge_configuration.keyword_number - else: - raise ValueError("Invalid index method") - dataset.retrieval_model = knowledge_configuration.retrieval_model.model_dump() - # Update summary_index_setting if provided - if knowledge_configuration.summary_index_setting is not None: - dataset.summary_index_setting = knowledge_configuration.summary_index_setting - session.add(dataset) - else: - if dataset.chunk_structure and dataset.chunk_structure != knowledge_configuration.chunk_structure: - raise ValueError("Chunk structure is not allowed to be updated.") - action = None - if dataset.indexing_technique != knowledge_configuration.indexing_technique: - # if update indexing_technique - if knowledge_configuration.indexing_technique == IndexTechniqueType.ECONOMY: - raise ValueError("Knowledge base indexing technique is not allowed to be updated to economy.") - elif knowledge_configuration.indexing_technique == IndexTechniqueType.HIGH_QUALITY: - action = "add" - # get embedding model setting - try: - model_manager = ModelManager.for_tenant(tenant_id=current_user.current_tenant_id) - embedding_model = model_manager.get_model_instance( - tenant_id=current_user.current_tenant_id, - provider=knowledge_configuration.embedding_model_provider, - model_type=ModelType.TEXT_EMBEDDING, - model=knowledge_configuration.embedding_model, - ) - embedding_model_name = embedding_model.model_name - dataset.embedding_model = embedding_model_name - dataset.embedding_model_provider = embedding_model.provider - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( - embedding_model.provider, - embedding_model_name, - session, - ) - is_multimodal = DatasetService.check_is_multimodal_model( - current_user.current_tenant_id, - knowledge_configuration.embedding_model_provider, - knowledge_configuration.embedding_model, - ) - dataset.is_multimodal = is_multimodal - dataset.collection_binding_id = dataset_collection_binding.id - dataset.indexing_technique = IndexTechniqueType(knowledge_configuration.indexing_technique) - except LLMBadRequestError: - raise ValueError( - "No Embedding Model available. Please configure a valid provider " - "in the Settings -> Model Provider." - ) - except ProviderTokenNotInitError as ex: - raise ValueError(ex.description) - else: - # add default plugin id to both setting sets, to make sure the plugin model provider is consistent - # Skip embedding model checks if not provided in the update request - if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY: - skip_embedding_update = False - try: - # Handle existing model provider - plugin_model_provider = dataset.embedding_model_provider - plugin_model_provider_str = None - if plugin_model_provider: - plugin_model_provider_str = str(ModelProviderID(plugin_model_provider)) - - # Handle new model provider from request - new_plugin_model_provider = knowledge_configuration.embedding_model_provider - new_plugin_model_provider_str = None - if new_plugin_model_provider: - new_plugin_model_provider_str = str(ModelProviderID(new_plugin_model_provider)) - - # Only update embedding model if both values are provided and different from current - if ( - plugin_model_provider_str != new_plugin_model_provider_str - or knowledge_configuration.embedding_model != dataset.embedding_model - ): - action = "update" - model_manager = ModelManager.for_tenant(tenant_id=current_user.current_tenant_id) - embedding_model = None - try: - embedding_model = model_manager.get_model_instance( - tenant_id=current_user.current_tenant_id, - provider=knowledge_configuration.embedding_model_provider, - model_type=ModelType.TEXT_EMBEDDING, - model=knowledge_configuration.embedding_model, - ) - except ProviderTokenNotInitError: - # If we can't get the embedding model, skip updating it - # and keep the existing settings if available - # Skip the rest of the embedding model update - skip_embedding_update = True - if not skip_embedding_update: - if embedding_model: - embedding_model_name = embedding_model.model_name - dataset.embedding_model = embedding_model_name - dataset.embedding_model_provider = embedding_model.provider - dataset_collection_binding = ( - DatasetCollectionBindingService.get_dataset_collection_binding( - embedding_model.provider, - embedding_model_name, - session, - ) - ) - dataset.collection_binding_id = dataset_collection_binding.id - is_multimodal = DatasetService.check_is_multimodal_model( - current_user.current_tenant_id, - knowledge_configuration.embedding_model_provider, - knowledge_configuration.embedding_model, - ) - dataset.is_multimodal = is_multimodal - except LLMBadRequestError: - raise ValueError( - "No Embedding Model available. Please configure a valid provider " - "in the Settings -> Model Provider." - ) - except ProviderTokenNotInitError as ex: - raise ValueError(ex.description) - elif dataset.indexing_technique == IndexTechniqueType.ECONOMY: - if dataset.keyword_number != knowledge_configuration.keyword_number: - dataset.keyword_number = knowledge_configuration.keyword_number - dataset.retrieval_model = knowledge_configuration.retrieval_model.model_dump() - # Update summary_index_setting if provided - if knowledge_configuration.summary_index_setting is not None: - dataset.summary_index_setting = knowledge_configuration.summary_index_setting - session.add(dataset) - session.commit() - if action: - deal_dataset_index_update_task.delay(dataset.id, action) - @staticmethod def delete_dataset(dataset_id, user, session: Session): dataset = DatasetService.get_dataset(dataset_id, session) @@ -1704,11 +1539,11 @@ def prepare_document_batch_download_zip( """ dataset = DatasetService.get_dataset(dataset_id, session) if not dataset: - raise NotFound("Dataset not found.") + raise DatasetNotFoundError("Dataset not found.") try: DatasetService.check_dataset_permission(dataset, current_user, session) except NoPermissionError as e: - raise Forbidden(str(e)) + raise DocumentAccessDeniedError(str(e)) from e upload_files_by_document_id = DocumentService._get_upload_files_by_document_id_for_zip_download( dataset_id=dataset_id, @@ -1738,12 +1573,12 @@ def _get_upload_file_id_for_upload_file_document( Normalize and validate `Document -> UploadFile` linkage for download flows. """ if document.data_source_type != DataSourceType.UPLOAD_FILE: - raise NotFound(invalid_source_message) + raise DocumentSourceNotFoundError(invalid_source_message) data_source_info: dict[str, Any] = document.data_source_info_dict or {} upload_file_id: str | None = data_source_info.get("upload_file_id") if not upload_file_id: - raise NotFound(missing_file_message) + raise DocumentSourceNotFoundError(missing_file_message) return str(upload_file_id) @@ -1760,7 +1595,7 @@ def _get_upload_file_for_upload_file_document(document: Document, session: Sessi upload_files_by_id = FileService.get_upload_files_by_ids(document.tenant_id, [upload_file_id], session=session) upload_file = upload_files_by_id.get(upload_file_id) if not upload_file: - raise NotFound("Uploaded file not found.") + raise DocumentSourceNotFoundError("Uploaded file not found.") return upload_file @staticmethod @@ -1783,7 +1618,7 @@ def _get_upload_files_by_document_id_for_zip_download( missing_document_ids: set[str] = set(document_id_list) - set(documents_by_id.keys()) if missing_document_ids: - raise NotFound("Document not found.") + raise DocumentNotFoundError("Document not found.") upload_file_ids: list[str] = [] upload_file_ids_by_document_id: dict[str, str] = {} @@ -1799,7 +1634,7 @@ def _get_upload_files_by_document_id_for_zip_download( upload_files_by_id = FileService.get_upload_files_by_ids(tenant_id, upload_file_ids, session=session) missing_upload_file_ids: set[str] = set(upload_file_ids) - set(upload_files_by_id.keys()) if missing_upload_file_ids: - raise NotFound("Only uploaded-file documents can be downloaded as ZIP.") + raise DocumentSourceNotFoundError("Only uploaded-file documents can be downloaded as ZIP.") return { document_id: upload_files_by_id[upload_file_id] @@ -2123,16 +1958,6 @@ def sync_website_document(dataset: Dataset, document: Document, session: Session sync_website_document_indexing_task.delay(dataset.id, document.id) - @staticmethod - def get_documents_position(dataset_id, session: Session): - document = session.scalar( - select(Document).where(Document.dataset_id == dataset_id).order_by(Document.position.desc()).limit(1) - ) - if document: - return document.position + 1 - else: - return 1 - @staticmethod def save_document_with_dataset_id( dataset: Dataset, @@ -2271,7 +2096,7 @@ def save_prepared_documents( dataset_embedding_model_provider = knowledge_config.embedding_model_provider dataset.embedding_model = dataset_embedding_model dataset.embedding_model_provider = dataset_embedding_model_provider - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( + dataset_collection_binding = DatasetCollectionBindingRepository.get_dataset_collection_binding( dataset_embedding_model_provider, dataset_embedding_model, session ) dataset.collection_binding_id = dataset_collection_binding.id @@ -2349,7 +2174,7 @@ def save_prepared_documents( session.add(dataset_process_rule) session.flush() assert dataset_process_rule - position = DocumentService.get_documents_position(dataset.id, session) + position = next_document_position(dataset.id, session) document_ids = [] duplicate_document_ids = [] removed_notion_ids: list[str] = [] @@ -2595,7 +2420,7 @@ def save_prepared_documents( # dataset_embedding_model_provider = embedding_model.provider # dataset.embedding_model = dataset_embedding_model # dataset.embedding_model_provider = dataset_embedding_model_provider - # dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( + # dataset_collection_binding = DatasetCollectionBindingRepository.get_dataset_collection_binding( # dataset_embedding_model_provider, dataset_embedding_model # ) # dataset.collection_binding_id = dataset_collection_binding.id @@ -2648,7 +2473,7 @@ def save_prepared_documents( # session.commit() # lock_name = "add_document_lock_dataset_id_{}".format(dataset.id) # with redis_client.lock(lock_name, timeout=600): - # position = DocumentService.get_documents_position(dataset.id) + # position = next_document_position(dataset.id) # document_ids = [] # duplicate_document_ids = [] # if knowledge_config.data_source.info_list.data_source_type == "upload_file": @@ -2935,7 +2760,7 @@ def update_prepared_document( document = DocumentService.get_document(dataset.id, document_data.original_document_id, session=session) if document is None: - raise NotFound("Document not found") + raise DocumentNotFoundError("Document not found") if document.display_status != "available": raise ValueError("Document is not available") # save process rule @@ -3107,7 +2932,7 @@ def build_dataset_for_documents( if knowledge_config.indexing_technique == IndexTechniqueType.HIGH_QUALITY: assert knowledge_config.embedding_model_provider assert knowledge_config.embedding_model - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( + dataset_collection_binding = DatasetCollectionBindingRepository.get_dataset_collection_binding( knowledge_config.embedding_model_provider, knowledge_config.embedding_model, session, @@ -3847,51 +3672,6 @@ def get_segments_by_document_and_dataset( return session.scalars(query).all() -class DatasetCollectionBindingService: - @classmethod - def get_dataset_collection_binding( - cls, provider_name: str, model_name: str, session: Session, collection_type: str = "dataset" - ) -> DatasetCollectionBinding: - dataset_collection_binding = session.scalar( - select(DatasetCollectionBinding) - .where( - DatasetCollectionBinding.provider_name == provider_name, - DatasetCollectionBinding.model_name == model_name, - DatasetCollectionBinding.type == collection_type, - ) - .order_by(DatasetCollectionBinding.created_at) - .limit(1) - ) - - if not dataset_collection_binding: - dataset_collection_binding = DatasetCollectionBinding( - provider_name=provider_name, - model_name=model_name, - collection_name=Dataset.gen_collection_name_by_id(str(uuid.uuid4())), - type=collection_type, - ) - session.add(dataset_collection_binding) - session.flush() - return dataset_collection_binding - - @classmethod - def get_dataset_collection_binding_by_id_and_type( - cls, collection_binding_id: str, session: Session, collection_type: str = "dataset" - ) -> DatasetCollectionBinding: - dataset_collection_binding = session.scalar( - select(DatasetCollectionBinding) - .where( - DatasetCollectionBinding.id == collection_binding_id, DatasetCollectionBinding.type == collection_type - ) - .order_by(DatasetCollectionBinding.created_at) - .limit(1) - ) - if not dataset_collection_binding: - raise ValueError("Dataset collection binding not found") - - return dataset_collection_binding - - class DatasetPermissionService: @classmethod def get_dataset_partial_member_list(cls, dataset_id, session: Session): diff --git a/api/services/message_service.py b/api/services/message_service.py index 639b06debbfdb2..2a89416c421d07 100644 --- a/api/services/message_service.py +++ b/api/services/message_service.py @@ -7,7 +7,6 @@ from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker -from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager from core.app.apps.agent_app.app_feature_projection import merge_agent_app_features from core.app.entities.app_invoke_entities import InvokeFrom, get_credit_usage_app_type from core.llm_generator.llm_generator import LLMGenerator @@ -45,6 +44,7 @@ MessageNotExistsError, SuggestedQuestionsAfterAnswerDisabledError, ) +from services.workflow.execution.adapters.chatflow.app_config_manager import AdvancedChatAppConfigManager from services.workflow_service import WorkflowService logger = logging.getLogger(__name__) diff --git a/api/services/plugin/app_invocation.py b/api/services/plugin/app_invocation.py new file mode 100644 index 00000000000000..f7747e35a95b20 --- /dev/null +++ b/api/services/plugin/app_invocation.py @@ -0,0 +1,363 @@ +import uuid +from collections.abc import Generator, Mapping +from typing import Any, Protocol, cast + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from core.app.app_config.common.parameters_mapping import get_parameters_from_feature_dict +from core.app.entities.app_invoke_entities import InvokeFrom +from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig +from core.db.session_factory import create_session +from core.plugin.backwards_invocation.base import BaseBackwardsInvocation +from extensions.ext_database import db +from models import Account, TenantAccountJoin +from models.annotation_reply import AnnotationReplies +from models.model import ( + App, + AppMode, + AppModelConfig, + AppModelConfigDict, + EndUser, + load_annotation_reply_config, +) +from models.workflow import Workflow +from services.app.generation.adapters.agent_chat import AgentChatAppGenerator +from services.app.generation.adapters.chat import ChatAppGenerator +from services.app.generation.adapters.completion import CompletionAppGenerator +from services.app.generation.ports import ChatRecords +from services.app.generation.runtime import AppGenerationRuntime +from services.knowledge.retrieval.ports import DatasetRetrievalFactory +from services.workflow.execution.adapters.chatflow.app_generator import AdvancedChatAppGenerator +from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator + + +class AppScopedEndUserProvisioner(Protocol): + def get_or_create_end_user( + self, + tenant_id: str, + app_id: str, + user_id: str | None = None, + ) -> EndUser: ... + + +class PluginAppBackwardsInvocation(BaseBackwardsInvocation): + @classmethod + def fetch_app_info(cls, app_id: str, tenant_id: str) -> Mapping: + """ + Fetch app info + """ + app = cls._get_app(app_id, tenant_id) + + """Retrieve app parameters.""" + if app.mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}: + workflow = cls._get_workflow(app) + if workflow is None: + raise ValueError("unexpected app type") + + features_dict: dict[str, Any] = workflow.features_dict + user_input_form = workflow.user_input_form(to_old_structure=True) + else: + app_model_config_dict = cls._get_app_model_config_dict(app) + if app_model_config_dict is None: + raise ValueError("unexpected app type") + + features_dict = cast(dict[str, Any], app_model_config_dict) + + user_input_form = features_dict.get("user_input_form", []) + + return { + "data": get_parameters_from_feature_dict(features_dict=features_dict, user_input_form=user_input_form), + } + + @classmethod + def invoke_app( + cls, + app_id: str, + user_id: str, + tenant_id: str, + conversation_id: str | None, + query: str | None, + stream: bool, + inputs: Mapping, + files: list[dict], + session: Session, + end_users: AppScopedEndUserProvisioner, + runtime: AppGenerationRuntime, + ) -> Generator[Mapping | str, None, None] | Mapping: + """ + invoke app + """ + app = cls._get_app(app_id, tenant_id) + if not user_id: + user = end_users.get_or_create_end_user(app.tenant_id, app.id, None) + else: + try: + user = cls._get_user(user_id, app) + except ValueError: + # Plugins such as WeCom Bot pass external sender IDs rather than EndUser UUIDs. + user = end_users.get_or_create_end_user(app.tenant_id, app.id, user_id) + + conversation_id = conversation_id or "" + + match app.mode: + case AppMode.ADVANCED_CHAT | AppMode.AGENT_CHAT | AppMode.CHAT: + if not query: + raise ValueError("missing query") + + return cls.invoke_chat_app( + app, user, conversation_id, query, stream, inputs, files, session, runtime=runtime + ) + case AppMode.WORKFLOW: + workflow = cls._get_workflow(app) + if not workflow: + raise ValueError("unexpected app type") + return cls.invoke_workflow_app(app, workflow, user, stream, inputs, files, runtime=runtime) + case AppMode.COMPLETION: + return cls.invoke_completion_app( + app, + user, + stream, + inputs, + files, + session, + annotations=runtime.annotation_replies, + retrieval=runtime.retrieval, + records=runtime.chat_records, + ) + case _: + raise ValueError("unexpected app type") + + @classmethod + def invoke_chat_app( + cls, + app: App, + user: Account | EndUser, + conversation_id: str, + query: str, + stream: bool, + inputs: Mapping, + files: list[dict], + session: Session, + runtime: AppGenerationRuntime, + ) -> Generator[Mapping | str, None, None] | Mapping: + """ + invoke chat app + """ + match app.mode: + case AppMode.ADVANCED_CHAT: + workflow = cls._get_workflow(app) + if not workflow: + raise ValueError("unexpected app type") + + pause_config = PauseStateLayerConfig( + session_factory=db.engine, + state_owner_user_id=workflow.created_by, + ) + + return AdvancedChatAppGenerator( + runtime=runtime, + ).generate( + app_model=app, + workflow=workflow, + user=user, + args={ + "inputs": inputs, + "query": query, + "files": files, + "conversation_id": conversation_id, + }, + invoke_from=InvokeFrom.SERVICE_API, + workflow_run_id=str(uuid.uuid4()), + streaming=stream, + pause_state_config=pause_config, + ) + case AppMode.AGENT_CHAT: + return AgentChatAppGenerator( + annotations=runtime.annotation_replies, + records=runtime.chat_records, + dataset_tools=runtime.dataset_tools, + workflow_runtime=runtime, + ).generate( + app_model=app, + user=user, + args={ + "inputs": inputs, + "query": query, + "files": files, + "conversation_id": conversation_id, + }, + invoke_from=InvokeFrom.SERVICE_API, + streaming=stream, + session=session, + ) + case AppMode.CHAT: + return ChatAppGenerator( + retrieval=runtime.retrieval, annotations=runtime.annotation_replies, records=runtime.chat_records + ).generate( + app_model=app, + user=user, + args={ + "inputs": inputs, + "query": query, + "files": files, + "conversation_id": conversation_id, + }, + invoke_from=InvokeFrom.SERVICE_API, + streaming=stream, + session=session, + ) + case _: + raise ValueError("unexpected app type") + + @classmethod + def invoke_workflow_app( + cls, + app: App, + workflow: Workflow, + user: EndUser | Account, + stream: bool, + inputs: Mapping, + files: list[dict], + *, + runtime: AppGenerationRuntime, + ) -> Generator[Mapping | str, None, None] | Mapping: + """ + invoke workflow app + """ + pause_config = PauseStateLayerConfig( + session_factory=db.engine, + state_owner_user_id=workflow.created_by, + ) + + return WorkflowAppGenerator( + runtime=runtime, + ).generate( + app_model=app, + workflow=workflow, + user=user, + args={"inputs": inputs, "files": files}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=stream, + call_depth=1, + pause_state_config=pause_config, + ) + + @classmethod + def invoke_completion_app( + cls, + app: App, + user: EndUser | Account, + stream: bool, + inputs: Mapping, + files: list[dict], + session: Session, + *, + records: ChatRecords, + annotations: AnnotationReplies, + retrieval: DatasetRetrievalFactory, + ) -> Generator[Mapping | str, None, None] | Mapping: + """ + invoke completion app + """ + return CompletionAppGenerator(retrieval=retrieval, annotations=annotations, records=records).generate( + app_model=app, + user=user, + args={"inputs": inputs, "files": files}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=stream, + session=session, + ) + + @classmethod + def _get_user(cls, user_id: str, app: App) -> EndUser | Account: + """ + get the user by user id + """ + with create_session() as session: + stmt = select(EndUser).where( + EndUser.id == user_id, + EndUser.tenant_id == app.tenant_id, + EndUser.app_id == app.id, + ) + user = session.scalar(stmt) + if not user: + stmt = select(EndUser).where( + EndUser.session_id == user_id, + EndUser.tenant_id == app.tenant_id, + EndUser.app_id == app.id, + ) + user = session.scalar(stmt) + if not user: + stmt = select(Account).where( + Account.id == user_id, + Account.id == TenantAccountJoin.account_id, + TenantAccountJoin.tenant_id == app.tenant_id, + ) + user = session.scalar(stmt) + if user: + session.expunge(user) + + if not user: + raise ValueError("user not found") + + return user + + @classmethod + def _get_app(cls, app_id: str, tenant_id: str) -> App: + """ + get app + """ + try: + with create_session() as session: + app = session.scalar(select(App).where(App.id == app_id, App.tenant_id == tenant_id).limit(1)) + if app: + session.expunge(app) + except Exception as e: + raise ValueError("app not found") from e + + if not app: + raise ValueError("app not found") + + return app + + @classmethod + def _get_workflow(cls, app: App) -> Workflow | None: + """ + get workflow with an owned session, additionally scoped by tenant and app + + `App.workflow_with_session` resolves by `workflow_id` alone; a plugin + invocation must also confirm the workflow belongs to this tenant and app. + """ + if not app.workflow_id: + return None + + with create_session() as session: + workflow = session.scalar( + select(Workflow) + .where(Workflow.id == app.workflow_id, Workflow.tenant_id == app.tenant_id, Workflow.app_id == app.id) + .limit(1) + ) + if workflow: + session.expunge(workflow) + return workflow + + @classmethod + def _get_app_model_config_dict(cls, app: App) -> AppModelConfigDict | None: + """ + get app model config features without relying on request-scoped session-backed model properties + """ + if not app.app_model_config_id: + return None + + with create_session() as session: + app_model_config = session.scalar( + select(AppModelConfig) + .where(AppModelConfig.id == app.app_model_config_id, AppModelConfig.app_id == app.id) + .limit(1) + ) + if app_model_config is None: + return None + + annotation_reply = load_annotation_reply_config(session, app_model_config.app_id) + return app_model_config.to_dict(annotation_reply=annotation_reply) diff --git a/api/services/rag_pipeline/pipeline_generate_service.py b/api/services/rag_pipeline/pipeline_generate_service.py index 12e85611b040a1..908ba9abefb75c 100644 --- a/api/services/rag_pipeline/pipeline_generate_service.py +++ b/api/services/rag_pipeline/pipeline_generate_service.py @@ -1,19 +1,11 @@ from collections.abc import Mapping from typing import Any -from sqlalchemy.orm import Session - -from configs import dify_config -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom from models.dataset import Pipeline -from models.enums import IndexingStatus -from models.model import Account, App, EndUser -from models.workflow import Workflow -from repositories.knowledge.dataset_read_repository import get_pipeline_dataset -from services.knowledge.dataset_service import DocumentService -from services.knowledge.resource_scope import DatasetRef, DocumentRef -from services.rag_pipeline.rag_pipeline import RagPipelineService +from models.model import Account, EndUser +from services.app.generation.response import convert_to_event_stream +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator class PipelineGenerateService: @@ -26,7 +18,6 @@ def generate( invoke_from: InvokeFrom, streaming: bool = True, *, - session: Session, generator: PipelineGenerator, ): """ @@ -39,17 +30,9 @@ def generate( :return: """ try: - workflow = cls._get_workflow(pipeline, invoke_from, session) - if original_document_id := args.get("original_document_id"): - dataset = get_pipeline_dataset(pipeline, session) - if dataset is None or dataset.tenant_id != pipeline.tenant_id: - raise ValueError("Pipeline dataset is required") - dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id) - document_ref = dataset_ref.document(original_document_id) - cls.update_document_status(document_ref, session=session) - return PipelineGenerator.convert_to_event_stream( + workflow = generator.load_workflow(pipeline, invoke_from) + return convert_to_event_stream( generator.generate( - session=session, pipeline=pipeline, workflow=workflow, user=user, @@ -64,14 +47,6 @@ def generate( except Exception: raise - @staticmethod - def _get_max_active_requests(app_model: App) -> int: - app_limit = app_model.max_active_requests or dify_config.APP_DEFAULT_ACTIVE_REQUESTS - config_limit = dify_config.APP_MAX_ACTIVE_REQUESTS - # Filter out infinite (0) values and return the minimum, or 0 if both are infinite - limits = [limit for limit in [app_limit, config_limit] if limit > 0] - return min(limits) if limits else 0 - @classmethod def generate_single_iteration( cls, @@ -79,13 +54,12 @@ def generate_single_iteration( user: Account, node_id: str, args: Any, - session: Session, streaming: bool = True, *, generator: PipelineGenerator, ): - workflow = cls._get_workflow(pipeline, InvokeFrom.DEBUGGER, session) - return PipelineGenerator.convert_to_event_stream( + workflow = generator.load_workflow(pipeline, InvokeFrom.DEBUGGER) + return convert_to_event_stream( generator.single_iteration_generate( pipeline=pipeline, workflow=workflow, @@ -93,7 +67,6 @@ def generate_single_iteration( user=user, args=args, streaming=streaming, - session=session, ) ) @@ -104,13 +77,12 @@ def generate_single_loop( user: Account, node_id: str, args: Any, - session: Session, streaming: bool = True, *, generator: PipelineGenerator, ): - workflow = cls._get_workflow(pipeline, InvokeFrom.DEBUGGER, session) - return PipelineGenerator.convert_to_event_stream( + workflow = generator.load_workflow(pipeline, InvokeFrom.DEBUGGER) + return convert_to_event_stream( generator.single_loop_generate( pipeline=pipeline, workflow=workflow, @@ -118,41 +90,5 @@ def generate_single_loop( user=user, args=args, streaming=streaming, - session=session, ) ) - - @classmethod - def _get_workflow(cls, pipeline: Pipeline, invoke_from: InvokeFrom, session: Session) -> Workflow: - """ - Get workflow - :param pipeline: pipeline - :param invoke_from: invoke from - :return: - """ - rag_pipeline_service = RagPipelineService(session) - if invoke_from == InvokeFrom.DEBUGGER: - # fetch draft workflow by app_model - workflow = rag_pipeline_service.get_draft_workflow(pipeline=pipeline) - - if not workflow: - raise ValueError("Workflow not initialized") - else: - # fetch published workflow by app_model - workflow = rag_pipeline_service.get_published_workflow(pipeline=pipeline) - - if not workflow: - raise ValueError("Workflow not published") - - return workflow - - @classmethod - def update_document_status(cls, document_ref: DocumentRef, *, session: Session) -> None: - """Set a document in the owner-bound dataset to waiting.""" - document = next( - iter(DocumentService.get_documents_by_ids(document_ref.dataset, [document_ref.document_id], session)), None - ) - if document is None: - raise ValueError("Pipeline document not found") - document.indexing_status = IndexingStatus.WAITING - session.add(document) diff --git a/api/services/rag_pipeline/rag_pipeline.py b/api/services/rag_pipeline/rag_pipeline.py index 485b6d3f7a6fbf..29d6b9c9044910 100644 --- a/api/services/rag_pipeline/rag_pipeline.py +++ b/api/services/rag_pipeline/rag_pipeline.py @@ -2,59 +2,18 @@ import logging import re import threading -import time -from collections.abc import Callable, Generator, Mapping, Sequence -from datetime import UTC, datetime -from typing import Any, cast -from uuid import uuid4 +from collections.abc import Mapping, Sequence +from typing import Any from sqlalchemy import func, select from sqlalchemy.orm import Session, sessionmaker import contexts from configs import dify_config -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom -from core.datasource.datasource_manager import DatasourceManager -from core.datasource.entities.datasource_entities import ( - DatasourceMessage, - DatasourceProviderType, - GetOnlineDocumentPageContentRequest, - OnlineDocumentPagesMessage, - OnlineDriveBrowseFilesRequest, - OnlineDriveBrowseFilesResponse, - WebsiteCrawlMessage, -) -from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin -from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin -from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin from core.helper import marketplace -from core.rag.entities import DatasourceCompletedEvent, DatasourceErrorEvent, DatasourceProcessingEvent -from core.repositories.factory import DifyCoreRepositoryFactory -from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository -from core.workflow.llm_environment_variable import validate_llm_environment_model_references -from core.workflow.node_factory import LATEST_VERSION, get_node_type_classes_mapping -from core.workflow.system_variables import ( - SystemVariableKey, - build_bootstrap_variables, - build_system_variables, - default_system_variables, - get_system_segment, -) -from core.workflow.variable_pool_initializer import add_variables_to_pool -from core.workflow.workflow_entry import WorkflowEntry -from enterprise.telemetry.draft_trace import enqueue_draft_node_execution_trace -from extensions.ext_database import db -from graphon.entities import WorkflowNodeExecution -from graphon.enums import BuiltinNodeTypes, ErrorStrategy, NodeType, WorkflowNodeExecutionStatus -from graphon.errors import WorkflowNodeRunFailedError -from graphon.graph_events import GraphNodeEventBase, NodeRunFailedEvent, NodeRunSucceededEvent -from graphon.node_events import NodeRunResult -from graphon.nodes.base.node import Node -from graphon.nodes.container_effects import ContainerAwaitRequest +from graphon.enums import BuiltinNodeTypes, NodeType from graphon.nodes.http_request import HTTP_REQUEST_CONFIG_FILTER_KEY, build_http_request_config -from graphon.runtime import VariablePool -from graphon.variables.variables import Variable, VariableBase from libs.infinite_scroll_pagination import InfiniteScrollPagination from libs.login import resolve_account_fallback, resolve_tenant_id_fallback from models import Account @@ -66,46 +25,37 @@ PipelineCustomizedTemplate, PipelineRecommendedPlugin, ) -from models.enums import IndexingStatus, WorkflowRunTriggeredFrom +from models.enums import WorkflowRunTriggeredFrom from models.model import EndUser from models.workflow import ( Workflow, WorkflowNodeExecutionModel, - WorkflowNodeExecutionTriggeredFrom, WorkflowRun, - WorkflowType, ) from repositories.factory import DifyAPIRepositoryFactory -from repositories.knowledge.dataset_read_repository import get_pipeline_dataset +from repositories.workflow.definition_repository import WorkflowDefinitionStore, workflow_record from services.credentials.query import CredentialQuery from services.data_source.provider_service import DatasourceProviderService from services.entities.knowledge_entities.rag_pipeline_entities import ( - KnowledgeConfiguration, PipelineTemplateInfoEntity, ) -from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError from services.errors.rag_pipeline import RagPipelineResourceNotFoundError -from services.knowledge.resource_scope import DatasetRef from services.rag_pipeline.pipeline_template.pipeline_template_factory import PipelineTemplateRetrievalFactory from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineDslService -from services.tools.legacy_builtin_tools_manage_service import BuiltinToolManageService -from services.workflow_draft_variable_service import DraftVariableSaver, DraftVarLoader +from services.tools.builtin_tools_manage_service import BuiltinToolManageService +from services.tools.provider_queries import ToolProviders +from services.workflow.contracts import WorkflowRecord +from services.workflow.execution.adapters.node_factory import LATEST_VERSION, get_node_type_classes_mapping +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator from services.workflow_node_execution_trace_service import ( WorkflowNodeExecutionTrace, assemble_workflow_node_execution_traces, ) from services.workflow_ref_service import WorkflowRef -from services.workflow_restore import apply_published_workflow_snapshot_to_draft logger = logging.getLogger(__name__) -def _build_seeded_variable_pool(variables: Sequence[Variable]) -> VariablePool: - variable_pool = VariablePool() - add_variables_to_pool(variable_pool, variables) - return variable_pool - - class RagPipelineService: _session: Session @@ -113,7 +63,7 @@ def __init__(self, session: Session, session_maker: sessionmaker | None = None): """Initialize RagPipelineService with repository dependencies.""" self._session = session if session_maker is None: - session_maker = sessionmaker(bind=db.engine, expire_on_commit=False) + session_maker = sessionmaker(bind=session.get_bind(), expire_on_commit=False) self._node_execution_service_repo = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( session_maker ) @@ -269,60 +219,22 @@ def get_draft_workflow(self, pipeline: Pipeline) -> Workflow | None: """ Get draft workflow """ - # fetch draft workflow by rag pipeline - workflow = self._session.scalar( - select(Workflow) - .where( - Workflow.tenant_id == pipeline.tenant_id, - Workflow.app_id == pipeline.id, - Workflow.version == "draft", - ) - .limit(1) - ) + return WorkflowDefinitionStore.get_draft_workflow(pipeline, session=self._session) - # return draft workflow - return workflow + def get_draft_workflow_record(self, pipeline: Pipeline) -> WorkflowRecord | None: + workflow = self.get_draft_workflow(pipeline) + return workflow_record(workflow, self._session) if workflow is not None else None def get_published_workflow(self, pipeline: Pipeline) -> Workflow | None: """ Get published workflow """ - if not pipeline.workflow_id: - return None + return WorkflowDefinitionStore.get_published_workflow(pipeline, session=self._session) - # fetch published workflow by workflow_id - workflow = self._session.scalar( - select(Workflow) - .where( - Workflow.tenant_id == pipeline.tenant_id, - Workflow.app_id == pipeline.id, - Workflow.id == pipeline.workflow_id, - ) - .limit(1) - ) - - return workflow - - def get_published_workflow_by_id(self, pipeline: Pipeline, workflow_id: str) -> Workflow | None: - """Fetch and lock a published Workflow snapshot for restoration. - - The source lock is held until the service transaction ends, preventing - concurrent deletion while restore copies its Workflow snapshot fields. - """ - workflow = self._session.scalar( - select(Workflow) - .where( - Workflow.tenant_id == pipeline.tenant_id, - Workflow.app_id == pipeline.id, - Workflow.id == workflow_id, - ) - .limit(1) - .with_for_update() - ) - if workflow and workflow.version == Workflow.VERSION_DRAFT: - raise IsDraftWorkflowError("source workflow must be published") - return workflow + def get_published_workflow_record(self, pipeline: Pipeline) -> WorkflowRecord | None: + workflow = self.get_published_workflow(pipeline) + return workflow_record(workflow, self._session) if workflow is not None else None def get_all_published_workflow( self, @@ -333,188 +245,20 @@ def get_all_published_workflow( limit: int, user_id: str | None, named_only: bool = False, - ) -> tuple[Sequence[Workflow], bool]: + ) -> tuple[Sequence[WorkflowRecord], bool]: """ Get published workflow with pagination """ - if not pipeline.workflow_id: - return [], False - - stmt = ( - select(Workflow) - .where(Workflow.app_id == pipeline.id) - .order_by(Workflow.version.desc()) - .limit(limit + 1) - .offset((page - 1) * limit) - ) - - if user_id: - stmt = stmt.where(Workflow.created_by == user_id) - - if named_only: - stmt = stmt.where(Workflow.marked_name != "") - - workflows = session.scalars(stmt).all() - - has_more = len(workflows) > limit - if has_more: - workflows = workflows[:-1] - - return workflows, has_more - - def sync_draft_workflow( - self, - *, - pipeline: Pipeline, - graph: dict[str, Any], - unique_hash: str | None, - account: Account, - environment_variables: Sequence[VariableBase], - conversation_variables: Sequence[VariableBase], - rag_pipeline_variables: list, - ) -> Workflow: - """ - Sync draft workflow - :raises WorkflowHashNotEqualError - """ - # fetch draft workflow by app_model - workflow = self.get_draft_workflow(pipeline=pipeline) - - if workflow and workflow.unique_hash != unique_hash: - raise WorkflowHashNotEqualError() - - # create draft workflow if not found - if not workflow: - workflow = Workflow( - tenant_id=pipeline.tenant_id, - app_id=pipeline.id, - features="{}", - type=WorkflowType.RAG_PIPELINE.value, - version="draft", - graph=json.dumps(graph), - created_by=account.id, - environment_variables=environment_variables, - conversation_variables=conversation_variables, - rag_pipeline_variables=rag_pipeline_variables, - ) - self._session.add(workflow) - self._session.flush() - pipeline.workflow_id = workflow.id - # update draft workflow if found - else: - workflow.graph = json.dumps(graph) - workflow.updated_by = account.id - workflow.updated_at = datetime.now(UTC).replace(tzinfo=None) - workflow.environment_variables = environment_variables - workflow.conversation_variables = conversation_variables - workflow.rag_pipeline_variables = rag_pipeline_variables - # commit db session changes - self._session.commit() - - # trigger workflow events TODO - # app_draft_workflow_was_synced.send(pipeline, synced_draft_workflow=workflow) - - # return draft workflow - return workflow - - def restore_published_workflow_to_draft( - self, - *, - pipeline: Pipeline, - workflow_id: str, - account: Account, - ) -> Workflow: - """Restore a published pipeline workflow snapshot into the draft workflow. - - Pipelines reuse the shared draft-restore field copy helper, but still own - the pipeline-specific flush/link step that wires a newly created draft - back onto ``pipeline.workflow_id``. The source version remains locked - through snapshot-field copy and commit. - """ - source_workflow = self.get_published_workflow_by_id(pipeline=pipeline, workflow_id=workflow_id) - if not source_workflow: - raise WorkflowNotFoundError("Workflow not found.") - - draft_workflow = self.get_draft_workflow(pipeline=pipeline) - draft_workflow, is_new_draft = apply_published_workflow_snapshot_to_draft( - tenant_id=pipeline.tenant_id, - app_id=pipeline.id, - source_workflow=source_workflow, - draft_workflow=draft_workflow, - account=account, - updated_at_factory=lambda: datetime.now(UTC).replace(tzinfo=None), - ) - - if is_new_draft: - self._session.add(draft_workflow) - self._session.flush() - pipeline.workflow_id = draft_workflow.id - - self._session.commit() - - return draft_workflow - - def publish_workflow( - self, - *, - session: Session, - pipeline: Pipeline, - account: Account, - ) -> Workflow: - draft_workflow_stmt = select(Workflow).where( - Workflow.tenant_id == pipeline.tenant_id, - Workflow.app_id == pipeline.id, - Workflow.version == "draft", + workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( + session=session, + app_model=pipeline, + page=page, + limit=limit, + user_id=user_id, + named_only=named_only, ) - draft_workflow = session.scalar(draft_workflow_stmt) - if not draft_workflow: - raise ValueError("No valid workflow found.") - validate_llm_environment_model_references( - graph=draft_workflow.graph_dict, - environment_variables=draft_workflow.environment_variables, - ) - - # create new workflow - workflow = Workflow.new( - tenant_id=pipeline.tenant_id, - app_id=pipeline.id, - type=draft_workflow.type, - version=str(datetime.now(UTC).replace(tzinfo=None)), - graph=draft_workflow.graph, - features=draft_workflow.features, - created_by=account.id, - environment_variables=draft_workflow.environment_variables, - conversation_variables=draft_workflow.conversation_variables, - rag_pipeline_variables=draft_workflow.rag_pipeline_variables, - marked_name="", - marked_comment="", - ) - # commit db session changes - session.add(workflow) - - graph = workflow.graph_dict - nodes = graph.get("nodes", []) - # dataset_service imports RagPipelineService, so defer this reverse dependency. - from services.knowledge.dataset_service import DatasetService - - for node in nodes: - if node.get("data", {}).get("type") == "knowledge-index": - knowledge_configuration = node.get("data", {}) - knowledge_configuration = KnowledgeConfiguration.model_validate(knowledge_configuration) - - # update dataset - dataset = get_pipeline_dataset(pipeline, session=session) - if not dataset: - raise ValueError("Dataset not found") - DatasetService.update_rag_pipeline_dataset_settings( - session=session, - dataset=dataset, - knowledge_configuration=knowledge_configuration, - has_published=pipeline.is_published, - ) - # return new workflow - return workflow + return [workflow_record(workflow, session) for workflow in workflows], has_more def get_default_block_configs(self) -> list[dict]: """ @@ -577,518 +321,6 @@ def get_default_block_config( return default_config - def run_draft_workflow_node( - self, pipeline: Pipeline, node_id: str, user_inputs: dict[str, Any], account: Account - ) -> WorkflowNodeExecutionModel | None: - """ - Run draft workflow node - """ - # fetch draft workflow by app_model - draft_workflow = self.get_draft_workflow(pipeline=pipeline) - if not draft_workflow: - raise ValueError("Workflow not initialized") - - # run draft workflow node - start_at = time.perf_counter() - node_config = draft_workflow.get_node_config_by_id(node_id) - - eclosing_node_type_and_id = draft_workflow.get_enclosing_node_type_and_id(node_config) - if eclosing_node_type_and_id: - _, enclosing_node_id = eclosing_node_type_and_id - else: - enclosing_node_id = None - - workflow_node_execution = self._handle_node_run_result( - getter=lambda: WorkflowEntry.single_step_run( - workflow=draft_workflow, - node_id=node_id, - user_inputs=user_inputs, - user_id=account.id, - variable_pool=_build_seeded_variable_pool( - build_bootstrap_variables( - system_variables=default_system_variables(), - environment_variables=draft_workflow.environment_variables, - ) - ), - variable_loader=DraftVarLoader( - engine=db.engine, - app_id=pipeline.id, - tenant_id=pipeline.tenant_id, - user_id=account.id, - ), - ), - start_at=start_at, - tenant_id=pipeline.tenant_id, - node_id=node_id, - ) - workflow_node_execution.workflow_id = draft_workflow.id - - # Create repository and save the node execution - - repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( - session_factory=db.engine, - tenant_id=pipeline.tenant_id, - user=account, - app_id=pipeline.id, - triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP, - ) - repository.save(workflow_node_execution) - - # Convert node_execution to WorkflowNodeExecution after save - workflow_node_execution_db_model = self._node_execution_service_repo.get_execution_by_id( - workflow_node_execution.id - ) - - with sessionmaker(bind=db.engine).begin() as session: - draft_var_saver = DraftVariableSaver( - session=session, - tenant_id=pipeline.tenant_id, - app_id=pipeline.id, - node_id=workflow_node_execution.node_id, - node_type=workflow_node_execution.node_type, - enclosing_node_id=enclosing_node_id, - node_execution_id=workflow_node_execution.id, - user=account, - ) - draft_var_saver.save( - process_data=workflow_node_execution.process_data, - outputs=workflow_node_execution.outputs, - ) - if isinstance(workflow_node_execution_db_model, WorkflowNodeExecutionModel): - enqueue_draft_node_execution_trace( - execution=workflow_node_execution_db_model, - outputs=workflow_node_execution.outputs, - workflow_execution_id=None, - user_id=account.id, - ) - return workflow_node_execution_db_model - - def run_datasource_workflow_node( - self, - pipeline: Pipeline, - node_id: str, - user_inputs: dict[str, Any], - account: Account, - datasource_type: str, - is_published: bool, - credential_id: str | None = None, - *, - datasource_providers: DatasourceProviderService, - ) -> Generator[Mapping[str, Any], None, None]: - """ - Run published workflow datasource - """ - try: - if is_published: - # fetch published workflow by app_model - workflow = self.get_published_workflow(pipeline=pipeline) - else: - workflow = self.get_draft_workflow(pipeline=pipeline) - if not workflow: - raise ValueError("Workflow not initialized") - - # run draft workflow node - datasource_node_data = None - datasource_nodes = workflow.graph_dict.get("nodes", []) - for datasource_node in datasource_nodes: - if datasource_node.get("id") == node_id: - datasource_node_data = datasource_node.get("data", {}) - break - if not datasource_node_data: - raise ValueError("Datasource node data not found") - - variables_map = {} - - datasource_parameters = datasource_node_data.get("datasource_parameters", {}) - for key, value in datasource_parameters.items(): - param_value = value.get("value") - - match param_value: - case None | "" | [] | {}: - variables_map[key] = param_value - case str(): - # handle string type parameter value, check if it contains variable reference pattern - pattern = r"\{\{#([a-zA-Z0-9_]{1,50}(?:\.[a-zA-Z0-9_][a-zA-Z0-9_]{0,29}){1,10})#\}\}" - match_result = re.match(pattern, param_value) - if match_result: - # extract variable path and try to get value from user inputs - full_path = match_result.group(1) - last_part = full_path.split(".")[-1] - variables_map[key] = user_inputs.get(last_part, param_value) - else: - variables_map[key] = param_value - case list() if param_value: - # handle list type parameter value, check if the last element is in user inputs - last_part = param_value[-1] - variables_map[key] = user_inputs.get(last_part, param_value) - case _: - # other type directly use original value - variables_map[key] = param_value - - datasource_runtime = DatasourceManager.get_datasource_runtime( - provider_id=f"{datasource_node_data.get('plugin_id')}/{datasource_node_data.get('provider_name')}", - datasource_name=datasource_node_data.get("datasource_name"), - tenant_id=pipeline.tenant_id, - datasource_type=DatasourceProviderType(datasource_type), - ) - credentials = datasource_providers.get_datasource_credentials( - tenant_id=pipeline.tenant_id, - provider=datasource_node_data.get("provider_name"), - plugin_id=datasource_node_data.get("plugin_id"), - credential_id=credential_id, - ) - if credentials: - datasource_runtime.runtime.credentials = credentials - match datasource_type: - case DatasourceProviderType.ONLINE_DOCUMENT: - datasource_runtime = cast(OnlineDocumentDatasourcePlugin, datasource_runtime) - online_document_result: Generator[OnlineDocumentPagesMessage, None, None] = ( - datasource_runtime.get_online_document_pages( - user_id=account.id, - datasource_parameters=user_inputs, - provider_type=datasource_runtime.datasource_provider_type(), - ) - ) - start_time = time.time() - start_event = DatasourceProcessingEvent( - total=0, - completed=0, - ) - yield start_event.model_dump() - try: - for online_document_message in online_document_result: - end_time = time.time() - online_document_event = DatasourceCompletedEvent( - data=online_document_message.result, time_consuming=round(end_time - start_time, 2) - ) - yield online_document_event.model_dump() - except Exception as e: - logger.exception("Error during online document.") - yield DatasourceErrorEvent(error=str(e)).model_dump() - case DatasourceProviderType.ONLINE_DRIVE: - datasource_runtime = cast(OnlineDriveDatasourcePlugin, datasource_runtime) - online_drive_result: Generator[OnlineDriveBrowseFilesResponse, None, None] = ( - datasource_runtime.online_drive_browse_files( - user_id=account.id, - request=OnlineDriveBrowseFilesRequest( - bucket=user_inputs.get("bucket"), - prefix=user_inputs.get("prefix", ""), - max_keys=user_inputs.get("max_keys", 20), - next_page_parameters=user_inputs.get("next_page_parameters"), - ), - provider_type=datasource_runtime.datasource_provider_type(), - ) - ) - start_time = time.time() - start_event = DatasourceProcessingEvent( - total=0, - completed=0, - ) - yield start_event.model_dump() - for online_drive_message in online_drive_result: - end_time = time.time() - online_drive_event = DatasourceCompletedEvent( - data=online_drive_message.result, - time_consuming=round(end_time - start_time, 2), - total=None, - completed=None, - ) - yield online_drive_event.model_dump() - case DatasourceProviderType.WEBSITE_CRAWL: - datasource_runtime = cast(WebsiteCrawlDatasourcePlugin, datasource_runtime) - website_crawl_result: Generator[WebsiteCrawlMessage, None, None] = ( - datasource_runtime.get_website_crawl( - user_id=account.id, - datasource_parameters=variables_map, - provider_type=datasource_runtime.datasource_provider_type(), - ) - ) - start_time = time.time() - try: - for website_crawl_message in website_crawl_result: - end_time = time.time() - crawl_event: DatasourceCompletedEvent | DatasourceProcessingEvent - if website_crawl_message.result.status == "completed": - crawl_event = DatasourceCompletedEvent( - data=website_crawl_message.result.web_info_list or [], - total=website_crawl_message.result.total, - completed=website_crawl_message.result.completed, - time_consuming=round(end_time - start_time, 2), - ) - else: - crawl_event = DatasourceProcessingEvent( - total=website_crawl_message.result.total, - completed=website_crawl_message.result.completed, - ) - yield crawl_event.model_dump() - except Exception as e: - logger.exception("Error during website crawl.") - yield DatasourceErrorEvent(error=str(e)).model_dump() - case _: - raise ValueError(f"Unsupported datasource provider: {datasource_runtime.datasource_provider_type}") - except Exception as e: - logger.exception("Error in run_datasource_workflow_node.") - yield DatasourceErrorEvent(error=str(e)).model_dump() - - def run_datasource_node_preview( - self, - pipeline: Pipeline, - node_id: str, - user_inputs: dict[str, Any], - account: Account, - datasource_type: str, - is_published: bool, - credential_id: str | None = None, - *, - datasource_providers: DatasourceProviderService, - ) -> Mapping[str, Any]: - """ - Run published workflow datasource - """ - try: - if is_published: - # fetch published workflow by app_model - workflow = self.get_published_workflow(pipeline=pipeline) - else: - workflow = self.get_draft_workflow(pipeline=pipeline) - if not workflow: - raise ValueError("Workflow not initialized") - - # run draft workflow node - datasource_node_data = None - datasource_nodes = workflow.graph_dict.get("nodes", []) - for datasource_node in datasource_nodes: - if datasource_node.get("id") == node_id: - datasource_node_data = datasource_node.get("data", {}) - break - if not datasource_node_data: - raise ValueError("Datasource node data not found") - - datasource_parameters = datasource_node_data.get("datasource_parameters", {}) - for key, value in datasource_parameters.items(): - if not user_inputs.get(key): - user_inputs[key] = value["value"] - - datasource_runtime = DatasourceManager.get_datasource_runtime( - provider_id=f"{datasource_node_data.get('plugin_id')}/{datasource_node_data.get('provider_name')}", - datasource_name=datasource_node_data.get("datasource_name"), - tenant_id=pipeline.tenant_id, - datasource_type=DatasourceProviderType(datasource_type), - ) - credentials = datasource_providers.get_datasource_credentials( - tenant_id=pipeline.tenant_id, - provider=datasource_node_data.get("provider_name"), - plugin_id=datasource_node_data.get("plugin_id"), - credential_id=credential_id, - ) - if credentials: - datasource_runtime.runtime.credentials = credentials - match datasource_type: - case DatasourceProviderType.ONLINE_DOCUMENT: - datasource_runtime = cast(OnlineDocumentDatasourcePlugin, datasource_runtime) - online_document_result: Generator[DatasourceMessage, None, None] = ( - datasource_runtime.get_online_document_page_content( - user_id=account.id, - datasource_parameters=GetOnlineDocumentPageContentRequest( - workspace_id=user_inputs.get("workspace_id", ""), - page_id=user_inputs.get("page_id", ""), - type=user_inputs.get("type", ""), - ), - provider_type=datasource_type, - ) - ) - try: - variables: dict[str, Any] = {} - for online_document_message in online_document_result: - if online_document_message.type == DatasourceMessage.MessageType.VARIABLE: - assert isinstance(online_document_message.message, DatasourceMessage.VariableMessage) - variable_name = online_document_message.message.variable_name - variable_value = online_document_message.message.variable_value - if online_document_message.message.stream: - if not isinstance(variable_value, str): - raise ValueError("When 'stream' is True, 'variable_value' must be a string.") - if variable_name not in variables: - variables[variable_name] = "" - variables[variable_name] += variable_value - else: - variables[variable_name] = variable_value - return variables - except Exception as e: - logger.exception("Error during get online document content.") - raise RuntimeError(str(e)) - # TODO Online Drive - case _: - raise ValueError(f"Unsupported datasource provider: {datasource_runtime.datasource_provider_type}") - except Exception as e: - logger.exception("Error in run_datasource_node_preview.") - raise RuntimeError(str(e)) - - def run_free_workflow_node( - self, node_data: dict, tenant_id: str, user_id: str, node_id: str, user_inputs: dict[str, Any] - ) -> WorkflowNodeExecution: - """ - Run draft workflow node - """ - # run draft workflow node - start_at = time.perf_counter() - - workflow_node_execution = self._handle_node_run_result( - getter=lambda: WorkflowEntry.run_free_node( - node_id=node_id, - node_data=node_data, - tenant_id=tenant_id, - user_id=user_id, - user_inputs=user_inputs, - ), - start_at=start_at, - tenant_id=tenant_id, - node_id=node_id, - ) - - return workflow_node_execution - - def _handle_node_run_result( - self, - getter: Callable[ - [], - tuple[Node, Generator[GraphNodeEventBase | ContainerAwaitRequest, None, None]], - ], - start_at: float, - tenant_id: str, - node_id: str, - ) -> WorkflowNodeExecution: - """ - Handle node run result - - :param getter: Callable[[], tuple[BaseNode, Generator[RunEvent | InNodeEvent, None, None]]] - :param start_at: float - :param tenant_id: str - :param node_id: str - """ - try: - node_instance, generator = getter() - - node_run_result: NodeRunResult | None = None - for event in generator: - if isinstance(event, (NodeRunSucceededEvent, NodeRunFailedEvent)): - node_run_result = event.node_run_result - if node_run_result: - # sign output files - node_run_result.outputs = WorkflowEntry.handle_special_values(node_run_result.outputs) or {} - break - - if not node_run_result: - raise ValueError("Node run failed with no run result") - # single step debug mode error handling return - if node_run_result.status == WorkflowNodeExecutionStatus.FAILED and node_instance.error_strategy: - node_error_args: dict[str, Any] = { - "status": WorkflowNodeExecutionStatus.EXCEPTION, - "error": node_run_result.error, - "inputs": node_run_result.inputs, - "metadata": {"error_strategy": node_instance.error_strategy}, - } - if node_instance.error_strategy is ErrorStrategy.DEFAULT_VALUE: - node_run_result = NodeRunResult( - **node_error_args, - outputs={ - **node_instance.default_value_dict, - "error_message": node_run_result.error, - "error_type": node_run_result.error_type, - }, - ) - else: - node_run_result = NodeRunResult( - **node_error_args, - outputs={ - "error_message": node_run_result.error, - "error_type": node_run_result.error_type, - }, - ) - run_succeeded = node_run_result.status in ( - WorkflowNodeExecutionStatus.SUCCEEDED, - WorkflowNodeExecutionStatus.EXCEPTION, - ) - error = node_run_result.error if not run_succeeded else None - except WorkflowNodeRunFailedError as e: - node_instance = e._node # type: ignore - run_succeeded = False - node_run_result = None - error = e._error # type: ignore - - workflow_node_execution = WorkflowNodeExecution( - id=str(uuid4()), - workflow_id=node_instance.workflow_id, - index=1, - node_id=node_id, - node_type=node_instance.node_type, - title=node_instance.title, - elapsed_time=time.perf_counter() - start_at, - finished_at=datetime.now(UTC).replace(tzinfo=None), - created_at=datetime.now(UTC).replace(tzinfo=None), - ) - if run_succeeded and node_run_result: - # create workflow node execution - inputs = WorkflowEntry.handle_special_values(node_run_result.inputs) if node_run_result.inputs else None - process_data = ( - WorkflowEntry.handle_special_values(node_run_result.process_data) - if node_run_result.process_data - else None - ) - outputs = WorkflowEntry.handle_special_values(node_run_result.outputs) if node_run_result.outputs else None - - workflow_node_execution.inputs = inputs - workflow_node_execution.process_data = process_data - workflow_node_execution.outputs = outputs - workflow_node_execution.metadata = node_run_result.metadata - if node_run_result.status == WorkflowNodeExecutionStatus.SUCCEEDED: - workflow_node_execution.status = WorkflowNodeExecutionStatus.SUCCEEDED - elif node_run_result.status == WorkflowNodeExecutionStatus.EXCEPTION: - workflow_node_execution.status = WorkflowNodeExecutionStatus.EXCEPTION - workflow_node_execution.error = node_run_result.error - else: - # create workflow node execution - workflow_node_execution.status = WorkflowNodeExecutionStatus.FAILED - workflow_node_execution.error = error - # update document status - variable_pool = node_instance.graph_runtime_state.variable_pool - invoke_from = get_system_segment(variable_pool, SystemVariableKey.INVOKE_FROM) - if invoke_from: - if invoke_from.value == InvokeFrom.PUBLISHED_PIPELINE: - document_id = get_system_segment(variable_pool, SystemVariableKey.DOCUMENT_ID) - dataset_id = get_system_segment(variable_pool, SystemVariableKey.DATASET_ID) - pipeline_id = get_system_segment(variable_pool, SystemVariableKey.APP_ID) - if document_id and dataset_id and pipeline_id: - dataset = self._session.scalar( - select(Dataset) - .where( - Dataset.id == dataset_id.value, - Dataset.tenant_id == tenant_id, - Dataset.pipeline_id == pipeline_id.value, - ) - .limit(1) - ) - if dataset: - dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id) - document_ref = dataset_ref.document(document_id.value) - # dataset_service imports RagPipelineService; importing here avoids the cycle. - from services.knowledge.dataset_service import DocumentService - - document = next( - iter( - DocumentService.get_documents_by_ids( - document_ref.dataset, [document_ref.document_id], self._session - ) - ), - None, - ) - if document: - document.indexing_status = IndexingStatus.ERROR - document.error = error - self._session.add(document) - self._session.commit() - - return workflow_node_execution - def update_workflow( self, *, @@ -1096,7 +328,7 @@ def update_workflow( account_id: str, data: dict[str, Any], workflow_ref: WorkflowRef, - ) -> Workflow | None: + ) -> WorkflowRecord | None: """ Update workflow attributes @@ -1106,26 +338,10 @@ def update_workflow( :param workflow_ref: Owner-bound workflow reference :return: Updated workflow or None if not found """ - stmt = select(Workflow).where( - Workflow.id == workflow_ref.workflow_id, - Workflow.tenant_id == workflow_ref.tenant_id, - Workflow.app_id == workflow_ref.owner_id, + workflow = WorkflowDefinitionStore.update_workflow( + session=session, account_id=account_id, data=data, workflow_ref=workflow_ref ) - workflow = session.scalar(stmt) - - if not workflow: - return None - - allowed_fields = ["marked_name", "marked_comment"] - - for field, value in data.items(): - if field in allowed_fields: - setattr(workflow, field, value) - - workflow.updated_by = account_id - workflow.updated_at = datetime.now(UTC).replace(tzinfo=None) - - return workflow + return workflow_record(workflow, session) if workflow is not None else None def get_first_step_parameters(self, pipeline: Pipeline, node_id: str, is_draft: bool = False) -> list[dict]: """ @@ -1258,10 +474,7 @@ def get_rag_pipeline_workflow_run(self, pipeline: Pipeline, run_id: str) -> Work ) def get_rag_pipeline_workflow_run_node_executions( - self, - pipeline: Pipeline, - run_id: str, - user: Account | EndUser, + self, pipeline: Pipeline, run_id: str, user: Account | EndUser, *, tool_providers: ToolProviders ) -> list[WorkflowNodeExecutionTrace]: """ Get workflow run node execution list @@ -1280,7 +493,7 @@ def get_rag_pipeline_workflow_run_node_executions( workflow_run_id=run_id, ) return assemble_workflow_node_execution_traces( - node_executions, self._node_execution_service_repo, session=self._session + node_executions, self._node_execution_service_repo, session=self._session, tool_providers=tool_providers ) @staticmethod @@ -1349,18 +562,6 @@ def publish_customized_pipeline_template( session.add(pipeline_customized_template) session.commit() - def is_workflow_exist(self, pipeline: Pipeline) -> bool: - return ( - self._session.scalar( - select(func.count(Workflow.id)).where( - Workflow.tenant_id == pipeline.tenant_id, - Workflow.app_id == pipeline.id, - Workflow.version == Workflow.VERSION_DRAFT, - ) - ) - or 0 - ) > 0 - def get_node_last_run( self, pipeline: Pipeline, workflow: Workflow, node_id: str ) -> WorkflowNodeExecutionModel | None: @@ -1372,102 +573,15 @@ def get_node_last_run( ) return node_exec - def set_datasource_variables(self, pipeline: Pipeline, args: dict[str, Any], current_user: Account): - """ - Set datasource variables - """ - - # fetch draft workflow by app_model - draft_workflow = self.get_draft_workflow(pipeline=pipeline) - if not draft_workflow: - raise ValueError("Workflow not initialized") - - # run draft workflow node - start_at = time.perf_counter() - node_id = args.get("start_node_id") - if not node_id: - raise ValueError("Node id is required") - node_config = draft_workflow.get_node_config_by_id(node_id) - - eclosing_node_type_and_id = draft_workflow.get_enclosing_node_type_and_id(node_config) - if eclosing_node_type_and_id: - _, enclosing_node_id = eclosing_node_type_and_id - else: - enclosing_node_id = None - - system_inputs = build_system_variables( - datasource_type=args.get("datasource_type", "online_document"), - datasource_info=args.get("datasource_info", {}), - ) - - workflow_node_execution = self._handle_node_run_result( - getter=lambda: WorkflowEntry.single_step_run( - workflow=draft_workflow, - node_id=node_id, - user_inputs={}, - user_id=current_user.id, - variable_pool=_build_seeded_variable_pool( - build_bootstrap_variables( - system_variables=system_inputs, - rag_pipeline_variables=(), - ) - ), - variable_loader=DraftVarLoader( - engine=db.engine, - app_id=pipeline.id, - tenant_id=pipeline.tenant_id, - user_id=current_user.id, - ), - ), - start_at=start_at, - tenant_id=pipeline.tenant_id, - node_id=node_id, - ) - workflow_node_execution.workflow_id = draft_workflow.id - - # Create repository and save the node execution - repository = SQLAlchemyWorkflowNodeExecutionRepository( - session_factory=db.engine, - tenant_id=pipeline.tenant_id, - user=current_user, - app_id=pipeline.id, - triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP, - ) - repository.save(workflow_node_execution) - - # Convert node_execution to WorkflowNodeExecution after save - workflow_node_execution_db_model = repository._to_db_model(workflow_node_execution) # type: ignore - - with sessionmaker(bind=db.engine).begin() as session: - draft_var_saver = DraftVariableSaver( - session=session, - tenant_id=pipeline.tenant_id, - app_id=pipeline.id, - node_id=workflow_node_execution_db_model.node_id, - node_type=workflow_node_execution_db_model.node_type, - enclosing_node_id=enclosing_node_id, - node_execution_id=workflow_node_execution.id, - user=current_user, - ) - draft_var_saver.save( - process_data=workflow_node_execution.process_data, - outputs=workflow_node_execution.outputs, - ) - enqueue_draft_node_execution_trace( - execution=workflow_node_execution_db_model, - outputs=workflow_node_execution.outputs, - workflow_execution_id=None, - user_id=current_user.id, - ) - return workflow_node_execution_db_model - def _fetch_recommended_plugin_manifests(self, plugin_ids: list[str]) -> list[Any]: if not dify_config.MARKETPLACE_ENABLED: logger.info("Marketplace disabled; recommended-plugins list empty") return [] return marketplace.batch_fetch_plugin_by_ids(plugin_ids) - def get_recommended_plugins(self, type: str, current_user: Account, current_tenant_id: str) -> dict[str, Any]: + def get_recommended_plugins( + self, type: str, current_user: Account, current_tenant_id: str, *, tool_providers: ToolProviders + ) -> dict[str, Any]: # Query active recommended plugins stmt = select(PipelineRecommendedPlugin).where(PipelineRecommendedPlugin.active == True) if type and type != "all": @@ -1486,8 +600,7 @@ def get_recommended_plugins(self, type: str, current_user: Account, current_tena # Batch fetch plugin manifests plugin_ids = [plugin.plugin_id for plugin in pipeline_recommended_plugins] providers = BuiltinToolManageService.list_builtin_tools( - user_id=current_user.id, - tenant_id=current_tenant_id, + user_id=current_user.id, tenant_id=current_tenant_id, tool_providers=tool_providers ) providers_map = {provider.plugin_id: provider.to_dict() for provider in providers} @@ -1529,7 +642,6 @@ def retry_error_document( if not workflow: raise ValueError("Workflow not found") generator.generate( - session=self._session, pipeline=pipeline, workflow=workflow, user=user, diff --git a/api/services/snippet_dsl_service.py b/api/services/snippet_dsl_service.py index a67d507d58fe42..d10f0094f5d610 100644 --- a/api/services/snippet_dsl_service.py +++ b/api/services/snippet_dsl_service.py @@ -15,12 +15,12 @@ from extensions.ext_redis import redis_client from graphon.enums import BuiltinNodeTypes from graphon.model_runtime.utils.encoders import jsonable_encoder +from machinery.context import RequestContext from models import Account from models.snippet import CustomizedSnippet, SnippetType from models.workflow import Workflow from services.agent.dsl_service import AgentDslService -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService -from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService +from services.agent.workflow_contracts import WorkflowAgentBindingStore from services.dsl_content import DSL_MAX_SIZE, dsl_content_size from services.dsl_version import check_version_compatibility from services.entities.dsl_entities import ( @@ -31,7 +31,10 @@ PendingImportOwner, ) from services.plugin.dependencies_analysis import DependenciesAnalysisService -from services.snippet_service import SNIPPET_FORBIDDEN_NODE_TYPES, SnippetService +from services.snippet_service import SnippetService +from services.workflow.contracts import DraftImportResult, DraftSyncCommand, WorkflowOwner, WorkflowSnapshot +from services.workflow.draft_service import WorkflowDraftService +from services.workflow.snippet_policy import SNIPPET_FORBIDDEN_NODE_TYPES logger = logging.getLogger(__name__) @@ -70,13 +73,18 @@ class CheckDependenciesPendingData(BaseModel): class SnippetDslService: - def __init__(self, session: Session): + def __init__( + self, + session: Session, + *, + snippets: SnippetService, + drafts: WorkflowDraftService[WorkflowAgentBindingStore], + ): self._session = session + self._snippets = snippets + self._drafts = drafts self._warnings: list[DslImportWarning] = [] - def _snippet_service(self) -> SnippetService: - return SnippetService(session=self._session) - def import_snippet( self, *, @@ -382,7 +390,7 @@ def check_dependencies(self, snippet: CustomizedSnippet) -> CheckDependenciesRes """ Check dependencies for a snippet """ - snippet_service = self._snippet_service() + snippet_service = self._snippets workflow = snippet_service.get_draft_workflow(snippet=snippet) if not workflow: return CheckDependenciesResult(leaked_dependencies=[]) @@ -447,7 +455,6 @@ def _create_or_update_snippet( self._session.flush() # Create or update draft workflow - retirement_candidates: set[str] = set() if workflow_data: graph = workflow_data.get("graph", {}) raw_agent_packages = data.get("agent_packages") or {} @@ -455,49 +462,32 @@ def _create_or_update_snippet( raise ValueError("agent_packages must be a mapping") graph_for_sync = AgentDslService.graph_without_package_bindings(graph) if raw_agent_packages else graph - snippet_service = self._snippet_service() - # Get existing workflow hash if exists - existing_workflow = snippet_service.get_draft_workflow(snippet=snippet) - unique_hash = existing_workflow.unique_hash if existing_workflow else None - - draft_workflow = snippet_service.sync_draft_workflow( - snippet=snippet, - graph=graph_for_sync, - unique_hash=unique_hash, - account=account, - input_fields=input_fields, - sync_agent_bindings=False, - ) - if raw_agent_packages: - _, warnings, retirement_candidates = AgentDslService(self._session).import_workflow_packages( - workflow=draft_workflow, - portable_graph=graph, - raw_packages=raw_agent_packages, - account=account, + def materialize(workflow: WorkflowSnapshot) -> DraftImportResult: + imported_graph, warnings, retired = AgentDslService(self._session).import_workflow_packages( + workflow=workflow, portable_graph=graph, raw_packages=raw_agent_packages, account=account ) self._warnings.extend(warnings) - WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync( - session=self._session, - draft_workflow=draft_workflow, - ) - else: - retirement_candidates = WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=self._session, - draft_workflow=draft_workflow, - account_id=account.id, - ) - WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync( - session=self._session, - draft_workflow=draft_workflow, - ) + return DraftImportResult(imported_graph, retired) + + self._drafts.sync( + RequestContext(str(uuid.uuid4()), None, account.id, snippet.tenant_id), + WorkflowOwner(snippet.id, "snippet"), + DraftSyncCommand( + graph=graph_for_sync, + features={}, + unique_hash=None, + check_hash=False, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=[], + conversation_variables=[], + input_fields=input_fields, + ), + materialize=materialize if raw_agent_packages else None, + ) self._session.commit() - if workflow_data: - WorkflowAgentRetirementService.retire_unowned( - tenant_id=snippet.tenant_id, - agent_ids=retirement_candidates, - account_id=account.id, - ) return snippet def export_snippet_dsl( @@ -510,7 +500,7 @@ def export_snippet_dsl( :param workflow_id: Optional published workflow version to export; defaults to the draft workflow :return: YAML string """ - snippet_service = self._snippet_service() + snippet_service = self._snippets workflow = ( snippet_service.get_published_workflow_by_id(snippet=snippet, workflow_id=workflow_id) if workflow_id diff --git a/api/services/snippet_generate_service.py b/api/services/snippet_generate_service.py index 6fd70de233dd96..c445cf575e97f8 100644 --- a/api/services/snippet_generate_service.py +++ b/api/services/snippet_generate_service.py @@ -21,12 +21,11 @@ import json import logging from collections.abc import Generator, Mapping, Sequence -from typing import Any, Union, cast +from typing import Any, Protocol, Union, cast -from sqlalchemy.orm import Session, make_transient, sessionmaker +from sqlalchemy.orm import make_transient from core.app.app_config.features.file_upload.manager import FileUploadConfigManager -from core.app.apps.workflow.app_generator import WorkflowAppGenerator from core.app.entities.app_invoke_entities import InvokeFrom from core.app.file_access import DatabaseFileAccessController from core.workflow.snippet_start import SNIPPET_VIRTUAL_START_NODE_ID @@ -36,13 +35,20 @@ from models.model import App, AppMode, EndUser from models.snippet import CustomizedSnippet from models.workflow import Workflow, WorkflowNodeExecutionModel -from services.snippet_service import SnippetService +from services.app.generation.response import convert_to_event_stream +from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables from services.workflow_service import WorkflowService logger = logging.getLogger(__name__) _file_access_controller = DatabaseFileAccessController() +class SnippetWorkflowReader(Protocol): + def get_draft_workflow(self, snippet: CustomizedSnippet) -> Workflow | None: ... + + class _SnippetAsApp: """ Minimal adapter that wraps a CustomizedSnippet to satisfy the App-like @@ -83,6 +89,19 @@ class SnippetGenerateService: # Specific ID for the injected virtual Start node so it can be recognised _VIRTUAL_START_NODE_ID = SNIPPET_VIRTUAL_START_NODE_ID + def __init__( + self, + *, + snippets: SnippetWorkflowReader, + variables: WorkflowExecutionVariables, + runtime: WorkflowRuntime, + workflows: WorkflowService, + ) -> None: + self._snippets = snippets + self._variables = variables + self._runtime = runtime + self._workflows = workflows + @classmethod def _is_virtual_start_event(cls, message: Mapping[str, Any] | str) -> bool: """ @@ -127,15 +146,13 @@ def _stream() -> Generator[Mapping[str, Any] | str, None, None]: return _stream() - @classmethod def generate( - cls, + self, snippet: CustomizedSnippet, user: Union[Account, EndUser], args: Mapping[str, Any], invoke_from: InvokeFrom, streaming: bool = True, - session_maker: sessionmaker[Session] | None = None, ) -> Mapping[str, Any] | Generator[str, None, None]: """ Run a snippet's draft workflow. @@ -156,18 +173,21 @@ def generate( :return: Blocking response mapping or SSE streaming generator :raises ValueError: If the snippet has no draft workflow """ - snippet_service = SnippetService(session_maker) - workflow = snippet_service.get_draft_workflow(snippet=snippet) + workflow = self._snippets.get_draft_workflow(snippet=snippet) if not workflow: raise ValueError("Workflow not initialized") # Inject a virtual Start node when the graph doesn't have one. - workflow = cls._ensure_start_node(workflow, snippet) + workflow = self._ensure_start_node(workflow, snippet) # Adapt snippet to App-like interface for WorkflowAppGenerator app_proxy = cast(App, _SnippetAsApp(snippet)) - response = WorkflowAppGenerator().generate( + response = WorkflowAppGenerator( + runtime=self._runtime, + draft_variable_loader=self._variables.workflow_loader, + draft_variable_saver=self._variables.saver_factory, + ).generate( app_model=app_proxy, workflow=workflow, user=user, @@ -177,56 +197,7 @@ def generate( call_depth=0, ) - return WorkflowAppGenerator.convert_to_event_stream(cls._filter_virtual_start_events(response)) - - @classmethod - def run_published( - cls, - snippet: CustomizedSnippet, - user: Union[Account, EndUser], - args: Mapping[str, Any], - invoke_from: InvokeFrom, - session_maker: sessionmaker[Session] | None = None, - ) -> Mapping[str, Any]: - """ - Run a snippet's published workflow in non-streaming (blocking) mode. - - Similar to :meth:`generate` but targets the published workflow instead - of the draft, and returns the raw blocking response without SSE - wrapping. Designed for programmatic callers that need direct workflow outputs. - - :param snippet: CustomizedSnippet instance (must be published) - :param user: Account or EndUser initiating the run - :param args: Workflow inputs (must include "inputs" key) - :param invoke_from: Source of invocation - :return: Blocking response mapping with workflow outputs - :raises ValueError: If the snippet has no published workflow - """ - snippet_service = SnippetService(session_maker) - workflow = snippet_service.get_published_workflow(snippet) - if not workflow: - raise ValueError("No published workflow found for snippet") - - # Inject a virtual Start node when the graph doesn't have one. - workflow = cls._ensure_start_node(workflow, snippet) - - app_proxy = cast(App, _SnippetAsApp(snippet)) - - response = WorkflowAppGenerator().generate( - app_model=app_proxy, - workflow=workflow, - user=user, - args=args, - invoke_from=invoke_from, - streaming=False, - call_depth=0, - ) - return response - - @classmethod - def ensure_start_node_for_worker(cls, workflow: Workflow, snippet: CustomizedSnippet) -> Workflow: - """Public wrapper for worker-thread start-node injection.""" - return cls._ensure_start_node(workflow, snippet) + return convert_to_event_stream(self._filter_virtual_start_events(response)) @classmethod def _ensure_start_node(cls, workflow: Workflow, snippet: CustomizedSnippet) -> Workflow: @@ -324,16 +295,14 @@ def _inject_virtual_start_node( "edges": [*edges, *new_edges], } - @classmethod def run_draft_node( - cls, + self, snippet: CustomizedSnippet, node_id: str, user_inputs: Mapping[str, Any], account: Account, query: str = "", files: Sequence[File] | None = None, - session_maker: sessionmaker[Session] | None = None, ) -> WorkflowNodeExecutionModel: """ Run a single node in a snippet's draft workflow (single-step debugging). @@ -350,15 +319,14 @@ def run_draft_node( :return: WorkflowNodeExecutionModel with execution results :raises ValueError: If the snippet has no draft workflow """ - snippet_service = SnippetService(session_maker) - draft_workflow = snippet_service.get_draft_workflow(snippet=snippet) + draft_workflow = self._snippets.get_draft_workflow(snippet=snippet) if not draft_workflow: raise ValueError("Workflow not initialized") app_proxy = cast(App, _SnippetAsApp(snippet)) - workflow_service = WorkflowService() - return workflow_service.run_draft_workflow_node( + return self._workflows.run_draft_workflow_node( + variables=self._variables, app_model=app_proxy, draft_workflow=draft_workflow, node_id=node_id, @@ -368,16 +336,13 @@ def run_draft_node( files=files, ) - @classmethod def generate_single_iteration( - cls, + self, snippet: CustomizedSnippet, user: Union[Account, EndUser], node_id: str, args: Mapping[str, Any], streaming: bool = True, - *, - session_maker: sessionmaker[Session], ) -> Mapping[str, Any] | Generator[str, None, None]: """ Run a single iteration node in a snippet's draft workflow. @@ -391,40 +356,37 @@ def generate_single_iteration( :param node_id: ID of the iteration node to run :param args: Dict containing 'inputs' key with iteration input data :param streaming: Whether to stream the response (should be True) - :param session_maker: Factory for the synchronous database work before worker startup :return: SSE streaming generator :raises ValueError: If the snippet has no draft workflow """ - snippet_service = SnippetService(session_maker) - workflow = snippet_service.get_draft_workflow(snippet=snippet) + workflow = self._snippets.get_draft_workflow(snippet=snippet) if not workflow: raise ValueError("Workflow not initialized") app_proxy = cast(App, _SnippetAsApp(snippet)) - with session_maker() as session: - return WorkflowAppGenerator.convert_to_event_stream( - WorkflowAppGenerator().single_iteration_generate( - app_model=app_proxy, - workflow=workflow, - node_id=node_id, - user=user, - args=args, - streaming=streaming, - session=session, - ) + return convert_to_event_stream( + WorkflowAppGenerator( + runtime=self._runtime, + draft_variable_loader=self._variables.workflow_loader, + draft_variable_saver=self._variables.saver_factory, + ).single_iteration_generate( + app_model=app_proxy, + workflow=workflow, + node_id=node_id, + user=user, + args=args, + streaming=streaming, ) + ) - @classmethod def generate_single_loop( - cls, + self, snippet: CustomizedSnippet, user: Union[Account, EndUser], node_id: str, - args: Any, + args: Mapping[str, Any], streaming: bool = True, - *, - session_maker: sessionmaker[Session], ) -> Mapping[str, Any] | Generator[str, None, None]: """ Run a single loop node in a snippet's draft workflow. @@ -436,31 +398,31 @@ def generate_single_loop( :param snippet: CustomizedSnippet instance :param user: Account or EndUser initiating the run :param node_id: ID of the loop node to run - :param args: Pydantic model with 'inputs' attribute containing loop input data + :param args: Mapping with 'inputs' containing loop input data :param streaming: Whether to stream the response (should be True) - :param session_maker: Factory for the synchronous database work before worker startup :return: SSE streaming generator :raises ValueError: If the snippet has no draft workflow """ - snippet_service = SnippetService(session_maker) - workflow = snippet_service.get_draft_workflow(snippet=snippet) + workflow = self._snippets.get_draft_workflow(snippet=snippet) if not workflow: raise ValueError("Workflow not initialized") app_proxy = cast(App, _SnippetAsApp(snippet)) - with session_maker() as session: - return WorkflowAppGenerator.convert_to_event_stream( - WorkflowAppGenerator().single_loop_generate( - app_model=app_proxy, - workflow=workflow, - node_id=node_id, - user=user, - args=args, # type: ignore[arg-type] - streaming=streaming, - session=session, - ) + return convert_to_event_stream( + WorkflowAppGenerator( + runtime=self._runtime, + draft_variable_loader=self._variables.workflow_loader, + draft_variable_saver=self._variables.saver_factory, + ).single_loop_generate( + app_model=app_proxy, + workflow=workflow, + node_id=node_id, + user=user, + args=args, + streaming=streaming, ) + ) @staticmethod def parse_files(workflow: Workflow, files: list[dict] | None = None) -> Sequence[File]: diff --git a/api/services/snippet_service.py b/api/services/snippet_service.py index 8268db205eafa7..2992eed6b9362f 100644 --- a/api/services/snippet_service.py +++ b/api/services/snippet_service.py @@ -1,7 +1,8 @@ import json import logging -from collections.abc import Generator, Mapping, Sequence +from collections.abc import Callable, Generator, Mapping, Sequence from contextlib import contextmanager +from dataclasses import replace from datetime import UTC, datetime from typing import Any @@ -9,19 +10,12 @@ from sqlalchemy.orm import Session, sessionmaker from configs import dify_config -from core.workflow.node_factory import LATEST_VERSION, NODE_TYPE_CLASSES_MAPPING from enums import DeploymentEdition -from graphon.enums import BuiltinNodeTypes, NodeType +from enums.agent import WorkflowAgentBindingType +from graphon.enums import NodeType from libs.infinite_scroll_pagination import InfiniteScrollPagination from models import Account, TagBinding -from models.agent import ( - WORKFLOW_ONLY_AGENT_SOURCES, - Agent, - AgentScope, - AgentStatus, - WorkflowAgentBindingType, - WorkflowAgentNodeBinding, -) +from models.agent import WORKFLOW_ONLY_AGENT_SOURCES, Agent, AgentScope, AgentStatus, WorkflowAgentNodeBinding from models.enums import WorkflowRunTriggeredFrom from models.model import UploadFile from models.snippet import CustomizedSnippet, SnippetType @@ -35,30 +29,29 @@ WorkflowKind, WorkflowNodeExecutionModel, WorkflowRun, - WorkflowType, ) +from repositories.agent.retirement_repository import WorkflowAgentRetirementRepository +from repositories.agent.workflow_binding_repository import workflow_binding_scope from repositories.factory import DifyAPIRepositoryFactory -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService -from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError -from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError +from repositories.workflow.definition_repository import WorkflowDefinitionStore, workflow_from_snapshot, workflow_record +from repositories.workflow.snippet_publication_repository import SnippetPublicationRepository +from services.agent.retirement_service import WorkflowAgentRetirementService +from services.agent.workflow_contracts import AgentSkillReader +from services.agent.workflow_publish_service import WorkflowAgentPublishService +from services.errors.app import IsDraftWorkflowError from services.tag_application_service import TagTargetQuery +from services.tools.provider_queries import ToolProviders +from services.workflow.contracts import WorkflowRecord +from services.workflow.execution.adapters.node_factory import LATEST_VERSION, NODE_TYPE_CLASSES_MAPPING +from services.workflow.snippet_policy import validate_snippet_graph_forbidden_nodes from services.workflow_node_execution_trace_service import ( WorkflowNodeExecutionTrace, assemble_workflow_node_execution_traces, ) -from services.workflow_restore import apply_published_workflow_snapshot_to_draft +from services.workflow_ref_service import WorkflowRef logger = logging.getLogger(__name__) -# Node types not allowed in snippet workflows (sync, publish, DSL import). -SNIPPET_FORBIDDEN_NODE_TYPES: frozenset[str] = frozenset( - { - BuiltinNodeTypes.START, - BuiltinNodeTypes.HUMAN_INPUT, - BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL, - } -) - class SnippetService: """Service for managing customized snippets.""" @@ -67,6 +60,9 @@ def __init__( self, session_maker: sessionmaker[Session] | Session | None = None, session: Session | None = None, + *, + agent_bindings: Callable[[Session], WorkflowAgentPublishService], + skills: AgentSkillReader, ): """Initialize SnippetService with repository dependencies.""" if isinstance(session_maker, Session): @@ -77,7 +73,10 @@ def __init__( if session_maker is None: raise ValueError("SnippetService requires a session or session_maker.") self._session = session + self._agent_bindings = agent_bindings self._session_maker = session_maker + self._publication = SnippetPublicationRepository(session_maker) + self._skills = skills self._node_execution_service_repo = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( session_maker ) @@ -177,30 +176,6 @@ def _delete_archived_workflow_run_files(*, snippet: CustomizedSnippet) -> None: except Exception: logger.exception("Failed to delete snippet archive file %s", key) - @staticmethod - def validate_snippet_graph_forbidden_nodes(graph: Mapping[str, Any]) -> None: - """Reject graphs that contain node types not allowed in snippets.""" - nodes = graph.get("nodes") or [] - disallowed: list[tuple[str, str]] = [] - for node in nodes: - if not isinstance(node, dict): - continue - node_data = node.get("data") or {} - node_type = node_data.get("type") - if not isinstance(node_type, str): - continue - if node_type in SNIPPET_FORBIDDEN_NODE_TYPES: - node_id = node.get("id") - disallowed.append((str(node_id) if node_id is not None else "?", node_type)) - if not disallowed: - return - detail = ", ".join(f"{nid}:{t}" for nid, t in disallowed) - raise ValueError( - f"Snippet workflow cannot contain start, human-input, or knowledge-retrieval nodes. Found: {detail}" - ) - - # --- CRUD Operations --- - def get_snippets( self, *, @@ -294,6 +269,7 @@ def create_snippet( icon_info: dict | None, input_fields: list[dict] | None, account: Account, + graph: dict[str, Any] | None = None, ) -> CustomizedSnippet: """ Create a new snippet. @@ -307,6 +283,8 @@ def create_snippet( :param account: Creator account :return: Created CustomizedSnippet """ + if graph is not None: + validate_snippet_graph_forbidden_nodes(graph) snippet = CustomizedSnippet( tenant_id=tenant_id, name=name, @@ -400,7 +378,9 @@ def delete_snippet( tenant_id = snippet.tenant_id def collect_agent_resources(_session: Session) -> None: - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService( + WorkflowAgentRetirementRepository(sessionmaker(bind=session.get_bind(), expire_on_commit=False)) + ).retire_unowned( tenant_id=tenant_id, agent_ids=candidate_agent_ids, account_id=account_id, @@ -492,13 +472,20 @@ def get_draft_workflow(self, snippet: CustomizedSnippet) -> Workflow | None: :return: Draft Workflow or None """ with self._session_scope() as session: - stmt = select(Workflow).where( - Workflow.tenant_id == snippet.tenant_id, - Workflow.app_id == snippet.id, - self._snippet_kind_filter(), - Workflow.version == "draft", + return WorkflowDefinitionStore.get_draft_workflow(snippet, session=session, kind=WorkflowKind.SNIPPET.value) + + def get_draft_workflow_record(self, snippet: CustomizedSnippet) -> WorkflowRecord | None: + """Materialize editor data, including Agent bindings, before serialization.""" + with self._session_scope() as session: + workflow = WorkflowDefinitionStore.get_draft_workflow( + snippet, session=session, kind=WorkflowKind.SNIPPET.value ) - return session.scalar(stmt) + if workflow is None: + return None + graph = self._agent_bindings(session).project_draft_bindings_to_graph( + draft_workflow=workflow_binding_scope(workflow) + ) + return replace(workflow_record(workflow, session), graph=graph, conversation_variables=[]) def get_published_workflow(self, snippet: CustomizedSnippet) -> Workflow | None: """ @@ -511,13 +498,16 @@ def get_published_workflow(self, snippet: CustomizedSnippet) -> Workflow | None: return None with self._session_scope() as session: - stmt = select(Workflow).where( - Workflow.tenant_id == snippet.tenant_id, - Workflow.app_id == snippet.id, - self._snippet_kind_filter(), - Workflow.id == snippet.workflow_id, + return WorkflowDefinitionStore.get_published_workflow( + snippet, session=session, kind=WorkflowKind.SNIPPET.value ) - return session.scalar(stmt) + + def get_published_workflow_record(self, snippet: CustomizedSnippet) -> WorkflowRecord | None: + with self._session_scope() as session: + workflow = WorkflowDefinitionStore.get_published_workflow( + snippet, session=session, kind=WorkflowKind.SNIPPET.value + ) + return workflow_record(workflow, session) if workflow is not None else None def get_published_workflow_by_id(self, snippet: CustomizedSnippet, workflow_id: str) -> Workflow | None: """ @@ -542,217 +532,28 @@ def get_published_workflow_by_id(self, snippet: CustomizedSnippet, workflow_id: raise IsDraftWorkflowError("source workflow must be published") return workflow - def sync_draft_workflow( - self, - *, - snippet: CustomizedSnippet, - graph: dict, - unique_hash: str | None, - account: Account, - input_fields: list[dict] | None = None, - sync_agent_bindings: bool = True, - ) -> Workflow: - """ - Sync draft workflow for snippet. - - Snippet workflows do not persist environment variables (always empty) or - conversation variables (always empty). - - :param snippet: CustomizedSnippet instance - :param graph: Workflow graph configuration - :param unique_hash: Hash for conflict detection - :param account: Account making the change - :param input_fields: Input fields for snippet - :return: Synced Workflow - :raises WorkflowHashNotEqualError: If hash mismatch - """ - SnippetService.validate_snippet_graph_forbidden_nodes(graph) - - workflow = self.get_draft_workflow(snippet=snippet) - - if workflow and workflow.unique_hash != unique_hash: - raise WorkflowHashNotEqualError() - - # Create draft workflow if not found - if not workflow: - workflow = Workflow( - tenant_id=snippet.tenant_id, - app_id=snippet.id, - features="{}", - type=WorkflowType.WORKFLOW, - kind=WorkflowKind.SNIPPET, - version="draft", - graph=json.dumps(graph), - created_by=account.id, - environment_variables=[], - conversation_variables=[], - ) - else: - # Update existing draft workflow - workflow.graph = json.dumps(graph) - workflow.type = WorkflowType.WORKFLOW - workflow.kind = WorkflowKind.SNIPPET - workflow.updated_by = account.id - workflow.updated_at = datetime.now(UTC).replace(tzinfo=None) - workflow.environment_variables = [] - workflow.conversation_variables = [] - - # Update snippet's input_fields if provided - if input_fields is not None: - snippet.input_fields = json.dumps(input_fields) - snippet.updated_by = account.id - snippet.updated_at = datetime.now(UTC).replace(tzinfo=None) - - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - retirement_candidates: set[str] = set() - with self._session_scope() as session: - session.add(workflow) - session.add(snippet) - if sync_agent_bindings: - session.flush() - retirement_candidates = WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, - account_id=account.id, - ) - WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync( - session=session, - draft_workflow=workflow, - ) - self._commit_if_owned(session) - if self._session is None: - WorkflowAgentRetirementService.retire_unowned( - tenant_id=snippet.tenant_id, - agent_ids=retirement_candidates, - account_id=account.id, - ) - return workflow - - def restore_published_workflow_to_draft( - self, - *, - snippet: CustomizedSnippet, - workflow_id: str, - account: Account, - ) -> Workflow: - """ - Restore a published snippet workflow snapshot into the draft workflow. - - :param snippet: CustomizedSnippet instance - :param workflow_id: Published workflow ID - :param account: Account making the change - :return: Restored draft Workflow - :raises WorkflowNotFoundError: If the source workflow does not exist - :raises IsDraftWorkflowError: If the source workflow is a draft - :raises ValueError: If the restored graph is invalid for snippets - """ - source_workflow = self.get_published_workflow_by_id(snippet=snippet, workflow_id=workflow_id) - if not source_workflow: - raise WorkflowNotFoundError("Workflow not found.") - - SnippetService.validate_snippet_graph_forbidden_nodes(source_workflow.graph_dict) - - draft_workflow = self.get_draft_workflow(snippet=snippet) - draft_workflow, _is_new_draft = apply_published_workflow_snapshot_to_draft( - tenant_id=snippet.tenant_id, - app_id=snippet.id, - source_workflow=source_workflow, - draft_workflow=draft_workflow, - account=account, - updated_at_factory=lambda: datetime.now(UTC).replace(tzinfo=None), - ) - - with self._session_scope() as session: - session.add(draft_workflow) - session.flush() - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - retirement_candidates = WorkflowAgentPublishService.restore_agent_node_bindings_to_draft( - session=session, - source_workflow=source_workflow, - draft_workflow=draft_workflow, - account_id=account.id, - ) - self._commit_if_owned(session) - if self._session is None: - WorkflowAgentRetirementService.retire_unowned( - tenant_id=snippet.tenant_id, - agent_ids=retirement_candidates, - account_id=account.id, - ) - return draft_workflow - - def publish_workflow( - self, - *, - session: Session, - snippet: CustomizedSnippet, - account: Account, - ) -> Workflow: - """ - Publish the draft workflow as a new version. - - :param session: Database session - :param snippet: CustomizedSnippet instance - :param account: Account making the change - :return: Published Workflow - :raises ValueError: If no draft workflow exists - """ - draft_workflow_stmt = select(Workflow).where( - Workflow.tenant_id == snippet.tenant_id, - Workflow.app_id == snippet.id, - self._snippet_kind_filter(), - Workflow.version == "draft", - ) - draft_workflow = session.scalar(draft_workflow_stmt) - if not draft_workflow: - raise ValueError("No valid workflow found.") - - SnippetService.validate_snippet_graph_forbidden_nodes(draft_workflow.graph_dict) + def publish_workflow(self, *, snippet: CustomizedSnippet, account: Account) -> Workflow: + """Prepare external dependencies without a transaction, then recheck and publish.""" + with self._publication.draft(tenant_id=snippet.tenant_id, snippet_id=snippet.id) as (source, bindings): + agents = WorkflowAgentPublishService(repository=bindings).publication_state(draft_workflow=source) + draft = workflow_from_snapshot(source) + validate_snippet_graph_forbidden_nodes(draft.graph_dict) from core.workflow.llm_environment_variable import validate_llm_environment_model_references validate_llm_environment_model_references( - graph=draft_workflow.graph_dict, - environment_variables=draft_workflow.environment_variables, + graph=draft.graph_dict, + environment_variables=draft.environment_variables, ) - - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - WorkflowAgentPublishService.validate_agent_nodes_for_publish( - session=session, - draft_workflow=draft_workflow, - ) - - # Create new published workflow - workflow = Workflow.new( - tenant_id=snippet.tenant_id, - app_id=snippet.id, - type=WorkflowType.WORKFLOW.value, - version=str(datetime.now(UTC).replace(tzinfo=None)), - graph=draft_workflow.graph, - features=draft_workflow.features, - created_by=account.id, - environment_variables=[], - conversation_variables=[], - rag_pipeline_variables=draft_workflow.rag_pipeline_variables, - kind=WorkflowKind.SNIPPET.value, - ) - session.add(workflow) - WorkflowAgentPublishService.copy_agent_node_bindings_to_published( - session=session, - draft_workflow=draft_workflow, - published_workflow=workflow, - ) - - # Update snippet version - snippet.version += 1 - snippet.is_published = True - snippet.workflow_id = workflow.id - snippet.updated_by = account.id - session.add(snippet) - + WorkflowAgentPublishService.validate_publication_state(agents, skills=self._skills) + with self._publication.publication(source) as transaction: + agent_service = WorkflowAgentPublishService(repository=transaction.bindings) + agent_service.validate_prepared_publication(draft_workflow=transaction.draft, prepared=agents) + workflow = transaction.create_version(account_id=account.id) + agent_service.copy_agent_node_bindings_to_published( + draft_workflow=transaction.draft, published_workflow=workflow_binding_scope(workflow) + ) + transaction.activate(workflow, account_id=account.id) return workflow def get_all_published_workflows( @@ -762,7 +563,7 @@ def get_all_published_workflows( snippet: CustomizedSnippet, page: int, limit: int, - ) -> tuple[Sequence[Workflow], bool]: + ) -> tuple[Sequence[WorkflowRecord], bool]: """ Get all published workflow versions for snippet. @@ -772,27 +573,17 @@ def get_all_published_workflows( :param limit: Items per page :return: Tuple of (workflows list, has_more flag) """ - if not snippet.workflow_id: - return [], False - - stmt = ( - select(Workflow) - .where( - Workflow.app_id == snippet.id, - self._snippet_kind_filter(), - Workflow.version != "draft", - ) - .order_by(Workflow.version.desc()) - .limit(limit + 1) - .offset((page - 1) * limit) + workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( + session=session, + app_model=snippet, + page=page, + limit=limit, + user_id=None, + kind=WorkflowKind.SNIPPET, + include_draft=False, ) - workflows = list(session.scalars(stmt).all()) - has_more = len(workflows) > limit - if has_more: - workflows = workflows[:-1] - - return workflows, has_more + return [workflow_record(workflow, session) for workflow in workflows], has_more def update_workflow( self, @@ -802,7 +593,7 @@ def update_workflow( workflow_id: str, account: Account, data: dict[str, Any], - ) -> Workflow | None: + ) -> WorkflowRecord | None: """ Update a published snippet workflow version's display metadata. @@ -813,73 +604,15 @@ def update_workflow( :param data: Dictionary containing fields to update :return: Updated workflow or None if not found """ - stmt = select(Workflow).where( - Workflow.id == workflow_id, - Workflow.tenant_id == snippet.tenant_id, - Workflow.app_id == snippet.id, - self._snippet_kind_filter(), - Workflow.version != Workflow.VERSION_DRAFT, - ) - workflow = session.scalar(stmt) - if not workflow: - return None - - allowed_fields = {"marked_name", "marked_comment"} - for field, value in data.items(): - if field in allowed_fields: - setattr(workflow, field, value) - - workflow.updated_by = account.id - workflow.updated_at = datetime.now(UTC).replace(tzinfo=None) - session.add(workflow) - return workflow - - def delete_workflow( - self, - *, - session: Session, - snippet: CustomizedSnippet, - workflow_id: str, - ) -> bool: - """ - Delete a published snippet workflow version. - - :param session: Database session - :param snippet: CustomizedSnippet instance - :param workflow_id: Workflow ID - :return: True if successful - :raises: ValueError if workflow not found - :raises: WorkflowInUseError if workflow is the snippet's active version or published as a tool - :raises: DraftWorkflowDeletionError if workflow is a draft version - """ - stmt = select(Workflow).where( - Workflow.id == workflow_id, - Workflow.tenant_id == snippet.tenant_id, - Workflow.app_id == snippet.id, - self._snippet_kind_filter(), - ) - workflow = session.scalar(stmt) - if not workflow: - raise ValueError(f"Workflow with ID {workflow_id} not found") - - if workflow.version == Workflow.VERSION_DRAFT: - raise DraftWorkflowDeletionError("Cannot delete draft workflow versions") - - if snippet.workflow_id == workflow.id: - raise WorkflowInUseError(f"Cannot delete workflow that is currently in use by snippet '{snippet.id}'") - - tool_provider = session.scalar( - select(WorkflowToolProvider).where( - WorkflowToolProvider.tenant_id == snippet.tenant_id, - WorkflowToolProvider.app_id == snippet.id, - WorkflowToolProvider.version == workflow.version, - ) + workflow = WorkflowDefinitionStore.update_workflow( + session=session, + account_id=account.id, + data=data, + workflow_ref=WorkflowRef(tenant_id=snippet.tenant_id, owner_id=snippet.id, workflow_id=workflow_id), + kind=WorkflowKind.SNIPPET.value, + published_only=True, ) - if tool_provider: - raise WorkflowInUseError("Cannot delete workflow that is published as a tool") - - session.delete(workflow) - return True + return workflow_record(workflow, session) if workflow is not None else None # --- Default Block Configs --- @@ -968,10 +701,7 @@ def get_snippet_workflow_run( ) def get_snippet_workflow_run_node_executions( - self, - *, - snippet: CustomizedSnippet, - run_id: str, + self, *, snippet: CustomizedSnippet, run_id: str, tool_providers: ToolProviders ) -> list[WorkflowNodeExecutionTrace]: """ Get workflow run node execution list. @@ -992,7 +722,7 @@ def get_snippet_workflow_run_node_executions( with self._session_scope() as session: return assemble_workflow_node_execution_traces( - node_executions, self._node_execution_service_repo, session=session + node_executions, self._node_execution_service_repo, session=session, tool_providers=tool_providers ) # --- Node Execution Operations --- diff --git a/api/services/trial_app_generation_adapters.py b/api/services/trial_app_generation_adapters.py index c151f12a822ab8..bddb9ebead5890 100644 --- a/api/services/trial_app_generation_adapters.py +++ b/api/services/trial_app_generation_adapters.py @@ -13,17 +13,27 @@ from core.app.entities.app_invoke_entities import InvokeFrom from models import Account, App from services.account_errors import AccountNotFoundError +from services.app.generation.runtime import AppGenerationRuntime from services.app_definition_query_service import AppDefinitionUnavailableError from services.app_generate_service import AppGenerateService from services.trial_app_access_service import TrialAppRef from services.trial_app_generation_service import GenerationResponse, TrialAppGenerationRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables logger = logging.getLogger(__name__) class AppGenerateServiceRuntime(TrialAppGenerationRuntime): - def __init__(self, *, session_factory: sessionmaker[Session]) -> None: + def __init__( + self, + *, + session_factory: sessionmaker[Session], + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, + ) -> None: self._session_factory: sessionmaker[Session] = session_factory + self._variables = variables + self._runtime = runtime @override def generate( @@ -47,6 +57,8 @@ def generate( # receives its own session, never the admission or usage transaction. with self._session_factory(expire_on_commit=False) as session: generated_response: GenerationResponse = AppGenerateService.generate( + variables=self._variables, + runtime=self._runtime, app_model=app_model, user=account, args=args, diff --git a/api/services/trigger/errors.py b/api/services/trigger/errors.py new file mode 100644 index 00000000000000..6f56ebf08d28de --- /dev/null +++ b/api/services/trigger/errors.py @@ -0,0 +1,5 @@ +"""Framework-neutral trigger failures.""" + + +class WebhookBodyTooLargeError(Exception): + """The webhook payload exceeds the configured admission limit.""" diff --git a/api/services/trigger/schedule_service.py b/api/services/trigger/schedule_service.py index 31fd3f004f4297..cf73900efbe1c4 100644 --- a/api/services/trigger/schedule_service.py +++ b/api/services/trigger/schedule_service.py @@ -1,123 +1,23 @@ -import json import logging -from datetime import datetime from sqlalchemy import select from sqlalchemy.orm import Session -from core.trigger.constants import TRIGGER_SCHEDULE_NODE_TYPE from core.workflow.nodes.trigger_schedule.entities import ( ScheduleConfig, - SchedulePlanUpdate, TriggerScheduleNodeData, VisualConfig, ) -from core.workflow.nodes.trigger_schedule.exc import ScheduleConfigError, ScheduleNotFoundError +from core.workflow.nodes.trigger_schedule.exc import ScheduleConfigError from graphon.entities.graph_config import NodeConfigDict -from libs.schedule_utils import calculate_next_run_at, convert_12h_to_24h from models.account import Account, TenantAccountJoin -from models.trigger import WorkflowSchedulePlan -from models.workflow import Workflow from services.account_errors import AccountNotFoundError +from services.trigger.schedule_policy import visual_to_cron logger = logging.getLogger(__name__) class ScheduleService: - @staticmethod - def create_schedule( - tenant_id: str, app_id: str, config: ScheduleConfig, *, session: Session - ) -> WorkflowSchedulePlan: - """ - Create a new schedule with validated configuration. - - Args: - session: Database session - tenant_id: Tenant ID - app_id: Application ID - config: Validated schedule configuration - - Returns: - Created WorkflowSchedulePlan instance - """ - next_run_at = calculate_next_run_at( - config.cron_expression, - config.timezone, - ) - - schedule = WorkflowSchedulePlan( - tenant_id=tenant_id, - app_id=app_id, - node_id=config.node_id, - cron_expression=config.cron_expression, - timezone=config.timezone, - next_run_at=next_run_at, - ) - - session.add(schedule) - session.flush() - - return schedule - - @staticmethod - def update_schedule(schedule_id: str, updates: SchedulePlanUpdate, *, session: Session) -> WorkflowSchedulePlan: - """ - Update an existing schedule with validated configuration. - - Args: - session: Database session - schedule_id: Schedule ID to update - updates: Validated update configuration - - Raises: - ScheduleNotFoundError: If schedule not found - - Returns: - Updated WorkflowSchedulePlan instance - """ - schedule = session.get(WorkflowSchedulePlan, schedule_id) - if not schedule: - raise ScheduleNotFoundError(f"Schedule not found: {schedule_id}") - - # If time-related fields are updated, synchronously update the next_run_at. - time_fields_updated = False - - if updates.node_id is not None: - schedule.node_id = updates.node_id - - if updates.cron_expression is not None: - schedule.cron_expression = updates.cron_expression - time_fields_updated = True - - if updates.timezone is not None: - schedule.timezone = updates.timezone - time_fields_updated = True - - if time_fields_updated: - schedule.next_run_at = calculate_next_run_at( - schedule.cron_expression, - schedule.timezone, - ) - - session.flush() - return schedule - - @staticmethod - def delete_schedule(schedule_id: str, *, session: Session) -> None: - """ - Delete a schedule plan. - - Args: - session: Database session - schedule_id: Schedule ID to delete - """ - schedule = session.get(WorkflowSchedulePlan, schedule_id) - if not schedule: - raise ScheduleNotFoundError(f"Schedule not found: {schedule_id}") - - session.delete(schedule) - session.flush() - @staticmethod def get_tenant_owner(tenant_id: str, *, session: Session) -> Account: """ @@ -146,26 +46,6 @@ def get_tenant_owner(tenant_id: str, *, session: Session) -> Account: else: raise AccountNotFoundError(f"Account not found for tenant: {tenant_id}") - @staticmethod - def update_next_run_at(schedule_id: str, *, session: Session) -> datetime: - """ - Advances the schedule to its next execution time after a successful trigger. - Uses current time as base to prevent missing executions during delays. - """ - schedule = session.get(WorkflowSchedulePlan, schedule_id) - if not schedule: - raise ScheduleNotFoundError(f"Schedule not found: {schedule_id}") - - # Base on current time to handle execution delays gracefully - next_run_at = calculate_next_run_at( - schedule.cron_expression, - schedule.timezone, - ) - - schedule.next_run_at = next_run_at - session.flush() - return next_run_at - @staticmethod def to_schedule_config(node_config: NodeConfigDict) -> ScheduleConfig: """ @@ -187,119 +67,9 @@ def to_schedule_config(node_config: NodeConfigDict) -> ScheduleConfig: if not frequency: raise ScheduleConfigError("Frequency is required for visual mode") visual_config = VisualConfig.model_validate(node_data.visual_config or {}) - cron_expression = ScheduleService.visual_to_cron(frequency=frequency, visual_config=visual_config) + cron_expression = visual_to_cron(frequency=frequency, visual_config=visual_config) if not cron_expression: raise ScheduleConfigError("Cron expression is required for visual mode") else: raise ScheduleConfigError(f"Invalid schedule mode: {mode}") return ScheduleConfig(node_id=node_id, cron_expression=cron_expression, timezone=timezone) - - @staticmethod - def extract_schedule_config(workflow: Workflow) -> ScheduleConfig | None: - """ - Extracts schedule configuration from workflow graph. - - Searches for the first schedule trigger node in the workflow and converts - its configuration (either visual or cron mode) into a unified ScheduleConfig. - - Args: - workflow: The workflow containing the graph definition - - Returns: - ScheduleConfig if a valid schedule node is found, None if no schedule node exists - - Raises: - ScheduleConfigError: If graph parsing fails or schedule configuration is invalid - - Note: - Currently only returns the first schedule node found. - Multiple schedule nodes in the same workflow are not supported. - """ - try: - graph_data = workflow.graph_dict - except (json.JSONDecodeError, TypeError, AttributeError) as e: - raise ScheduleConfigError(f"Failed to parse workflow graph: {e}") - - if not graph_data: - raise ScheduleConfigError("Workflow graph is empty") - - nodes = graph_data.get("nodes", []) - for node in nodes: - node_data = node.get("data", {}) - - if node_data.get("type") != TRIGGER_SCHEDULE_NODE_TYPE: - continue - - node_id = node.get("id", "start") - trigger_data = TriggerScheduleNodeData.model_validate(node_data) - mode = trigger_data.mode - timezone = trigger_data.timezone - - cron_expression = None - if mode == "cron": - cron_expression = trigger_data.cron_expression - if not cron_expression: - raise ScheduleConfigError("Cron expression is required for cron mode") - elif mode == "visual": - frequency = trigger_data.frequency - if not frequency: - raise ScheduleConfigError("Frequency is required for visual mode") - visual_config = VisualConfig.model_validate(trigger_data.visual_config or {}) - cron_expression = ScheduleService.visual_to_cron(frequency, visual_config) - else: - raise ScheduleConfigError(f"Invalid schedule mode: {mode}") - - return ScheduleConfig(node_id=node_id, cron_expression=cron_expression, timezone=timezone) - - return None - - @staticmethod - def visual_to_cron(frequency: str, visual_config: VisualConfig) -> str: - """ - Converts user-friendly visual schedule settings to cron expression. - Maintains consistency with frontend UI expectations while supporting croniter's extended syntax. - """ - if frequency == "hourly": - if visual_config.on_minute is None: - raise ScheduleConfigError("on_minute is required for hourly schedules") - return f"{visual_config.on_minute} * * * *" - - elif frequency == "daily": - if not visual_config.time: - raise ScheduleConfigError("time is required for daily schedules") - hour, minute = convert_12h_to_24h(visual_config.time) - return f"{minute} {hour} * * *" - - elif frequency == "weekly": - if not visual_config.time: - raise ScheduleConfigError("time is required for weekly schedules") - if not visual_config.weekdays: - raise ScheduleConfigError("Weekdays are required for weekly schedules") - hour, minute = convert_12h_to_24h(visual_config.time) - weekday_map = {"sun": "0", "mon": "1", "tue": "2", "wed": "3", "thu": "4", "fri": "5", "sat": "6"} - cron_weekdays = [weekday_map[day] for day in visual_config.weekdays] - return f"{minute} {hour} * * {','.join(sorted(cron_weekdays))}" - - elif frequency == "monthly": - if not visual_config.time: - raise ScheduleConfigError("time is required for monthly schedules") - if not visual_config.monthly_days: - raise ScheduleConfigError("Monthly days are required for monthly schedules") - hour, minute = convert_12h_to_24h(visual_config.time) - - numeric_days: list[int] = [] - has_last = False - for day in visual_config.monthly_days: - if day == "last": - has_last = True - else: - numeric_days.append(day) - - result_days = [str(d) for d in sorted(set(numeric_days))] - if has_last: - result_days.append("L") - - return f"{minute} {hour} {','.join(result_days)} * *" - - else: - raise ScheduleConfigError(f"Unsupported frequency: {frequency}") diff --git a/api/services/trigger/trigger_service.py b/api/services/trigger/trigger_service.py index 911331e357c28b..cec7945505f2a5 100644 --- a/api/services/trigger/trigger_service.py +++ b/api/services/trigger/trigger_service.py @@ -1,30 +1,20 @@ import logging import secrets import time -from collections.abc import Mapping -from typing import Any from flask import Request, Response -from pydantic import BaseModel -from sqlalchemy import select -from sqlalchemy.orm import sessionmaker from core.plugin.entities.plugin_daemon import CredentialType from core.plugin.entities.request import TriggerDispatchResponse, TriggerInvokeEventResponse from core.plugin.impl.exc import PluginNotFoundError -from core.trigger.constants import TRIGGER_PLUGIN_NODE_TYPE from core.trigger.debug.events import PluginTriggerDebugEvent from core.trigger.provider import PluginTriggerProviderController from core.trigger.trigger_manager import TriggerManager from core.trigger.utils.encryption import create_trigger_provider_encrypter_for_subscription from core.workflow.nodes.trigger_plugin.entities import TriggerEventNodeData -from extensions.ext_database import db -from extensions.ext_redis import redis_client from graphon.entities.graph_config import NodeConfigDict -from models.model import App from models.provider_ids import TriggerProviderID -from models.trigger import TriggerSubscription, WorkflowPluginTrigger -from models.workflow import Workflow +from models.trigger import TriggerSubscription from services.trigger.trigger_provider_service import TriggerProviderService from services.trigger.trigger_request_service import TriggerHttpRequestCachingService from services.workflow.entities import PluginTriggerDispatchData @@ -37,8 +27,6 @@ class TriggerService: __TEMPORARY_ENDPOINT_EXPIRE_MS__ = 5 * 60 * 1000 __ENDPOINT_REQUEST_CACHE_COUNT__ = 10 __ENDPOINT_REQUEST_CACHE_EXPIRE_MS__ = 5 * 60 * 1000 - __PLUGIN_TRIGGER_NODE_CACHE_KEY__ = "plugin_trigger_nodes" - MAX_PLUGIN_TRIGGER_NODES_PER_WORKFLOW = 5 # Maximum allowed plugin trigger nodes per workflow @classmethod def invoke_trigger_event( @@ -148,155 +136,3 @@ def process_endpoint(cls, endpoint_id: str, request: Request) -> Response | None request_id, ) return dispatch_response.response - - @classmethod - def sync_plugin_trigger_relationships(cls, app: App, workflow: Workflow): - """ - Sync plugin trigger relationships in DB. - - 1. Check if the workflow has any plugin trigger nodes - 2. Fetch the nodes from DB, see if there were any plugin trigger records already - 3. Diff the nodes and the plugin trigger records, create/update/delete the records as needed - - Approach: - Frequent DB operations may cause performance issues, using Redis to cache it instead. - If any record exists, cache it. - - Limits: - - Maximum 5 plugin trigger nodes per workflow - """ - - class Cache(BaseModel): - """ - Cache model for plugin trigger nodes - """ - - record_id: str - node_id: str - provider_id: str - event_name: str - subscription_id: str - - # Walk nodes to find plugin triggers - nodes_in_graph: list[Mapping[str, Any]] = [] - for node_id, node_config in workflow.walk_nodes(TRIGGER_PLUGIN_NODE_TYPE): - # Extract plugin trigger configuration from node - plugin_id = node_config.get("plugin_id", "") - provider_id = node_config.get("provider_id", "") - event_name = node_config.get("event_name", "") - subscription_id = node_config.get("subscription_id", "") - - if not subscription_id: - continue - - nodes_in_graph.append( - { - "node_id": node_id, - "plugin_id": plugin_id, - "provider_id": provider_id, - "event_name": event_name, - "subscription_id": subscription_id, - } - ) - - # Check plugin trigger node limit - if len(nodes_in_graph) > cls.MAX_PLUGIN_TRIGGER_NODES_PER_WORKFLOW: - raise ValueError( - f"Workflow exceeds maximum plugin trigger node limit. " - f"Found {len(nodes_in_graph)} plugin trigger nodes, " - f"maximum allowed is {cls.MAX_PLUGIN_TRIGGER_NODES_PER_WORKFLOW}" - ) - - not_found_in_cache: list[Mapping[str, Any]] = [] - for node_info in nodes_in_graph: - node_id = node_info["node_id"] - # firstly check if the node exists in cache - if not redis_client.get(f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:{app.id}:{node_id}"): - not_found_in_cache.append(node_info) - continue - - with sessionmaker(bind=db.engine, expire_on_commit=False).begin() as session: - try: - # lock the concurrent plugin trigger creation - redis_client.lock(f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:apps:{app.id}:lock", timeout=10) - # fetch the non-cached nodes from DB - all_records = session.scalars( - select(WorkflowPluginTrigger).where( - WorkflowPluginTrigger.app_id == app.id, - WorkflowPluginTrigger.tenant_id == app.tenant_id, - ) - ).all() - - nodes_id_in_db = {node.node_id: node for node in all_records} - nodes_id_in_graph = {node["node_id"] for node in nodes_in_graph} - - # get the nodes not found both in cache and DB - nodes_not_found = [ - node_info for node_info in not_found_in_cache if node_info["node_id"] not in nodes_id_in_db - ] - - # create new plugin trigger records - for node_info in nodes_not_found: - plugin_trigger = WorkflowPluginTrigger( - app_id=app.id, - tenant_id=app.tenant_id, - node_id=node_info["node_id"], - provider_id=node_info["provider_id"], - event_name=node_info["event_name"], - subscription_id=node_info["subscription_id"], - ) - session.add(plugin_trigger) - session.flush() # Get the ID for caching - - cache = Cache( - record_id=plugin_trigger.id, - node_id=node_info["node_id"], - provider_id=node_info["provider_id"], - event_name=node_info["event_name"], - subscription_id=node_info["subscription_id"], - ) - redis_client.set( - f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:{app.id}:{node_info['node_id']}", - cache.model_dump_json(), - ex=60 * 60, - ) - - # Update existing records if subscription_id changed - for node_info in nodes_in_graph: - node_id = node_info["node_id"] - if node_id in nodes_id_in_db: - existing_record = nodes_id_in_db[node_id] - if ( - existing_record.subscription_id != node_info["subscription_id"] - or existing_record.provider_id != node_info["provider_id"] - or existing_record.event_name != node_info["event_name"] - ): - existing_record.subscription_id = node_info["subscription_id"] - existing_record.provider_id = node_info["provider_id"] - existing_record.event_name = node_info["event_name"] - session.add(existing_record) - - # Update cache - cache = Cache( - record_id=existing_record.id, - node_id=node_id, - provider_id=node_info["provider_id"], - event_name=node_info["event_name"], - subscription_id=node_info["subscription_id"], - ) - redis_client.set( - f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:{app.id}:{node_id}", - cache.model_dump_json(), - ex=60 * 60, - ) - - # delete the nodes not found in the graph - for node_id in nodes_id_in_db: - if node_id not in nodes_id_in_graph: - session.delete(nodes_id_in_db[node_id]) - redis_client.delete(f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:{app.id}:{node_id}") - except Exception: - logger.exception("Failed to sync plugin trigger relationships for app %s", app.id) - raise - finally: - redis_client.delete(f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:apps:{app.id}:lock") diff --git a/api/services/trigger/webhook_service.py b/api/services/trigger/webhook_service.py index 7381e9b3e36096..b57f2a383f00d1 100644 --- a/api/services/trigger/webhook_service.py +++ b/api/services/trigger/webhook_service.py @@ -1,22 +1,18 @@ import json import logging import mimetypes -import secrets from collections.abc import Callable, Mapping, Sequence from typing import Any, NotRequired, Protocol, TypedDict import orjson from flask import request -from pydantic import BaseModel from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker from werkzeug.datastructures import FileStorage -from werkzeug.exceptions import RequestEntityTooLarge from configs import dify_config from core.app.file_access import DatabaseFileAccessController from core.tools.tool_file_manager import ToolFileManager -from core.trigger.constants import TRIGGER_WEBHOOK_NODE_TYPE from core.workflow.nodes.trigger_webhook.entities import ( ContentType, WebhookBodyParameter, @@ -25,7 +21,6 @@ ) from enums import QuotaType from extensions.ext_database import db -from extensions.ext_redis import redis_client from factories import file_factory from graphon.entities.graph_config import NodeConfigDict from graphon.file import FileTransferMethod @@ -34,10 +29,13 @@ from models.model import App, EndUser from models.trigger import AppTrigger, WorkflowWebhookTrigger from models.workflow import Workflow +from repositories.trigger.workflow_repository import WorkflowTriggerRepository from services.async_workflow_service import AsyncWorkflowService from services.errors.app import QuotaExceededError from services.quota_service import QuotaService from services.trigger.app_trigger_service import AppTriggerService +from services.trigger.errors import WebhookBodyTooLargeError +from services.trigger.workflow_policy import webhook_node_ids from services.workflow.entities import WebhookTriggerData from services.workflow_service import WorkflowService @@ -83,9 +81,6 @@ def get_or_create_end_user_by_type( class WebhookService: """Service for handling webhook operations.""" - __WEBHOOK_NODE_CACHE_KEY__ = "webhook_nodes" - MAX_WEBHOOK_NODES_PER_WORKFLOW = 5 # Maximum allowed webhook nodes per workflow - @staticmethod def _sanitize_key(key: str) -> str: """Normalize external keys (headers/params) to workflow-safe variables.""" @@ -291,7 +286,7 @@ def _validate_content_length(cls) -> None: """Validate request content length against maximum allowed size.""" content_length = request.content_length if content_length and content_length > dify_config.WEBHOOK_REQUEST_BODY_MAX_SIZE: - raise RequestEntityTooLarge( + raise WebhookBodyTooLargeError( f"Webhook request too large: {content_length} bytes exceeds maximum allowed size " f"of {dify_config.WEBHOOK_REQUEST_BODY_MAX_SIZE} bytes" ) @@ -901,115 +896,16 @@ def generate_webhook_response(cls, node_config: NodeConfigDict) -> tuple[dict[st return response_data, status_code @classmethod - def sync_webhook_relationships(cls, app: App, workflow: Workflow, *, remove_stale: bool = True): - """ - Sync webhook relationships in DB. - - 1. Check if the workflow has any webhook trigger nodes - 2. Fetch the nodes from DB, see if there were any webhook records already - 3. Diff the nodes and the webhook records, creating missing records and optionally deleting stale records - - Draft workflow synchronization preserves stale records so undo can restore a - webhook node without changing its URL. Published workflow synchronization - removes stale records after the deletion becomes effective. - - Approach: - Frequent DB operations may cause performance issues, using Redis to cache it instead. - If any record exists, cache it. - - Limits: - - Maximum 5 webhook nodes per workflow - """ - - class Cache(BaseModel): - """ - Cache model for webhook nodes - """ - - record_id: str - node_id: str - webhook_id: str - - nodes_id_in_graph = [node_id for node_id, _ in workflow.walk_nodes(TRIGGER_WEBHOOK_NODE_TYPE)] - - # Check webhook node limit - if len(nodes_id_in_graph) > cls.MAX_WEBHOOK_NODES_PER_WORKFLOW: - raise ValueError( - f"Workflow exceeds maximum webhook node limit. " - f"Found {len(nodes_id_in_graph)} webhook nodes, maximum allowed is {cls.MAX_WEBHOOK_NODES_PER_WORKFLOW}" - ) - - not_found_in_cache: list[str] = [] - for node_id in nodes_id_in_graph: - # firstly check if the node exists in cache - if not redis_client.get(f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:{node_id}"): - not_found_in_cache.append(node_id) - continue - - lock_key = f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:apps:{app.id}:lock" - lock = redis_client.lock(lock_key, timeout=10) - lock_acquired = False - - try: - # acquire the lock with blocking and timeout - lock_acquired = lock.acquire(blocking=True, blocking_timeout=10) - if not lock_acquired: - logger.warning("Failed to acquire lock for webhook sync, app %s", app.id) - raise RuntimeError("Failed to acquire lock for webhook trigger synchronization") - - with sessionmaker(bind=db.engine, expire_on_commit=False).begin() as session: - # fetch the non-cached nodes from DB - all_records = session.scalars( - select(WorkflowWebhookTrigger).where( - WorkflowWebhookTrigger.app_id == app.id, - WorkflowWebhookTrigger.tenant_id == app.tenant_id, - ) - ).all() - - nodes_id_in_db = {node.node_id: node for node in all_records} - - # get the nodes not found both in cache and DB - nodes_not_found = [node_id for node_id in not_found_in_cache if node_id not in nodes_id_in_db] - - # create new webhook records - for node_id in nodes_not_found: - webhook_record = WorkflowWebhookTrigger( - app_id=app.id, - tenant_id=app.tenant_id, - node_id=node_id, - webhook_id=cls.generate_webhook_id(), - created_by=app.created_by, - ) - session.add(webhook_record) - session.flush() - cache = Cache(record_id=webhook_record.id, node_id=node_id, webhook_id=webhook_record.webhook_id) - redis_client.set( - f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:{node_id}", cache.model_dump_json(), ex=60 * 60 - ) - - if remove_stale: - # Delete relationships only when reconciling an effective published workflow. - for node_id in nodes_id_in_db: - if node_id not in nodes_id_in_graph: - session.delete(nodes_id_in_db[node_id]) - redis_client.delete(f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:{node_id}") - except Exception: - logger.exception("Failed to sync webhook relationships for app %s", app.id) - raise - finally: - # release the lock only if it was acquired - if lock_acquired: - try: - lock.release() - except Exception: - logger.exception("Failed to release lock for webhook sync, app %s", app.id) - - @classmethod - def generate_webhook_id(cls) -> str: - """ - Generate unique 24-character webhook ID - - Deduplication is not needed, DB already has unique constraint on webhook_id. - """ - # Generate 24-character random string - return secrets.token_urlsafe(18)[:24] # token_urlsafe gives base64url, take first 24 chars + def sync_webhook_relationships( + cls, + app: App, + workflow: Workflow, + *, + remove_stale: bool = True, + session_factory: sessionmaker[Session] | None = None, + ) -> None: + """Allocate draft URLs with the same row lock and persistence used by publication.""" + node_ids = webhook_node_ids(workflow.graph_dict.get("nodes", [])) + with (session_factory or sessionmaker(db.engine)).begin() as session: + WorkflowTriggerRepository.lock_app(session, app) + WorkflowTriggerRepository.sync_webhooks(session, app, node_ids, remove_stale=remove_stale) diff --git a/api/services/workflow/execution/adapters/node_factory.py b/api/services/workflow/execution/adapters/node_factory.py index 381b4ec98b1d39..0a6e2e31928326 100644 --- a/api/services/workflow/execution/adapters/node_factory.py +++ b/api/services/workflow/execution/adapters/node_factory.py @@ -25,6 +25,7 @@ should_resolve_llm_model_selector, ) from core.workflow.llm_node import DifyLLMNode +from core.workflow.nodes.agent.message_transformer import AgentMessageTransformer from core.workflow.nodes.agent.plugin_strategy_adapter import ( PluginAgentStrategyPresentationProvider, PluginAgentStrategyResolver, @@ -53,7 +54,6 @@ from models import Account from models.agent_runtime_contracts import WorkflowAgentRuntimeBindings from models.human_input_entities import HumanInputNodeData as DifyHumanInputNodeData -from services.workflow.execution.adapters.agent_message_transformer import AgentMessageTransformer from services.workflow.execution.adapters.agent_runtime import AgentRuntimeSupport from services.workflow.execution.adapters.agent_v2.agent_node import DifyAgentNode from services.workflow.execution.adapters.agent_v2.runtime_request_builder import WorkflowAgentRuntimeRequestBuilder @@ -119,7 +119,24 @@ def _import_node_package(package_name: str, *, excluded_modules: frozenset[str] def register_nodes() -> None: """Import production node modules so they self-register with ``Node``.""" _import_node_package("graphon.nodes") - _import_node_package("core.workflow.nodes") + _import_node_package( + "core.workflow.nodes", + excluded_modules=frozenset( + { + "core.workflow.nodes.agent.agent_node", + "core.workflow.nodes.agent.runtime_support", + "core.workflow.nodes.agent_v2.agent_node", + "core.workflow.nodes.agent_v2.binding_resolver", + "core.workflow.nodes.agent_v2.dify_tools_builder", + "core.workflow.nodes.agent_v2.runtime_request_builder", + "core.workflow.nodes.agent_v2.validators", + "core.workflow.nodes.human_input.boundary", + "core.workflow.nodes.human_input.callback", + "core.workflow.nodes.knowledge_retrieval.knowledge_retrieval_node", + "core.workflow.nodes.trigger_webhook.node", + } + ), + ) for module_name in ( "services.workflow.execution.adapters.agent_node", "services.workflow.execution.adapters.knowledge_retrieval", diff --git a/api/services/workflow/node_output_inspector_service.py b/api/services/workflow/node_output_inspector_service.py index b972330484b4ad..cd0af728c677a4 100644 --- a/api/services/workflow/node_output_inspector_service.py +++ b/api/services/workflow/node_output_inspector_service.py @@ -26,7 +26,7 @@ Cross-tenant / cross-app rows still 404 via the standard tenant/app scope. 3. **Declared outputs by node kind**: * Agent v2 nodes resolve their declared list via - :class:`WorkflowAgentBindingResolver` (the binding owns custom declarations; + :class:`WorkflowAgentRuntimeBindings` (the binding owns custom declarations; the system ``text`` output is derived for display). * Other node kinds don't have a declared-output schema yet; we surface the keys present in the execution payload as a best-effort list typed @@ -55,16 +55,13 @@ from sqlalchemy.orm import Session from core.app.file_access import DatabaseFileAccessController -from core.workflow.nodes.agent_v2.binding_resolver import ( - WorkflowAgentBindingError, - WorkflowAgentBindingResolver, -) from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data from factories.file_factory.builders import build_from_mapping from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus from graphon.file import helpers as file_helpers from models import App from models.agent_config_entities import DeclaredOutputConfig, DeclaredOutputType, effective_declared_outputs +from models.agent_runtime_contracts import WorkflowAgentBindingError, WorkflowAgentRuntimeBindings from models.workflow import WorkflowNodeExecutionModel, WorkflowRun logger = logging.getLogger(__name__) @@ -396,7 +393,7 @@ class NodeOutputInspectorService: """Read-only Inspector for draft + published workflow runs. The service is dependency-light: it holds a single - :class:`WorkflowAgentBindingResolver` so agent v2 nodes can map to their + :class:`WorkflowAgentRuntimeBindings` so agent v2 nodes can map to their declared outputs without re-implementing binding lookup. All other I/O receives an explicit SQLAlchemy session from its caller so transaction ownership stays at the controller/task boundary. @@ -405,8 +402,8 @@ class NodeOutputInspectorService: every load — the same scope guard regardless of trigger source. """ - def __init__(self, binding_resolver: WorkflowAgentBindingResolver | None = None) -> None: - self._binding_resolver = binding_resolver or WorkflowAgentBindingResolver() + def __init__(self, binding_resolver: WorkflowAgentRuntimeBindings) -> None: + self._binding_resolver = binding_resolver # ── public API ──────────────────────────────────────────────────────── diff --git a/api/services/workflow/runtime_gateway.py b/api/services/workflow/runtime_gateway.py new file mode 100644 index 00000000000000..11acbaa852a3ef --- /dev/null +++ b/api/services/workflow/runtime_gateway.py @@ -0,0 +1,266 @@ +from services.workflow.node_execution_extras import node_execution_extras + +"""Adapt the existing workflow engine to framework-neutral Console ports.""" + +from collections.abc import Mapping, Sequence +from typing import Any + +from sqlalchemy.orm import Session, sessionmaker + +from core.app.app_config.features.file_upload.manager import FileUploadConfigManager +from core.app.apps.base_app_queue_manager import AppQueueManager +from core.app.entities.app_invoke_entities import InvokeFrom +from core.app.file_access import DatabaseFileAccessController +from core.plugin.impl.exc import PluginInvokeError +from core.trigger.constants import TRIGGER_SCHEDULE_NODE_TYPE +from core.trigger.debug.event_selectors import create_event_poller, select_trigger_debug_events +from factories import file_factory +from graphon.file import File +from graphon.graph_engine.manager import GraphEngineManager +from machinery.context import RequestContext +from models import Account, App +from models.workflow import Workflow, WorkflowNodeExecutionModel +from repositories.agent.retirement_repository import WorkflowAgentRetirementRepository +from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository +from repositories.workflow.definition_repository import ( + WorkflowDefinitionRepository, + workflow_from_snapshot, + workflow_snapshot, +) +from repositories.workflow.node_execution_repository import WorkflowNodeExecutionRepository +from services.agent.retirement_service import WorkflowAgentRetirementService +from services.app.generation.runtime import AppGenerationRuntime +from services.app_generate_service import AppGenerateService +from services.errors.app import WorkflowNotFoundError +from services.workflow.contracts import ( + WorkflowGeneration, + WorkflowSnapshot, + WorkflowTriggerError, + WorkflowTriggerEvent, +) +from services.workflow.stream_reservation import WorkflowStreamReservation +from services.workflow.variable_contracts import WorkflowExecutionVariables +from services.workflow_service import WorkflowService + + +class WorkflowRuntimeGateway: + def __init__( + self, + *, + session_factory: sessionmaker[Session], + workflows: WorkflowService, + definitions: WorkflowDefinitionRepository, + reservations: WorkflowDebugReservationRepository, + executions: WorkflowNodeExecutionRepository, + generator: type[AppGenerateService], + graph_engine: GraphEngineManager, + file_access: DatabaseFileAccessController, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, + ) -> None: + self._sessions = session_factory + self._workflows = workflows + self._definitions = definitions + self._reservations = reservations + self._executions = executions + self._generator = generator + self._graph_engine = graph_engine + self._file_access = file_access + self._variables = variables + self._runtime = runtime + + def generate( + self, + context: RequestContext, + app_id: str, + args: dict[str, Any], + *, + root_node_id: str | None, + workflow: WorkflowSnapshot | None, + ) -> WorkflowGeneration: + reservation = ( + WorkflowStreamReservation(lambda: self._cancel_trigger_debug(workflow, context.account_id)) + if workflow is not None and workflow.execution_id is not None + else None + ) + try: + app, actor, definition = self._definitions.debug_context( + context, app_id, workflow_id=args.get("workflow_id"), snapshot=workflow + ) + stream = self._generator.generate_workflow_stream( + runtime=self._runtime, + variables=self._variables, + app_model=app, + user=actor, + workflow=definition, + args=args, + invoke_from=InvokeFrom.DEBUGGER, + root_node_id=root_node_id, + workflow_snapshot=workflow, + reservation=reservation, + ) + return reservation.wrap(stream) if reservation is not None else stream + except Exception as error: + if reservation is not None: + reservation.close() + if isinstance(error, PluginInvokeError) and root_node_id is not None: + raise WorkflowTriggerError(error.to_user_friendly_error()) from error + raise + + def _cancel_trigger_debug(self, snapshot: WorkflowSnapshot, account_id: str) -> None: + if snapshot.execution_id is None: + return + WorkflowAgentRetirementService.finish_execution( + sessions=self._sessions, + tenant_id=snapshot.tenant_id, + app_id=snapshot.app_id, + workflow_id=snapshot.id, + execution_id=snapshot.execution_id, + account_id=account_id, + cancel=True, + ) + + def iteration( + self, context: RequestContext, app_id: str, node_id: str, inputs: dict[str, Any] | None + ) -> WorkflowGeneration: + app, actor, workflow = self._definitions.debug_context(context, app_id, workflow_id=None, snapshot=None) + return self._generator.generate_single_iteration( + runtime=self._runtime, + variables=self._variables, + app_model=app, + user=actor, + workflow=workflow, + node_id=node_id, + args={"inputs": inputs} if inputs is not None else {}, + streaming=True, + ) + + def loop( + self, context: RequestContext, app_id: str, node_id: str, inputs: dict[str, Any] | None + ) -> WorkflowGeneration: + app, actor, workflow = self._definitions.debug_context(context, app_id, workflow_id=None, snapshot=None) + return self._generator.generate_single_loop( + runtime=self._runtime, + variables=self._variables, + app_model=app, + user=actor, + workflow=workflow, + node_id=node_id, + args={"inputs": inputs}, + streaming=True, + ) + + def _draft(self, context: RequestContext, app_id: str, missing_message: str) -> tuple[App, Account, Workflow]: + try: + return self._definitions.debug_context(context, app_id, workflow_id=None, snapshot=None) + except ValueError as error: + raise ValueError(missing_message) from error + + def _files(self, workflow: Workflow, files: list[dict[str, Any]] | None) -> Sequence[File]: + config = FileUploadConfigManager.convert(workflow.features_dict, is_vision=False) + if config is None: + return [] + return file_factory.build_from_mappings( + mappings=files or [], + tenant_id=workflow.tenant_id, + config=config, + access_controller=self._file_access, + sessions=self._sessions, + ) + + def _execution(self, execution: WorkflowNodeExecutionModel, *, include_details: bool) -> dict[str, Any]: + detail = self._executions.execution_record(execution, include_details=include_details) + if include_details: + # Plugin icons can perform external I/O; repository reads are closed. + detail["extras"] = node_execution_extras(execution, tool_providers=self._runtime.tool_providers) + return detail + + def run_node( + self, + context: RequestContext, + app_id: str, + node_id: str, + args: dict[str, Any], + *, + include_details: bool, + workflow: WorkflowSnapshot | None, + ) -> dict[str, Any]: + if workflow is None: + app, actor, draft = self._draft(context, app_id, "Workflow not initialized") + else: + app, actor, draft = self._definitions.debug_context(context, app_id, workflow_id=None, snapshot=workflow) + raw_files = args.get("files") + execution = self._workflows.run_draft_workflow_node( + variables=self._variables, + app_model=app, + draft_workflow=draft, + account=actor, + node_id=node_id, + user_inputs=args.get("inputs") or {}, + query=args.get("query", ""), + files=self._files(draft, raw_files if isinstance(raw_files, list) else None), + ) + return self._execution(execution, include_details=include_details) + + def last_run(self, context: RequestContext, app_id: str, node_id: str) -> dict[str, Any] | None: + try: + app, _, workflow = self._draft(context, app_id, "Workflow not found") + except ValueError as error: + raise WorkflowNotFoundError(str(error)) from error + execution = self._workflows.get_node_last_run(app_model=app, workflow=workflow, node_id=node_id) + return self._execution(execution, include_details=True) if execution is not None else None + + def poll_trigger( + self, context: RequestContext, app_id: str, node_ids: list[str], *, single_node: bool, select_all: bool + ) -> WorkflowTriggerEvent | None: + if single_node: + app, _, workflow = self._draft(context, app_id, "Workflow not found") + snapshot = workflow_snapshot(workflow) + else: + app, snapshot = self._reservations.reserve_trigger_debug(context, app_id) + workflow = workflow_from_snapshot(snapshot) + if single_node: + node_id = node_ids[0] + config = workflow.get_node_config_by_id(node_id=node_id) + if not config: + raise ValueError("Node data not found for node %s", node_id) + if workflow.get_node_type_from_node_config(config) == TRIGGER_SCHEDULE_NODE_TYPE: + return WorkflowTriggerEvent(node_id=node_id, workflow_args={}, workflow=snapshot) + try: + if select_all: + event = select_trigger_debug_events( + draft_workflow=workflow, app_model=app, user_id=context.account_id, node_ids=node_ids + ) + else: + event = create_event_poller( + draft_workflow=workflow, + tenant_id=context.active_workspace_id, + user_id=context.account_id, + app_id=app_id, + node_id=node_ids[0], + ).poll() + except Exception as error: + self._cancel_trigger_debug(snapshot, context.account_id) + if isinstance(error, PluginInvokeError): + raise WorkflowTriggerError(error.to_user_friendly_error()) from error + raise + if event is None: + self._cancel_trigger_debug(snapshot, context.account_id) + return None + return WorkflowTriggerEvent(event.node_id, dict(event.workflow_args), snapshot) + + def stop(self, task_id: str) -> None: + # Both mechanisms are still consumed by running workflows. + AppQueueManager.set_stop_flag_no_user_check(task_id) + self._graph_engine.send_stop_command(task_id) + + def default_blocks(self) -> Sequence[Mapping[str, object]]: + return self._workflows.get_default_block_configs() + + def default_block(self, block_type: str, filters: dict[str, Any] | None) -> Mapping[str, object]: + return self._workflows.get_default_block_config(node_type=block_type, filters=filters) + + def retire_agents(self, context: RequestContext, agent_ids: list[str]) -> None: + WorkflowAgentRetirementService(WorkflowAgentRetirementRepository(self._sessions)).retire_unowned( + tenant_id=context.active_workspace_id, account_id=context.account_id, agent_ids=agent_ids + ) diff --git a/api/services/workflow/workflow_converter.py b/api/services/workflow/workflow_converter.py index c36edca335ae07..1087a381867379 100644 --- a/api/services/workflow/workflow_converter.py +++ b/api/services/workflow/workflow_converter.py @@ -1,33 +1,33 @@ import json -from typing import Any, TypedDict - -from sqlalchemy import select -from sqlalchemy.orm import Session - +from collections.abc import Mapping +from copy import deepcopy +from typing import Any, TypedDict, cast + +from core.app.app_config.easy_ui_based_app.dataset.manager import DatasetConfigManager +from core.app.app_config.easy_ui_based_app.model_config.manager import ModelConfigManager +from core.app.app_config.easy_ui_based_app.prompt_template.manager import PromptTemplateConfigManager +from core.app.app_config.easy_ui_based_app.variables.manager import BasicVariablesConfigManager from core.app.app_config.entities import ( + AppAdditionalFeatures, DatasetEntity, DatasetRetrieveConfigEntity, EasyUIBasedAppConfig, + EasyUIBasedAppModelConfigFrom, ExternalDataVariableEntity, ModelConfigEntity, PromptTemplateEntity, ) -from core.app.apps.agent_chat.app_config_manager import AgentChatAppConfigManager -from core.app.apps.chat.app_config_manager import ChatAppConfigManager -from core.app.apps.completion.app_config_manager import CompletionAppConfigManager -from core.helper import encrypter +from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.prompt.simple_prompt_transform import SimplePromptTransform from core.prompt.utils.prompt_template_parser import PromptTemplateParser -from events.app_event import app_was_created from graphon.file import FileUploadConfig from graphon.model_runtime.entities.llm_entities import LLMMode from graphon.model_runtime.utils.encoders import jsonable_encoder from graphon.nodes import BuiltinNodeTypes from graphon.variables.input_entities import VariableEntity -from models import Account -from models.api_based_extension import APIBasedExtension, APIBasedExtensionPoint -from models.model import App, AppMode, AppModelConfig, IconType, load_annotation_reply_config -from models.workflow import Workflow, WorkflowType +from models.api_based_extension import APIBasedExtensionPoint +from models.model import AppMode +from models.workflow_conversion import ConversionExtension, ConvertedWorkflow, WorkflowConversionSource class _NodeType(TypedDict): @@ -52,88 +52,11 @@ class WorkflowConverter: App Convert to Workflow Mode """ - def convert_to_workflow( - self, - app_model: App, - account: Account, - name: str, - icon_type: str, - icon: str, - icon_background: str, - session: Session, - ): - """ - Convert app to workflow - - - basic mode of chatbot app - - - expert mode of chatbot app - - - completion app - - :param app_model: App instance - :param account: Account - :param name: new app name - :param icon: new app icon - :param icon_type: new app icon type - :param icon_background: new app icon background - :return: new App instance - """ - # convert app model config - app_model_config = ( - session.get(AppModelConfig, app_model.app_model_config_id) if app_model.app_model_config_id else None - ) - if not app_model_config: - raise ValueError("App model config is required") - - workflow = self.convert_app_model_config_to_workflow( - app_model=app_model, app_model_config=app_model_config, account_id=account.id, session=session - ) - - # create new app - new_app = App() - new_app.tenant_id = app_model.tenant_id - new_app.name = name or app_model.name + "(workflow)" - new_app.mode = AppMode.ADVANCED_CHAT if app_model.mode == AppMode.CHAT else AppMode.WORKFLOW - new_app.icon_type = IconType(icon_type) if icon_type else app_model.icon_type - new_app.icon = icon or app_model.icon - new_app.icon_background = icon_background or app_model.icon_background - new_app.enable_site = app_model.enable_site - new_app.enable_api = app_model.enable_api - new_app.api_rpm = app_model.api_rpm - new_app.api_rph = app_model.api_rph - new_app.is_demo = False - new_app.is_public = app_model.is_public - new_app.created_by = account.id - new_app.maintainer = account.id - new_app.updated_by = account.id - session.add(new_app) - session.flush() - - workflow.app_id = new_app.id - session.commit() - - app_was_created.send(new_app, account=account, session=session) - session.commit() - - return new_app - - def convert_app_model_config_to_workflow( - self, app_model: App, app_model_config: AppModelConfig, account_id: str, session: Session - ): - """ - Convert app model config to workflow mode - :param app_model: App instance - :param app_model_config: AppModelConfig instance - :param account_id: Account ID - """ - # get new app mode - new_app_mode = self._get_new_app_mode(app_model) - - # convert app model config - app_config = self._convert_to_app_config( - app_model=app_model, app_model_config=app_model_config, session=session - ) + def convert( + self, app_config: EasyUIBasedAppConfig, extensions: Mapping[str, ConversionExtension] + ) -> ConvertedWorkflow: + """Build graph and features from materialized configuration; perform no I/O.""" + new_app_mode = AppMode.WORKFLOW if app_config.app_mode == AppMode.COMPLETION else AppMode.ADVANCED_CHAT # init workflow graph graph: WorkflowGraph = {"nodes": [], "edges": []} @@ -156,10 +79,11 @@ def convert_app_model_config_to_workflow( external_data_variable_node_mapping: dict[str, str] = {} if app_config.external_data_variables: http_request_nodes, external_data_variable_node_mapping = self._convert_to_http_request_node( - app_model=app_model, + app_id=app_config.app_id, + app_mode=app_config.app_mode, variables=app_config.variables, external_data_variables=app_config.external_data_variables, - session=session, + extensions=extensions, ) for http_request_node in http_request_nodes: @@ -176,7 +100,7 @@ def convert_app_model_config_to_workflow( # convert to llm node llm_node = self._convert_to_llm_node( - original_app_mode=AppMode.value_of(app_model.mode), + original_app_mode=app_config.app_mode, new_app_mode=new_app_mode, graph=graph, model_config=app_config.model, @@ -211,70 +135,27 @@ def convert_app_model_config_to_workflow( "sensitive_word_avoidance": app_model_config_dict.get("sensitive_word_avoidance"), "retriever_resource": app_model_config_dict.get("retriever_resource"), } - case _: - answer_node = self._convert_to_answer_node() - graph = self._append_node(graph, answer_node) - features = { - "text_to_speech": app_model_config_dict.get("text_to_speech"), - "file_upload": app_model_config_dict.get("file_upload"), - "sensitive_word_avoidance": app_model_config_dict.get("sensitive_word_avoidance"), - } - - # create workflow record - workflow = Workflow( - tenant_id=app_model.tenant_id, - app_id=app_model.id, - type=WorkflowType.from_app_mode(new_app_mode).value, - version=Workflow.VERSION_DRAFT, - graph=json.dumps(graph), - features=json.dumps(features), - created_by=account_id, - environment_variables=[], - conversation_variables=[], - ) - - session.add(workflow) - session.commit() - - return workflow - - def _convert_to_app_config( - self, app_model: App, app_model_config: AppModelConfig, *, session: Session - ) -> EasyUIBasedAppConfig: - app_mode_enum = AppMode.value_of(app_model.mode) - app_config: EasyUIBasedAppConfig - effective_mode = ( - AppMode.AGENT_CHAT - if app_model.is_agent_with_session(session=session) and app_mode_enum != AppMode.AGENT_CHAT - else app_mode_enum + return ConvertedWorkflow(mode=new_app_mode, graph=cast(dict[str, Any], graph), features=features) + + def app_config(self, source: WorkflowConversionSource) -> EasyUIBasedAppConfig: + """Reuse the config parsers without loading or mutating an ORM App.""" + assert source.model_config is not None + config = deepcopy(source.model_config) + variables, external_variables = BasicVariablesConfigManager.convert(config) + return EasyUIBasedAppConfig( + tenant_id=source.tenant_id, + app_id=source.id, + app_mode=source.mode, + app_model_config_from=EasyUIBasedAppModelConfigFrom.APP_LATEST_CONFIG, + app_model_config_id=source.model_config_id, + app_model_config_dict=cast(dict[str, Any], config), + model=ModelConfigManager.convert(config), + prompt_template=PromptTemplateConfigManager.convert(config), + dataset=DatasetConfigManager.convert(config), + variables=variables, + external_data_variables=external_variables, + additional_features=AppAdditionalFeatures(file_upload=FileUploadConfigManager.convert(config)), ) - match effective_mode: - case AppMode.AGENT_CHAT: - app_model.mode = AppMode.AGENT_CHAT - annotation_reply = load_annotation_reply_config(session, app_model_config.app_id) - app_config = AgentChatAppConfigManager.get_app_config( - app_model=app_model, - app_model_config=app_model_config, - annotation_reply=annotation_reply, - ) - case AppMode.CHAT: - annotation_reply = load_annotation_reply_config(session, app_model_config.app_id) - app_config = ChatAppConfigManager.get_app_config( - app_model=app_model, - app_model_config=app_model_config, - annotation_reply=annotation_reply, - ) - case AppMode.COMPLETION: - annotation_reply = load_annotation_reply_config(session, app_model_config.app_id) - app_config = CompletionAppConfigManager.get_app_config( - app_model=app_model, - app_model_config=app_model_config, - annotation_reply=annotation_reply, - ) - case _: - raise ValueError("Invalid app mode") - - return app_config def _convert_to_start_node(self, variables: list[VariableEntity]) -> _NodeType: """ @@ -294,14 +175,16 @@ def _convert_to_start_node(self, variables: list[VariableEntity]) -> _NodeType: def _convert_to_http_request_node( self, - app_model: App, + app_id: str, + app_mode: AppMode, variables: list[VariableEntity], external_data_variables: list[ExternalDataVariableEntity], - session: Session, + extensions: Mapping[str, ConversionExtension], ) -> tuple[list[_NodeType], dict[str, str]]: """ Convert API Based Extension to HTTP Request Node - :param app_model: App instance + :param app_id: source app id + :param app_mode: source app mode :param variables: list of variables :param external_data_variables: list of external data variables :return: @@ -309,7 +192,6 @@ def _convert_to_http_request_node( index = 1 nodes = [] external_data_variable_node_mapping = {} - tenant_id = app_model.tenant_id for external_data_variable in external_data_variables: tool_type = external_data_variable.type if tool_type != "api": @@ -323,13 +205,8 @@ def _convert_to_http_request_node( if not api_based_extension_id: continue - # get api_based_extension - api_based_extension = self._get_api_based_extension( - tenant_id=tenant_id, api_based_extension_id=api_based_extension_id, session=session - ) - - # decrypt api_key - api_key = encrypter.decrypt_token(tenant_id=tenant_id, token=api_based_extension.api_key) + api_based_extension = extensions[api_based_extension_id] + api_key = api_based_extension.api_key inputs = {} for v in variables: @@ -338,10 +215,10 @@ def _convert_to_http_request_node( request_body = { "point": APIBasedExtensionPoint.APP_EXTERNAL_DATA_TOOL_QUERY, "params": { - "app_id": app_model.id, + "app_id": app_id, "tool_variable": tool_variable, "inputs": inputs, - "query": "{{#sys.query#}}" if app_model.mode == AppMode.CHAT else "", + "query": "{{#sys.query#}}" if app_mode == AppMode.CHAT else "", }, } @@ -669,32 +546,3 @@ def _append_node(self, graph: WorkflowGraph, node: _NodeType): graph["nodes"].append(node) graph["edges"].append(self._create_edge(previous_node["id"], node["id"])) return graph - - def _get_new_app_mode(self, app_model: App) -> AppMode: - """ - Get new app mode - :param app_model: App instance - :return: AppMode - """ - if app_model.mode == AppMode.COMPLETION: - return AppMode.WORKFLOW - else: - return AppMode.ADVANCED_CHAT - - def _get_api_based_extension(self, tenant_id: str, api_based_extension_id: str, session: Session): - """ - Get API Based Extension - :param tenant_id: tenant id - :param api_based_extension_id: api based extension id - :return: - """ - api_based_extension = session.scalar( - select(APIBasedExtension) - .where(APIBasedExtension.tenant_id == tenant_id, APIBasedExtension.id == api_based_extension_id) - .limit(1) - ) - - if not api_based_extension: - raise ValueError(f"API Based Extension not found, id: {api_based_extension_id}") - - return api_based_extension diff --git a/api/services/workflow_node_execution_trace_service.py b/api/services/workflow_node_execution_trace_service.py index f016afa80ea1f0..37d9380351b4d9 100644 --- a/api/services/workflow_node_execution_trace_service.py +++ b/api/services/workflow_node_execution_trace_service.py @@ -14,7 +14,9 @@ from libs.helper import to_timestamp from models.workflow import WorkflowNodeExecutionModel from repositories.api_workflow_node_execution_repository import DifyAPIWorkflowNodeExecutionRepository +from services.tools.provider_queries import ToolProviderIcons from services.variable_truncator import VariableTruncator +from services.workflow.node_execution_extras import node_execution_extras logger = logging.getLogger(__name__) @@ -61,11 +63,12 @@ def assemble_workflow_node_execution_traces( repository: DifyAPIWorkflowNodeExecutionRepository, *, session: Session, + tool_providers: ToolProviderIcons, ) -> list[WorkflowNodeExecutionTrace]: """Expand valid persisted retry attempts before each terminal execution.""" traces: list[WorkflowNodeExecutionTrace] = [] for execution in executions: - traces.extend(_expand_execution(execution, repository, session=session)) + traces.extend(_expand_execution(execution, repository, session=session, tool_providers=tool_providers)) return traces @@ -74,12 +77,16 @@ def _expand_execution( repository: DifyAPIWorkflowNodeExecutionRepository, *, session: Session, + tool_providers: ToolProviderIcons, ) -> list[WorkflowNodeExecutionTrace]: full_process_data = _load_full_process_data(execution, repository) retry_attempts = _parse_retry_attempts(execution, full_process_data) terminal_metadata = execution.execution_metadata_dict - traces = [_retry_trace(execution, attempt, terminal_metadata, session=session) for attempt in retry_attempts] - traces.append(_terminal_trace(execution, session=session)) + traces = [ + _retry_trace(execution, attempt, terminal_metadata, session=session, tool_providers=tool_providers) + for attempt in retry_attempts + ] + traces.append(_terminal_trace(execution, session=session, tool_providers=tool_providers)) return traces @@ -140,6 +147,7 @@ def _retry_trace( terminal_metadata: Mapping[str, Any], *, session: Session, + tool_providers: ToolProviderIcons, ) -> WorkflowNodeExecutionTrace: truncator = VariableTruncator.default() inputs, inputs_truncated = truncator.truncate_variable_mapping(attempt.inputs) @@ -166,7 +174,7 @@ def _retry_trace( error=attempt.error, elapsed_time=attempt.elapsed_time, execution_metadata=execution_metadata, - extras=execution.extras, + extras=node_execution_extras(execution, tool_providers=tool_providers), created_at=attempt.created_at, created_by_role=_enum_value(execution.created_by_role), created_by=execution.created_by, @@ -180,7 +188,9 @@ def _retry_trace( ) -def _terminal_trace(execution: WorkflowNodeExecutionModel, *, session: Session) -> WorkflowNodeExecutionTrace: +def _terminal_trace( + execution: WorkflowNodeExecutionModel, *, session: Session, tool_providers: ToolProviderIcons +) -> WorkflowNodeExecutionTrace: process_data = execution.process_data_dict if process_data is not None and RETRY_HISTORY_PROCESS_DATA_KEY in process_data: process_data = dict(process_data) @@ -205,7 +215,7 @@ def _terminal_trace(execution: WorkflowNodeExecutionModel, *, session: Session) error=execution.error, elapsed_time=execution.elapsed_time, execution_metadata=execution.execution_metadata_dict, - extras=execution.extras, + extras=node_execution_extras(execution, tool_providers=tool_providers), created_at=to_timestamp(execution.created_at), created_by_role=_enum_value(execution.created_by_role), created_by=execution.created_by, diff --git a/api/services/workflow_run_service.py b/api/services/workflow_run_service.py index 50c2bc52140880..4b3dcfa28cacbf 100644 --- a/api/services/workflow_run_service.py +++ b/api/services/workflow_run_service.py @@ -15,6 +15,7 @@ ) from repositories.api_workflow_node_execution_repository import DifyAPIWorkflowNodeExecutionRepository from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository +from services.tools.provider_queries import ToolProviderIcons from services.workflow_node_execution_trace_service import ( WorkflowNodeExecutionTrace, assemble_workflow_node_execution_traces, @@ -49,9 +50,11 @@ def __init__( *, workflow_runs: DifyAPISQLAlchemyWorkflowRunRepository, node_executions: DifyAPIWorkflowNodeExecutionRepository, + tool_providers: ToolProviderIcons, ) -> None: self._workflow_runs = workflow_runs self._node_executions = node_executions + self._tool_providers = tool_providers def get_paginate_advanced_chat_workflow_runs( self, @@ -203,7 +206,9 @@ def get_workflow_run_node_executions( app_id=app_id, workflow_run_id=run_id, ) - return assemble_workflow_node_execution_traces(node_executions, self._node_executions, session=db.session()) + return assemble_workflow_node_execution_traces( + node_executions, self._node_executions, session=db.session(), tool_providers=self._tool_providers + ) def get_pause_details( self, diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index 710561c9e789cf..51847dc348abb1 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -1,24 +1,17 @@ -import json -import logging import time import uuid from collections.abc import Callable, Generator, Mapping, Sequence -from dataclasses import dataclass from typing import Any, cast -from sqlalchemy import exists, select from sqlalchemy.orm import Session, sessionmaker from configs import dify_config -from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager -from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager from core.app.file_access import DatabaseFileAccessController from core.entities import PluginCredentialType from core.plugin.impl.model_runtime_factory import create_plugin_model_assembly, create_plugin_provider_manager from core.repositories import DifyCoreRepositoryFactory -from core.repositories.human_input_repository import FormCreateParams, HumanInputFormRepositoryImpl from core.trigger.constants import is_trigger_node_type -from core.workflow.human_input_adapter import adapt_human_input_node_data_for_graph, parse_human_input_delivery_methods +from core.workflow.human_input_adapter import adapt_human_input_node_data_for_graph from core.workflow.llm_environment_variable import ( LLMEnvironmentVariable, parse_llm_model_selector, @@ -26,32 +19,13 @@ should_resolve_llm_model_selector, validate_llm_environment_model_references, ) -from core.workflow.node_factory import ( - LATEST_VERSION, - get_node_type_classes_mapping, - is_start_node_type, -) -from core.workflow.node_runtime import ( - apply_dify_debug_email_recipient, -) -from core.workflow.nodes.human_input.callback import ( - DifyHITLCallback, - render_form_content_before_submission, - resolve_default_values, -) -from core.workflow.nodes.human_input.pause_reason import HumanInputRequired from core.workflow.system_variables import build_bootstrap_variables, build_system_variables, default_system_variables from core.workflow.variable_pool_initializer import add_node_inputs_to_pool, add_variables_to_pool -from core.workflow.workflow_entry import WorkflowEntry from enterprise.telemetry.draft_trace import enqueue_draft_node_execution_trace from enums import CloudPlan, DeploymentEdition -from enums.human_input import HumanInputFormKind, RecipientType -from events.app_event import app_draft_workflow_was_synced, app_published_workflow_was_updated from extensions.ext_database import db -from extensions.ext_storage import storage from factories.file_factory import build_from_mapping, build_from_mappings from graphon.entities import WorkflowNodeExecution -from graphon.entities.graph_config import NodeConfigDict from graphon.enums import ( ErrorStrategy, NodeType, @@ -69,134 +43,35 @@ from graphon.nodes.llm.entities import ModelConfig from graphon.nodes.start.entities import StartNodeData from graphon.runtime import VariablePool -from graphon.variable_loader import load_into_variable_pool from graphon.variables import VariableBase from graphon.variables.input_entities import VariableEntityType from graphon.variables.variables import Variable from libs.datetime_utils import naive_utc_now from models import Account -from models.agent import WorkflowAgentBindingType, WorkflowAgentNodeBinding -from models.human_input import HumanInputFormRecipient -from models.human_input_delivery import DeliveryChannelConfig -from models.human_input_entities import FormInputConfig, HumanInputNodeData +from models.human_input_entities import HumanInputNodeData from models.model import App, AppMode -from models.tools import WorkflowToolProvider from models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowType from repositories.factory import DifyAPIRepositoryFactory -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService +from repositories.workflow.definition_repository import WorkflowDefinitionStore from services.billing_service import BillingService +from services.credentials.query import CredentialQuery from services.errors.app import ( - IsDraftWorkflowError, TriggerNodeLimitExceededError, - WorkflowHashNotEqualError, - WorkflowNotFoundError, ) from services.system_feature_service import SystemFeatureService -from tasks.new_agent_beta_task import register_new_agent_beta_workflow_publish_after_commit - - -@dataclass(frozen=True) -class _DebugHumanInputNode: - node_id: str - title: str - node_data: HumanInputNodeData - variable_pool: VariablePool - - def render_form_content_before_submission(self) -> str: - return render_form_content_before_submission( - self.node_data, - variable_pool=self.variable_pool, - ) - - def resolve_default_values(self) -> Mapping[str, Any]: - return resolve_default_values( - self.node_data, - variable_pool=self.variable_pool, - ) - - def render_form_content_with_outputs( - self, - form_content: str, - outputs: Mapping[str, Any], - field_names: Sequence[str], - form_inputs: Sequence[FormInputConfig] | None = None, - ) -> str: - return DifyHITLCallback.render_form_content_with_outputs( - form_content=form_content, - outputs=outputs, # type: ignore[arg-type] - field_names=field_names, - form_inputs=form_inputs, - ) - - @property - def workflow_execution_id(self) -> str: - return "debug-human-input" - - @property - def node_title(self) -> str: - return self.title - - -HumanInputNode = _DebugHumanInputNode -from services.human_input_service import HumanInputService -from services.workflow.workflow_converter import WorkflowConverter -from services.workflow_ref_service import WorkflowRef -from services.workflow_version_number_service import allocate_version_number - -from .errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError -from .human_input_delivery_test_service import ( - DeliveryTestContext, - DeliveryTestEmailRecipient, - DeliveryTestError, - DeliveryTestUnsupportedError, - HumanInputDeliveryTestService, +from services.workflow.execution.adapters.chatflow.app_config_manager import AdvancedChatAppConfigManager +from services.workflow.execution.adapters.node_factory import ( + LATEST_VERSION, + get_node_type_classes_mapping, + is_start_node_type, ) -from .workflow_draft_variable_service import DraftVariableSaver, DraftVarLoader, WorkflowDraftVariableService -from .workflow_restore import apply_published_workflow_snapshot_to_draft +from services.workflow.execution.adapters.workflow.app_config_manager import WorkflowAppConfigManager +from services.workflow.execution.adapters.workflow_entry import WorkflowEntry +from services.workflow.variable_contracts import WorkflowExecutionVariables _file_access_controller = DatabaseFileAccessController() - -def _merge_environment_variable_patch( - current_variables: Sequence[VariableBase], - environment_variable_upserts: Sequence[VariableBase], - deleted_environment_variable_ids: Sequence[str], -) -> list[VariableBase]: - """Merge a per-ID environment-variable patch while preserving untouched server values.""" - upserts_by_id: dict[str, VariableBase] = {} - for variable in environment_variable_upserts: - if not variable.id: - raise ValueError("Patched environment variables require an id.") - if variable.id in upserts_by_id: - raise ValueError(f"Duplicate patched environment variable id: {variable.id}") - upserts_by_id[variable.id] = variable - - deleted_ids = set(deleted_environment_variable_ids) - if len(deleted_ids) != len(deleted_environment_variable_ids): - raise ValueError("Deleted environment variable ids must be unique.") - if "" in deleted_ids: - raise ValueError("Deleted environment variable ids must not be empty.") - if conflicting_ids := deleted_ids.intersection(upserts_by_id): - conflicting_id = min(conflicting_ids) - raise ValueError(f"Environment variable cannot be upserted and deleted in the same patch: {conflicting_id}") - - existing_ids: set[str] = set() - merged_variables: list[VariableBase] = [] - for variable in current_variables: - variable_id = variable.id - if variable_id: - existing_ids.add(variable_id) - if variable_id in deleted_ids: - continue - merged_variables.append(upserts_by_id.get(variable_id, variable)) - - merged_variables.extend( - variable for variable_id, variable in upserts_by_id.items() if variable_id not in existing_ids - ) - names = [variable.name for variable in merged_variables] - if len(set(names)) != len(names): - raise ValueError("Environment variable names must be unique.") - return merged_variables +from services.workflow.execution.ports import WorkflowRuntime class WorkflowService: @@ -204,10 +79,17 @@ class WorkflowService: Workflow Service """ - def __init__(self, session_maker: sessionmaker | None = None): + def __init__( + self, + session_maker: sessionmaker | None = None, + *, + runtime: WorkflowRuntime | None = None, + ): """Initialize WorkflowService with repository dependencies.""" if session_maker is None: session_maker = sessionmaker(bind=db.engine, expire_on_commit=False) + self._runtime = runtime + self._sessions = session_maker self._node_execution_service_repo = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( session_maker ) @@ -231,53 +113,10 @@ def get_node_last_run(self, app_model: App, workflow: Workflow, node_id: str) -> node_id=node_id, ) - def is_workflow_exist(self, app_model: App, *, session: Session) -> bool: - stmt = select( - exists().where( - Workflow.tenant_id == app_model.tenant_id, - Workflow.app_id == app_model.id, - Workflow.version == Workflow.VERSION_DRAFT, - ) - ) - return session.execute(stmt).scalar_one() - def get_draft_workflow( self, app_model: App, workflow_id: str | None = None, *, session: Session ) -> Workflow | None: - """ - Get draft workflow - - Reuses the caller's active session so workflow reads stay in the same - transaction as the surrounding request or task. - """ - if workflow_id: - return self.get_published_workflow_by_id(app_model, workflow_id, session=session) - # fetch draft workflow by app_model - workflow = session.scalar( - select(Workflow) - .where( - Workflow.tenant_id == app_model.tenant_id, - Workflow.app_id == app_model.id, - Workflow.version == Workflow.VERSION_DRAFT, - ) - .limit(1) - ) - - # return draft workflow - return workflow - - def _get_draft_workflow_for_update(self, app_model: App, *, session: Session) -> Workflow | None: - """Return the app draft while holding its row lock for the caller's transaction.""" - return session.scalar( - select(Workflow) - .where( - Workflow.tenant_id == app_model.tenant_id, - Workflow.app_id == app_model.id, - Workflow.version == Workflow.VERSION_DRAFT, - ) - .limit(1) - .with_for_update() - ) + return WorkflowDefinitionStore.get_draft_workflow(app_model, workflow_id, session=session) def get_published_workflow_by_id( self, @@ -287,411 +126,15 @@ def get_published_workflow_by_id( session: Session, for_update: bool = False, ) -> Workflow | None: - """Fetch a published workflow by ID in the caller's transaction. - - With ``for_update=True``, the source version stays locked until that - transaction ends. Restore uses the lock while copying Agent bindings so - a concurrent delete cannot release the same owner. - """ - stmt = ( - select(Workflow) - .where( - Workflow.tenant_id == app_model.tenant_id, - Workflow.app_id == app_model.id, - Workflow.id == workflow_id, - ) - .limit(1) + return WorkflowDefinitionStore.get_published_workflow_by_id( + app_model, workflow_id, session=session, for_update=for_update ) - if for_update: - stmt = stmt.with_for_update() - workflow = session.scalar(stmt) - if not workflow: - return None - if workflow.version == Workflow.VERSION_DRAFT: - raise IsDraftWorkflowError( - f"Cannot use draft workflow version. Workflow ID: {workflow_id}. " - f"Please use a published workflow version or leave workflow_id empty." - ) - return workflow def get_published_workflow(self, app_model: App, *, session: Session) -> Workflow | None: - """ - Get published workflow - - Reuses the caller's active session so workflow reads stay in the same - transaction as the surrounding request or task. - """ - - if not app_model.workflow_id: - return None - - workflow = session.scalar( - select(Workflow) - .where( - Workflow.tenant_id == app_model.tenant_id, - Workflow.app_id == app_model.id, - Workflow.id == app_model.workflow_id, - ) - .limit(1) - ) - - return workflow - - def get_tenant_app_maintainers( - self, app_ids: Sequence[str], tenant_id: str, *, session: Session - ) -> dict[str, str | None]: - """Return requested normal apps and their maintainers within a tenant.""" - if not app_ids: - return {} - - stmt = select(App.id, App.maintainer).where( - App.id.in_(app_ids), App.tenant_id == tenant_id, App.status == "normal" - ) - return {str(app_id): maintainer for app_id, maintainer in session.execute(stmt)} - - def get_all_published_workflow( - self, - *, - session: Session, - app_model: App, - page: int, - limit: int, - user_id: str | None, - named_only: bool = False, - ) -> tuple[Sequence[Workflow], bool]: - """ - Get published workflow with pagination - """ - if not app_model.workflow_id: - return [], False - - stmt = ( - select(Workflow) - .where(Workflow.app_id == app_model.id) - # The draft leads the list; its `created_at` is the app's creation time, so it would - # otherwise sort last. Published versions then order by publish time: `version` is a - # stringified timestamp whose microseconds are omitted when zero, so ordering by it - # misplaces versions across second boundaries, and `version_number` is NULL for - # versions published before numbering was introduced. - .order_by( - (Workflow.version == Workflow.VERSION_DRAFT).desc(), - Workflow.created_at.desc(), - Workflow.id.desc(), - ) - .limit(limit + 1) - .offset((page - 1) * limit) - ) - - if user_id: - stmt = stmt.where(Workflow.created_by == user_id) - - if named_only: - stmt = stmt.where(Workflow.marked_name != "") - - workflows = session.scalars(stmt).all() - - has_more = len(workflows) > limit - if has_more: - workflows = workflows[:-1] - - return workflows, has_more - - def sync_draft_workflow( - self, - *, - app_model: App, - graph: dict[str, Any], - features: dict[str, Any], - unique_hash: str | None, - account: Account, - environment_variables: Sequence[VariableBase], - conversation_variables: Sequence[VariableBase], - session: Session, - environment_variable_upserts: Sequence[VariableBase] | None = None, - deleted_environment_variable_ids: Sequence[str] = (), - preserve_environment_variables: bool = False, - commit: bool = True, - sync_agent_bindings: bool = True, - graph_only: bool = False, - ) -> Workflow: - """ - Sync draft workflow. - - DSL import disables the intermediate commit and Agent binding sync so - portable package references can be materialized atomically after the - draft workflow has received its target-workspace id. - - Existing drafts are row-locked before the hash check. Collaborative - graph-only saves preserve independently persisted draft fields, while - an explicit per-ID environment patch is merged with the graph. - - :raises WorkflowHashNotEqualError - """ - if environment_variable_upserts is None and deleted_environment_variable_ids: - raise ValueError("Deleted environment variable ids require an environment variable patch.") - - # fetch draft workflow by app_model - workflow = self._get_draft_workflow_for_update(app_model=app_model, session=session) - - if workflow and workflow.unique_hash != unique_hash: - raise WorkflowHashNotEqualError() - - # Collaboration persists features and variables through dedicated endpoints. A graph save - # must not overwrite those newer database values with another collaborator's stale cache. - if not graph_only or not workflow: - self.validate_features_structure(app_model=app_model, features=features) - - # validate graph structure - self.validate_graph_structure(graph=graph) - - # create draft workflow if not found - if not workflow: - initial_environment_variables = ( - _merge_environment_variable_patch( - environment_variables, - environment_variable_upserts, - deleted_environment_variable_ids, - ) - if environment_variable_upserts is not None - else list(environment_variables) - ) - workflow = Workflow( - tenant_id=app_model.tenant_id, - app_id=app_model.id, - type=WorkflowType.from_app_mode(app_model.mode).value, - version=Workflow.VERSION_DRAFT, - graph=json.dumps(graph), - features=json.dumps(features), - created_by=account.id, - environment_variables=initial_environment_variables, - conversation_variables=conversation_variables, - ) - session.add(workflow) - # update draft workflow if found - else: - workflow.graph = json.dumps(graph) - workflow.updated_by = account.id - workflow.updated_at = naive_utc_now() - if not graph_only: - workflow.features = json.dumps(features) - workflow.conversation_variables = conversation_variables - if environment_variable_upserts is not None: - workflow.environment_variables = _merge_environment_variable_patch( - workflow.environment_variables, - environment_variable_upserts, - deleted_environment_variable_ids, - ) - elif not graph_only and not preserve_environment_variables: - workflow.environment_variables = environment_variables - - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - session.flush() - retirement_candidates: set[str] = set() - if sync_agent_bindings: - retirement_candidates = WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, - account_id=account.id, - ) - WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync( - session=session, - draft_workflow=workflow, - ) - - # commit db session changes - if commit: - session.commit() - WorkflowAgentRetirementService.retire_unowned( - tenant_id=app_model.tenant_id, - agent_ids=retirement_candidates, - account_id=account.id, - ) - - # trigger app workflow events - if commit: - app_draft_workflow_was_synced.send(app_model, synced_draft_workflow=workflow) - - # return draft workflow - return workflow - - def update_draft_workflow_environment_variables( - self, - *, - app_model: App, - environment_variables: Sequence[VariableBase], - account: Account, - session: Session, - ) -> None: - """Replace every environment variable on a draft workflow and commit the transaction.""" - # fetch draft workflow by app_model - workflow = self.get_draft_workflow(app_model=app_model, session=session) - - if not workflow: - raise ValueError("No draft workflow found.") - - workflow.environment_variables = environment_variables - workflow.updated_by = account.id - workflow.updated_at = naive_utc_now() - - # commit db session changes - session.commit() - - def patch_draft_workflow_environment_variables( - self, - *, - app_model: App, - environment_variables: Sequence[VariableBase], - deleted_environment_variable_ids: Sequence[str], - account: Account, - session: Session, - ) -> None: - """Atomically merge per-ID environment-variable upserts and deletions into a draft workflow. - - The draft row is locked before reading its current variables so concurrent patches preserve - variables they do not touch. Existing variables keep their order and new variables are appended. - The transaction is committed before this method returns. - """ - workflow = self._get_draft_workflow_for_update(app_model=app_model, session=session) - if not workflow: - raise ValueError("No draft workflow found.") - - workflow.environment_variables = _merge_environment_variable_patch( - workflow.environment_variables, - environment_variables, - deleted_environment_variable_ids, - ) - workflow.updated_by = account.id - workflow.updated_at = naive_utc_now() - session.commit() - - def update_draft_workflow_conversation_variables( - self, - *, - app_model: App, - conversation_variables: Sequence[VariableBase], - account: Account, - session: Session, - ): - """ - Update draft workflow conversation variables - """ - # fetch draft workflow by app_model - workflow = self.get_draft_workflow(app_model=app_model, session=session) - - if not workflow: - raise ValueError("No draft workflow found.") - - workflow.conversation_variables = conversation_variables - workflow.updated_by = account.id - workflow.updated_at = naive_utc_now() - - # commit db session changes - session.commit() - - def update_draft_workflow_features( - self, - *, - app_model: App, - features: dict, - account: Account, - session: Session, - ): - """ - Update draft workflow features - """ - # fetch draft workflow by app_model - workflow = self.get_draft_workflow(app_model=app_model, session=session) - - if not workflow: - raise ValueError("No draft workflow found.") - - # validate features structure - self.validate_features_structure(app_model=app_model, features=features) - - workflow.features = json.dumps(features) - workflow.updated_by = account.id - workflow.updated_at = naive_utc_now() - - # commit db session changes - session.commit() - - def restore_published_workflow_to_draft( - self, - *, - app_model: App, - workflow_id: str, - account: Account, - session: Session, - ) -> Workflow: - """Restore a published workflow snapshot into the draft workflow. - - Secret environment variables are copied server-side from the selected - published workflow so the normal draft sync flow stays stateless. - """ - source_workflow = self.get_published_workflow_by_id( - app_model=app_model, - workflow_id=workflow_id, - session=session, - for_update=True, - ) - if not source_workflow: - raise WorkflowNotFoundError("Workflow not found.") - - self.validate_features_structure(app_model=app_model, features=source_workflow.normalized_features_dict) - self.validate_graph_structure(graph=source_workflow.graph_dict) - - draft_workflow = self.get_draft_workflow(app_model=app_model, session=session) - draft_workflow, is_new_draft = apply_published_workflow_snapshot_to_draft( - tenant_id=app_model.tenant_id, - app_id=app_model.id, - source_workflow=source_workflow, - draft_workflow=draft_workflow, - account=account, - updated_at_factory=naive_utc_now, - ) - - if is_new_draft: - session.add(draft_workflow) - - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - session.flush() - retirement_candidates = WorkflowAgentPublishService.restore_agent_node_bindings_to_draft( - session=session, - source_workflow=source_workflow, - draft_workflow=draft_workflow, - account_id=account.id, - ) - - session.commit() - WorkflowAgentRetirementService.retire_unowned( - tenant_id=app_model.tenant_id, - agent_ids=retirement_candidates, - account_id=account.id, - ) - app_draft_workflow_was_synced.send(app_model, synced_draft_workflow=draft_workflow) - - return draft_workflow - - def publish_workflow( - self, - *, - session: Session, - app_model: App, - account: Account, - marked_name: str = "", - marked_comment: str = "", - ) -> Workflow: - draft_workflow_stmt = select(Workflow).where( - Workflow.tenant_id == app_model.tenant_id, - Workflow.app_id == app_model.id, - Workflow.version == Workflow.VERSION_DRAFT, - ) - draft_workflow = session.scalar(draft_workflow_stmt) - if not draft_workflow: - raise ValueError("No valid workflow found.") + return WorkflowDefinitionStore.get_published_workflow(app_model, session=session) + def validate_publication(self, app_model: App, draft_workflow: Workflow, *, credentials: CredentialQuery) -> None: + """Validate an already loaded revision without changing workflow state.""" validate_llm_environment_model_references( graph=draft_workflow.graph_dict, environment_variables=draft_workflow.environment_variables, @@ -699,18 +142,11 @@ def publish_workflow( # Validate credentials before publishing, for credential policy check if SystemFeatureService.is_plugin_manager_enabled(): - self._validate_workflow_credentials(draft_workflow, session=session) + self._validate_workflow_credentials(draft_workflow, credentials=credentials) # validate graph structure self.validate_graph_structure(graph=draft_workflow.graph_dict) - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - WorkflowAgentPublishService.validate_agent_nodes_for_publish( - session=session, - draft_workflow=draft_workflow, - ) - # billing check if dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD: limit_info = BillingService.get_info(app_model.tenant_id) @@ -726,44 +162,7 @@ def publish_workflow( if trigger_node_count > 2: raise TriggerNodeLimitExceededError(count=trigger_node_count, limit=2) - # create new workflow - workflow = Workflow.new( - tenant_id=app_model.tenant_id, - app_id=app_model.id, - type=draft_workflow.type, - version=Workflow.version_from_datetime(naive_utc_now()), - version_number=allocate_version_number(session=session, app_id=app_model.id), - graph=draft_workflow.graph, - created_by=account.id, - environment_variables=draft_workflow.environment_variables, - conversation_variables=draft_workflow.conversation_variables, - marked_name=marked_name, - marked_comment=marked_comment, - rag_pipeline_variables=draft_workflow.rag_pipeline_variables, - features=draft_workflow.features, - ) - - # commit db session changes - session.add(workflow) - has_inline_agent = WorkflowAgentPublishService.copy_agent_node_bindings_to_published( - session=session, - draft_workflow=draft_workflow, - published_workflow=workflow, - ) - if has_inline_agent: - register_new_agent_beta_workflow_publish_after_commit( - session=session, - published_workflow_id=workflow.id, - published_at=workflow.created_at, - ) - - # trigger app workflow events - app_published_workflow_was_updated.send(app_model, published_workflow=workflow) - - # return new workflow - return workflow - - def _validate_workflow_credentials(self, workflow: Workflow, *, session: Session) -> None: + def _validate_workflow_credentials(self, workflow: Workflow, *, credentials: CredentialQuery) -> None: """ Validate all credentials in workflow nodes before publishing. @@ -803,7 +202,7 @@ def _validate_workflow_credentials(self, workflow: Workflow, *, session: Session ) else: # Check default workspace credential for this provider - self._check_default_tool_credential(workflow.tenant_id, provider, session=session) + self._check_default_tool_credential(workflow.tenant_id, provider, credentials=credentials) elif node_type == "agent": agent_params = node_data.get("agent_parameters", {}) @@ -817,7 +216,7 @@ def _validate_workflow_credentials(self, workflow: Workflow, *, session: Session # Validate load balancing credentials for agent model if load balancing is enabled agent_model_node_data = {"model": model_config} self._validate_load_balancing_credentials( - workflow, agent_model_node_data, node_id, session=session + workflow, agent_model_node_data, node_id, credentials=credentials ) # Validate agent tools @@ -832,7 +231,9 @@ def _validate_workflow_credentials(self, workflow: Workflow, *, session: Session check_credential_policy_compliance(credential_id, provider, PluginCredentialType.TOOL) else: - self._check_default_tool_credential(workflow.tenant_id, provider, session=session) + self._check_default_tool_credential( + workflow.tenant_id, provider, credentials=credentials + ) elif node_type in ["llm", "knowledge_retrieval", "parameter_extractor", "question_classifier"]: validation_node_data = node_data @@ -859,7 +260,7 @@ def _validate_workflow_credentials(self, workflow: Workflow, *, session: Session self._validate_llm_model_config(workflow.tenant_id, provider, model_name) # Validate load balancing credentials if load balancing is enabled self._validate_load_balancing_credentials( - workflow, validation_node_data, node_id, session=session + workflow, validation_node_data, node_id, credentials=credentials ) else: raise ValueError(f"Node {node_id} ({node_type}): Missing provider or model configuration") @@ -923,7 +324,7 @@ def _validate_llm_model_config(self, tenant_id: str, provider: str, model_name: f"Failed to validate LLM model configuration (provider: {provider}, model: {model_name}): {str(e)}" ) - def _check_default_tool_credential(self, tenant_id: str, provider: str, *, session: Session) -> None: + def _check_default_tool_credential(self, tenant_id: str, provider: str, *, credentials: CredentialQuery) -> None: """ Check credential policy compliance for the default workspace credential of a tool provider. @@ -935,29 +336,15 @@ def _check_default_tool_credential(self, tenant_id: str, provider: str, *, sessi :raises ValueError: If no default credential exists or if it fails policy compliance """ try: - from models.tools import BuiltinToolProvider - - # Use the same fallback logic as runtime: get the first available credential - # ordered by is_default DESC, created_at ASC (same as tool_manager.py) - default_provider = session.scalar( - select(BuiltinToolProvider) - .where( - BuiltinToolProvider.tenant_id == tenant_id, - BuiltinToolProvider.provider == provider, - ) - .order_by(BuiltinToolProvider.is_default.desc(), BuiltinToolProvider.created_at.asc()) - .limit(1) - ) - - if not default_provider: - # plugin does not require credentials, skip + credential_id = credentials.default_tool_credential_id(workspace_id=tenant_id, provider=provider) + if credential_id is None: return # Check credential policy compliance using the default credential ID from core.helper.credential_utils import check_credential_policy_compliance check_credential_policy_compliance( - credential_id=default_provider.id, + credential_id=credential_id, provider=provider, credential_type=PluginCredentialType.TOOL, check_existence=False, @@ -967,7 +354,7 @@ def _check_default_tool_credential(self, tenant_id: str, provider: str, *, sessi raise ValueError(f"Failed to validate default credential for tool provider {provider}: {str(e)}") def _validate_load_balancing_credentials( - self, workflow: Workflow, node_data: dict[str, Any], node_id: str, *, session: Session + self, workflow: Workflow, node_data: dict[str, Any], node_id: str, *, credentials: CredentialQuery ) -> None: """ Validate load balancing credentials for a workflow node. @@ -987,19 +374,14 @@ def _validate_load_balancing_credentials( # Check if this model has load balancing enabled if self._is_load_balancing_enabled(workflow.tenant_id, provider, model_name): - # Get all load balancing configurations for this model - load_balancing_configs = self._get_load_balancing_configs( - workflow.tenant_id, provider, model_name, session=session + credential_ids = credentials.list_model_load_balancing_credential_ids( + workspace_id=workflow.tenant_id, provider=provider, model=model_name ) - # Validate each load balancing configuration try: - for config in load_balancing_configs: - if config.get("credential_id"): - from core.helper.credential_utils import check_credential_policy_compliance + from core.helper.credential_utils import check_credential_policy_compliance - check_credential_policy_compliance( - config["credential_id"], provider, PluginCredentialType.MODEL - ) + for credential_id in credential_ids: + check_credential_policy_compliance(credential_id, provider, PluginCredentialType.MODEL) except Exception as e: raise ValueError(f"Invalid load balancing credentials for {provider}/{model_name}: {str(e)}") @@ -1034,47 +416,6 @@ def _is_load_balancing_enabled(self, tenant_id: str, provider: str, model_name: # If we can't determine the status, assume load balancing is not enabled return False - def _get_load_balancing_configs( - self, tenant_id: str, provider: str, model_name: str, *, session: Session - ) -> list[dict[str, Any]]: - """ - Get all load balancing configurations for a model. - - :param tenant_id: The tenant ID - :param provider: The provider name - :param model_name: The model name - :return: List of load balancing configuration dictionaries - """ - try: - from services.model_load_balancing_service import ModelLoadBalancingService - - model_load_balancing_service = ModelLoadBalancingService() - _, configs = model_load_balancing_service.get_load_balancing_configs( - tenant_id=tenant_id, - provider=provider, - model=model_name, - model_type="llm", # Load balancing is primarily used for LLM models - session=session, - config_from="predefined-model", # Check both predefined and custom models - ) - - _, custom_configs = model_load_balancing_service.get_load_balancing_configs( - tenant_id=tenant_id, - provider=provider, - model=model_name, - model_type="llm", - session=session, - config_from="custom-model", - ) - all_configs = cast(list[dict[str, Any]], configs) + cast(list[dict[str, Any]], custom_configs) - - return [config for config in all_configs if config.get("credential_id")] - - except Exception: - # If we can't get the configurations, return empty list - # This will prevent validation errors from breaking the workflow - return [] - def get_default_block_configs(self) -> Sequence[Mapping[str, object]]: """ Get default block configs @@ -1145,45 +486,41 @@ def run_draft_workflow_node( account: Account, query: str = "", files: Sequence[File] | None = None, + *, + variables: WorkflowExecutionVariables, ) -> WorkflowNodeExecutionModel: """ Run draft workflow node """ files = files or [] - with Session(bind=db.engine, expire_on_commit=False) as session, session.begin(): - draft_var_srv = WorkflowDraftVariableService(session) - draft_var_srv.prefill_conversation_variable_default_values(draft_workflow, user_id=account.id) - node_config = draft_workflow.get_node_config_by_id(node_id) node_type = Workflow.get_node_type_from_node_config(node_config) node_data = node_config["data"] if is_start_node_type(node_type): - with Session(bind=db.engine) as session, session.begin(): - draft_var_srv = WorkflowDraftVariableService(session) - conversation_id = draft_var_srv.get_or_create_conversation( - account_id=account.id, - app=app_model, - workflow=draft_workflow, - ) - if node_type == BuiltinNodeTypes.START: - start_data = StartNodeData.model_validate(node_data, from_attributes=True) - user_inputs = _rebuild_file_for_user_inputs_in_start_node( - tenant_id=draft_workflow.tenant_id, start_node_data=start_data, user_inputs=user_inputs - ) - # init variable pool - variable_pool = _setup_variable_pool( - query=query, - files=files or [], - user_id=account.id, - user_inputs=user_inputs, - workflow=draft_workflow, - node_id=node_id, - # NOTE(QuantumGhost): We rely on `DraftVarLoader` to load conversation variables. - conversation_variables=[], - node_type=node_type, - conversation_id=conversation_id, + conversation_id = variables.get_or_create_conversation( + account_id=account.id, + app=app_model, + workflow=draft_workflow, + ) + if node_type == BuiltinNodeTypes.START: + start_data = StartNodeData.model_validate(node_data, from_attributes=True) + user_inputs = _rebuild_file_for_user_inputs_in_start_node( + tenant_id=draft_workflow.tenant_id, start_node_data=start_data, user_inputs=user_inputs ) + # init variable pool + variable_pool = _setup_variable_pool( + query=query, + files=files or [], + user_id=account.id, + user_inputs=user_inputs, + workflow=draft_workflow, + node_id=node_id, + # NOTE(QuantumGhost): We rely on `DraftVarLoader` to load conversation variables. + conversation_variables=[], + node_type=node_type, + conversation_id=conversation_id, + ) else: variable_pool = VariablePool() @@ -1195,12 +532,7 @@ def run_draft_workflow_node( ), ) - variable_loader = DraftVarLoader( - engine=db.engine, - app_id=app_model.id, - tenant_id=app_model.tenant_id, - user_id=account.id, - ) + variable_loader = variables.workflow_loader(draft_workflow, account.id) enclosing_node_type_and_id = draft_workflow.get_enclosing_node_type_and_id(node_config) if enclosing_node_type_and_id: @@ -1208,7 +540,12 @@ def run_draft_workflow_node( else: enclosing_node_id = None + if self._runtime is None: + raise ValueError("Workflow execution dependencies are required") run = WorkflowEntry.single_step_run( + workflow_runtime=self._runtime, + agent_binding_resolver=self._runtime.agent_bindings, + draft_variable_saver=variables.saver_factory, workflow=draft_workflow, node_id=node_id, user_inputs=user_inputs, @@ -1230,7 +567,7 @@ def run_draft_workflow_node( # Create repository and save the node execution repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( - session_factory=db.engine, + session_factory=self._sessions, tenant_id=app_model.tenant_id, user=account, app_id=app_model.id, @@ -1238,25 +575,22 @@ def run_draft_workflow_node( ) repository.save(node_execution) - workflow_node_execution = self._node_execution_service_repo.get_execution_by_id(node_execution.id) + workflow_node_execution = self._node_execution_service_repo.get_execution_by_id( + node_execution.id, tenant_id=app_model.tenant_id + ) if workflow_node_execution is None: raise ValueError(f"WorkflowNodeExecution with id {node_execution.id} not found after saving") - with sessionmaker(db.engine).begin() as session: - outputs = workflow_node_execution.load_full_outputs(session, storage) - - with sessionmaker(bind=db.engine).begin() as session: - draft_var_saver = DraftVariableSaver( - session=session, - tenant_id=app_model.tenant_id, - app_id=app_model.id, - node_id=workflow_node_execution.node_id, - node_type=workflow_node_execution.node_type, - enclosing_node_id=enclosing_node_id, - node_execution_id=node_execution.id, - user=account, - ) - draft_var_saver.save(process_data=node_execution.process_data, outputs=outputs) + outputs = variables.load_execution_outputs(workflow_node_execution) + + draft_var_saver = variables.saver_factory(app_model.tenant_id, account)( + app_id=app_model.id, + node_id=workflow_node_execution.node_id, + node_type=workflow_node_execution.node_type, + enclosing_node_id=enclosing_node_id, + node_execution_id=node_execution.id, + ) + draft_var_saver.save(process_data=node_execution.process_data, outputs=outputs) enqueue_draft_node_execution_trace( execution=workflow_node_execution, @@ -1267,351 +601,6 @@ def run_draft_workflow_node( return workflow_node_execution - def get_human_input_form_preview( - self, - *, - app_model: App, - account: Account, - node_id: str, - inputs: Mapping[str, Any] | None = None, - session: Session, - ) -> Mapping[str, Any]: - """ - Build a human input form preview for a draft workflow. - - Args: - app_model: Target application model. - account: Current account. - node_id: Human input node ID. - inputs: Values used to fill missing upstream variables referenced in form_content. - """ - draft_workflow = self.get_draft_workflow(app_model=app_model, session=session) - if not draft_workflow: - raise ValueError("Workflow not initialized") - - node_config = draft_workflow.get_node_config_by_id(node_id) - node_type = Workflow.get_node_type_from_node_config(node_config) - if node_type != BuiltinNodeTypes.HUMAN_INPUT: - raise ValueError("Node type must be human-input.") - - # inputs: values used to fill missing upstream variables referenced in form_content. - variable_pool = self._build_human_input_variable_pool( - app_model=app_model, - workflow=draft_workflow, - node_config=node_config, - manual_inputs=inputs or {}, - user_id=account.id, - ) - node = self._build_human_input_node_for_debugging( - workflow=draft_workflow, - account=account, - node_config=node_config, - variable_pool=variable_pool, - ) - - rendered_content = node.render_form_content_before_submission() - resolved_default_values = node.resolve_default_values() - node_data = node.node_data - human_input_required = HumanInputRequired( - form_id=node_id, - form_content=rendered_content, - inputs=node_data.inputs, - actions=node_data.user_actions, - node_id=node_id, - node_title=node.title, - resolved_default_values=resolved_default_values, - ) - return human_input_required.model_dump(mode="json") - - def submit_human_input_form_preview( - self, - *, - app_model: App, - account: Account, - node_id: str, - form_inputs: Mapping[str, Any], - inputs: Mapping[str, Any] | None = None, - action: str, - session: Session, - ) -> Mapping[str, Any]: - """ - Submit a human input form preview for a draft workflow. - - Args: - app_model: Target application model. - account: Current account. - node_id: Human input node ID. - form_inputs: Values the user provides for the form's own fields. - inputs: Values used to fill missing upstream variables referenced in form_content. - action: Selected action ID. - """ - draft_workflow = self.get_draft_workflow(app_model=app_model, session=session) - if not draft_workflow: - raise ValueError("Workflow not initialized") - - node_config = draft_workflow.get_node_config_by_id(node_id) - node_type = Workflow.get_node_type_from_node_config(node_config) - if node_type != BuiltinNodeTypes.HUMAN_INPUT: - raise ValueError("Node type must be human-input.") - - # inputs: values used to fill missing upstream variables referenced in form_content. - # form_inputs: values the user provides for the form's own fields. - variable_pool = self._build_human_input_variable_pool( - app_model=app_model, - workflow=draft_workflow, - node_config=node_config, - manual_inputs=inputs or {}, - user_id=account.id, - ) - node = self._build_human_input_node_for_debugging( - workflow=draft_workflow, - account=account, - node_config=node_config, - variable_pool=variable_pool, - ) - node_data = node.node_data - - human_input_service = HumanInputService(session_factory=sessionmaker(db.engine)) - normalized_form_inputs = human_input_service.validate_and_normalize_submission( - tenant_id=app_model.tenant_id, - form_definition=node_data, - selected_action_id=action, - form_data=form_inputs, - ) - - rendered_content = node.render_form_content_before_submission() - selected_action = next( - (user_action for user_action in node_data.user_actions if user_action.id == action), - None, - ) - outputs: dict[str, Any] = dict(normalized_form_inputs) - outputs["__action_id"] = action - outputs["__action_value"] = selected_action.title if selected_action else "" - outputs["__rendered_content"] = node.render_form_content_with_outputs( - rendered_content, - outputs, - node_data.outputs_field_names(), - node_data.inputs, - ) - - enclosing_node_type_and_id = draft_workflow.get_enclosing_node_type_and_id(node_config) - enclosing_node_id = enclosing_node_type_and_id[1] if enclosing_node_type_and_id else None - with sessionmaker(bind=db.engine).begin() as session: - draft_var_saver = DraftVariableSaver( - session=session, - tenant_id=app_model.tenant_id, - app_id=app_model.id, - node_id=node_id, - node_type=BuiltinNodeTypes.HUMAN_INPUT, - node_execution_id=str(uuid.uuid4()), - user=account, - enclosing_node_id=enclosing_node_id, - ) - draft_var_saver.save(outputs=outputs, process_data={}) - - return outputs - - def test_human_input_delivery( - self, - *, - app_model: App, - account: Account, - node_id: str, - delivery_method_id: str, - inputs: Mapping[str, Any] | None = None, - session: Session, - ) -> None: - draft_workflow = self.get_draft_workflow(app_model=app_model, session=session) - if not draft_workflow: - raise ValueError("Workflow not initialized") - - node_config = draft_workflow.get_node_config_by_id(node_id) - node_type = Workflow.get_node_type_from_node_config(node_config) - if node_type != BuiltinNodeTypes.HUMAN_INPUT: - raise ValueError("Node type must be human-input.") - - node_data = HumanInputNodeData.model_validate( - adapt_human_input_node_data_for_graph(node_config["data"]), - from_attributes=True, - ) - delivery_method = self._resolve_human_input_delivery_method( - node_data=node_data, - delivery_method_id=delivery_method_id, - ) - if delivery_method is None: - raise ValueError("Delivery method not found.") - delivery_method = apply_dify_debug_email_recipient( - delivery_method, - enabled=True, - actor_id=account.id, - ) - - variable_pool = self._build_human_input_variable_pool( - app_model=app_model, - workflow=draft_workflow, - node_config=node_config, - manual_inputs=inputs or {}, - user_id=account.id, - ) - node = self._build_human_input_node_for_debugging( - workflow=draft_workflow, - account=account, - node_config=node_config, - variable_pool=variable_pool, - ) - rendered_content = node.render_form_content_before_submission() - resolved_default_values = node.resolve_default_values() - form_id, recipients = self._create_human_input_delivery_test_form( - app_model=app_model, - node_id=node_id, - node_data=node_data, - delivery_method=delivery_method, - rendered_content=rendered_content, - resolved_default_values=resolved_default_values, - ) - test_service = HumanInputDeliveryTestService() - context = DeliveryTestContext( - tenant_id=app_model.tenant_id, - app_id=app_model.id, - node_id=node_id, - node_title=node_data.title, - rendered_content=rendered_content, - template_vars={"form_id": form_id}, - recipients=recipients, - variable_pool=variable_pool, - ) - try: - test_service.send_test(context=context, method=delivery_method) - except DeliveryTestUnsupportedError as exc: - raise ValueError("Delivery method does not support test send.") from exc - except DeliveryTestError as exc: - raise ValueError(str(exc)) from exc - - @staticmethod - def _resolve_human_input_delivery_method( - *, - node_data: HumanInputNodeData, - delivery_method_id: str, - ) -> DeliveryChannelConfig | None: - for method in parse_human_input_delivery_methods(node_data): - if str(method.id) == delivery_method_id: - return method - return None - - def _create_human_input_delivery_test_form( - self, - *, - app_model: App, - node_id: str, - node_data: HumanInputNodeData, - delivery_method: DeliveryChannelConfig, - rendered_content: str, - resolved_default_values: Mapping[str, Any], - ) -> tuple[str, list[DeliveryTestEmailRecipient]]: - repo = HumanInputFormRepositoryImpl(tenant_id=app_model.tenant_id, app_id=app_model.id) - params = FormCreateParams( - workflow_execution_id=None, - node_id=node_id, - form_config=node_data, - rendered_content=rendered_content, - delivery_methods=[delivery_method], - display_in_ui=False, - resolved_default_values=resolved_default_values, - form_kind=HumanInputFormKind.DELIVERY_TEST, - ) - form_entity = repo.create_form(params) - return form_entity.id, self._load_email_recipients(form_entity.id) - - @staticmethod - def _load_email_recipients(form_id: str) -> list[DeliveryTestEmailRecipient]: - logger = logging.getLogger(__name__) - - with Session(bind=db.engine) as session: - recipients = session.scalars( - select(HumanInputFormRecipient).where(HumanInputFormRecipient.form_id == form_id) - ).all() - recipients_data: list[DeliveryTestEmailRecipient] = [] - for recipient in recipients: - if recipient.recipient_type not in {RecipientType.EMAIL_MEMBER, RecipientType.EMAIL_EXTERNAL}: - continue - if not recipient.access_token: - continue - try: - payload = json.loads(recipient.recipient_payload) - except (json.JSONDecodeError, ValueError): - logger.exception("Failed to parse human input recipient payload for delivery test.") - continue - email = payload.get("email") - if email: - recipients_data.append(DeliveryTestEmailRecipient(email=email, form_token=recipient.access_token)) - return recipients_data - - def _build_human_input_node_for_debugging( - self, - *, - workflow: Workflow, - account: Account, - node_config: NodeConfigDict, - variable_pool: VariablePool, - ) -> _DebugHumanInputNode: - _ = workflow, account - node_data = HumanInputNodeData.model_validate(adapt_human_input_node_data_for_graph(node_config["data"])) - return HumanInputNode( - node_id=node_config["id"], - title=node_data.title, - node_data=node_data, - variable_pool=variable_pool, - ) - - def _build_human_input_variable_pool( - self, - *, - app_model: App, - workflow: Workflow, - node_config: NodeConfigDict, - manual_inputs: Mapping[str, Any], - user_id: str, - ) -> VariablePool: - with Session(bind=db.engine, expire_on_commit=False) as session, session.begin(): - draft_var_srv = WorkflowDraftVariableService(session) - draft_var_srv.prefill_conversation_variable_default_values(workflow, user_id=user_id) - - variable_pool = VariablePool() - add_variables_to_pool( - variable_pool, - build_bootstrap_variables( - system_variables=default_system_variables(), - environment_variables=workflow.environment_variables, - ), - ) - - variable_loader = DraftVarLoader( - engine=db.engine, - app_id=app_model.id, - tenant_id=app_model.tenant_id, - user_id=user_id, - ) - human_input_node_data = HumanInputNodeData.model_validate( - adapt_human_input_node_data_for_graph(node_config["data"]) - ) - variable_mapping = human_input_node_data.extract_variable_selector_to_variable_mapping(node_config["id"]) - normalized_user_inputs: dict[str, Any] = dict(manual_inputs) - - load_into_variable_pool( - variable_loader=variable_loader, - variable_pool=variable_pool, - variable_mapping=variable_mapping, - user_inputs=normalized_user_inputs, - ) - WorkflowEntry.mapping_user_inputs_to_variable_pool( - variable_mapping=variable_mapping, - user_inputs=normalized_user_inputs, - variable_pool=variable_pool, - tenant_id=app_model.tenant_id, - ) - - return variable_pool - def run_free_workflow_node( self, node_data: dict[str, Any], tenant_id: str, user_id: str, node_id: str, user_inputs: dict[str, Any] ) -> WorkflowNodeExecution: @@ -1767,35 +756,6 @@ def _populate_execution_result( node_execution.status = WorkflowNodeExecutionStatus.FAILED node_execution.error = error - def convert_to_workflow(self, app_model: App, account: Account, args: dict[str, Any], *, session: Session) -> App: - """ - Basic mode of chatbot app(expert mode) to workflow - Completion App to Workflow App - - :param app_model: App instance - :param account: Account instance - :param args: dict - :return: - """ - # chatbot convert to workflow mode - workflow_converter = WorkflowConverter() - - if app_model.mode not in {AppMode.CHAT, AppMode.COMPLETION}: - raise ValueError(f"Current App mode: {app_model.mode} is not supported convert to workflow.") - - # convert to workflow - new_app: App = workflow_converter.convert_to_workflow( - app_model=app_model, - account=account, - name=args.get("name", "Default Name"), - icon_type=args.get("icon_type", "emoji"), - icon=args.get("icon", "🤖"), - icon_background=args.get("icon_background", "#FFEAD5"), - session=session, - ) - - return new_app - def validate_graph_structure(self, graph: Mapping[str, Any]): """ Validate workflow graph structure. @@ -1851,125 +811,12 @@ def _validate_human_input_node_data(self, node_data: dict[str, Any]) -> None: Raises: ValueError: If the node data format is invalid """ - from models.human_input_entities import HumanInputNodeData try: HumanInputNodeData.model_validate(adapt_human_input_node_data_for_graph(node_data)) except Exception as e: raise ValueError(f"Invalid HumanInput node data: {str(e)}") - def update_workflow( - self, - *, - session: Session, - account_id: str, - data: dict[str, Any], - workflow_ref: WorkflowRef, - ) -> Workflow | None: - """ - Update workflow attributes - - :param session: SQLAlchemy database session - :param account_id: Account ID (for permission check) - :param data: Dictionary containing fields to update - :param workflow_ref: Owner-bound workflow reference - :return: Updated workflow or None if not found - """ - stmt = select(Workflow).where( - Workflow.id == workflow_ref.workflow_id, - Workflow.tenant_id == workflow_ref.tenant_id, - Workflow.app_id == workflow_ref.owner_id, - ) - workflow = session.scalar(stmt) - - if not workflow: - return None - - allowed_fields = ["marked_name", "marked_comment"] - - for field, value in data.items(): - if field in allowed_fields: - setattr(workflow, field, value) - - workflow.updated_by = account_id - workflow.updated_at = naive_utc_now() - - return workflow - - def delete_workflow(self, *, session: Session, workflow_ref: WorkflowRef) -> list[str]: - """Stage a published Workflow and its binding owners for deletion. - - The exact owner key is tenant, App, Workflow, and Workflow version. The - Workflow row lock serializes source-version reads and restoration with - deletion. The caller must commit successfully before retiring the - returned, sorted and deduplicated inline Agent candidates. - - :param session: SQLAlchemy database session - :param workflow_ref: Owner-bound workflow reference - :return: Inline Agent IDs whose owner binding is staged for deletion - :raises: ValueError if workflow not found - :raises: WorkflowInUseError if workflow is in use - :raises: DraftWorkflowDeletionError if workflow is a draft version - """ - stmt = ( - select(Workflow) - .where( - Workflow.id == workflow_ref.workflow_id, - Workflow.tenant_id == workflow_ref.tenant_id, - Workflow.app_id == workflow_ref.owner_id, - ) - .with_for_update() - ) - workflow = session.scalar(stmt) - - if not workflow: - raise ValueError(f"Workflow with ID {workflow_ref.workflow_id} not found") - - # Check if workflow is a draft version - if workflow.version == Workflow.VERSION_DRAFT: - raise DraftWorkflowDeletionError("Cannot delete draft workflow versions") - - # Check if this workflow is currently referenced by an app - app_stmt = select(App).where(App.workflow_id == workflow_ref.workflow_id) - app = session.scalar(app_stmt) - if app: - # Cannot delete a workflow that's currently in use by an app - raise WorkflowInUseError(f"Cannot delete workflow that is currently in use by app '{app.id}'") - - # Don't use workflow.tool_published as it's not accurate for specific workflow versions - # Check if there's a tool provider using this specific workflow version - tool_provider = session.scalar( - select(WorkflowToolProvider).where( - WorkflowToolProvider.tenant_id == workflow.tenant_id, - WorkflowToolProvider.app_id == workflow.app_id, - WorkflowToolProvider.version == workflow.version, - ) - ) - - if tool_provider: - # Cannot delete a workflow that's published as a tool - raise WorkflowInUseError("Cannot delete workflow that is published as a tool") - - bindings = session.scalars( - select(WorkflowAgentNodeBinding).where( - WorkflowAgentNodeBinding.tenant_id == workflow.tenant_id, - WorkflowAgentNodeBinding.app_id == workflow.app_id, - WorkflowAgentNodeBinding.workflow_id == workflow.id, - WorkflowAgentNodeBinding.workflow_version == workflow.version, - ) - ).all() - retirement_candidates = sorted( - { - binding.agent_id - for binding in bindings - if binding.binding_type == WorkflowAgentBindingType.INLINE_AGENT and binding.agent_id - } - ) - for binding in bindings: - session.delete(binding) - session.delete(workflow) - return retirement_candidates - def _setup_variable_pool( query: str, diff --git a/api/tasks/app_generate/resume_agent_app_task.py b/api/tasks/app_generate/resume_agent_app_task.py index eac6d273425b1d..505b6a02cdfc8d 100644 --- a/api/tasks/app_generate/resume_agent_app_task.py +++ b/api/tasks/app_generate/resume_agent_app_task.py @@ -13,12 +13,13 @@ from celery import shared_task -from core.app.apps.agent_app.app_generator import AgentAppGenerator from core.app.entities.app_invoke_entities import InvokeFrom +from extensions.ext_application_services import application_services from extensions.ext_database import db from models.account import Account from models.human_input import HumanInputForm from models.model import App, Conversation, EndUser +from services.app.generation.adapters.agent_app import AgentAppGenerator from tasks.app_generate.workflow_execute_task import WORKFLOW_BASED_APP_EXECUTION_QUEUE logger = logging.getLogger(__name__) @@ -46,14 +47,23 @@ def resume_agent_app_execution(*, conversation_id: str, form_id: str) -> None: logger.warning("Agent App resume: no user resolvable for conversation %s", conversation_id) return + invoke_from = _resolve_invoke_from(conversation) + db.session.expunge_all() + db.session.close() try: - AgentAppGenerator().resume_after_form_submission( + AgentAppGenerator( + forms=application_services().workflow_runtime.human_forms, + agent_configs=application_services().workflow_runtime.agent_configs, + tool_providers=application_services().tools.tool_providers, + workflow_queries=application_services().tools.workflow_queries, + records=application_services().workflow_runtime.chat_records, + annotations=application_services().workflow_runtime.annotation_replies, + ).resume_after_form_submission( app_model=app_model, user=user, conversation_id=conversation_id, form_id=form_id, - invoke_from=_resolve_invoke_from(conversation), - session=db.session(), + invoke_from=invoke_from, ) except Exception: logger.exception("Agent App resume failed for conversation %s form %s", conversation_id, form_id) diff --git a/api/tasks/app_generate/workflow_execute_task.py b/api/tasks/app_generate/workflow_execute_task.py index 3cd30f3b5b7fec..ac8b16635e3900 100644 --- a/api/tasks/app_generate/workflow_execute_task.py +++ b/api/tasks/app_generate/workflow_execute_task.py @@ -1,7 +1,7 @@ import contextlib import logging import uuid -from collections.abc import Generator, Mapping +from collections.abc import Callable, Generator, Mapping from enum import StrEnum from typing import Annotated, Any @@ -11,10 +11,7 @@ from sqlalchemy import Engine, select from sqlalchemy.orm import Session, sessionmaker -from core.app.apps.advanced_chat.app_generator import AdvancedChatAppGenerator from core.app.apps.execution_coordinator import clear_app_task_cancellation_signals -from core.app.apps.message_based_app_generator import MessageBasedAppGenerator -from core.app.apps.workflow.app_generator import WorkflowAppGenerator from core.app.entities.app_invoke_entities import ( AdvancedChatAppGenerateEntity, InvokeFrom, @@ -23,6 +20,7 @@ from core.app.entities.task_entities import WorkflowFinishStreamResponse, WorkflowStartStreamResponse from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig, WorkflowResumptionContext from core.repositories import DifyCoreRepositoryFactory +from extensions.application_services.workflow import build_workflow_execution_dependencies from extensions.ext_database import db from graphon.entities import WorkflowStartReason from graphon.enums import WorkflowExecutionStatus @@ -36,6 +34,18 @@ from models.model import App, AppMode, Conversation, EndUser, Message from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom, WorkflowRun from repositories.factory import DifyAPIRepositoryFactory +from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository +from repositories.workflow.definition_repository import workflow_from_snapshot +from repositories.workflow.execution_write_repository import WorkflowExecutionWriteRepository +from services.agent.retirement_service import WorkflowAgentRetirementService +from services.errors.workflow_service import WorkflowDebugReservationExpiredError +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.debug_cancellation import DebugExecutionCancellation +from services.workflow.execution.adapters.chatflow.app_generator import AdvancedChatAppGenerator +from services.workflow.execution.adapters.response_converter import WorkflowResponseConverter +from services.workflow.execution.adapters.response_stream import WorkflowEventStream +from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator +from services.workflow.variable_contracts import WorkflowExecutionVariables logger = logging.getLogger(__name__) @@ -94,6 +104,8 @@ class AppExecutionParams(BaseModel): streaming: bool = True call_depth: int = 0 root_node_id: str | None = None + # Only trigger debugging pins a consumed event to a detached draft revision. + workflow_snapshot: WorkflowSnapshot | None = None workflow_run_id: str = Field(default_factory=lambda: str(uuid.uuid4())) @classmethod @@ -108,6 +120,7 @@ def new( call_depth: int = 0, root_node_id: str | None = None, workflow_run_id: str | None = None, + workflow_snapshot: WorkflowSnapshot | None = None, ): user_params: _Account | _EndUser match user: @@ -128,16 +141,31 @@ def new( streaming=streaming, call_depth=call_depth, root_node_id=root_node_id, + workflow_snapshot=workflow_snapshot, workflow_run_id=workflow_run_id or str(uuid.uuid4()), ) class _AppRunner: - def __init__(self, session_factory: sessionmaker | Engine, exec_params: AppExecutionParams): + def __init__( + self, + session_factory: sessionmaker | Engine, + exec_params: AppExecutionParams, + *, + variables: WorkflowExecutionVariables, + ): if isinstance(session_factory, Engine): session_factory = sessionmaker(bind=session_factory) self._session_factory = session_factory self._exec_params = exec_params + self._variables = variables + self._runtime = build_workflow_execution_dependencies(session_factory) + self._validated_snapshot = False + self._execution_finalized = False + self._cancellation: DebugExecutionCancellation | None = None + + def _mark_execution_finalized(self) -> None: + self._execution_finalized = True @contextlib.contextmanager def _session(self): @@ -152,8 +180,57 @@ def _setup_flask_context(self, user: Account | EndUser): yield def run(self): + try: + return self._run() + except WorkflowDebugReservationExpiredError as exc: + # claim() runs only after validating the persisted owner and snapshot. + # An expired request needs to close SSE; duplicate delivery must not + # emit a failure into the stream owned by an already running worker. + if self._exec_params.streaming and not self._execution_finalized: + _publish_failed_workflow_terminal_events(exc=exc, exec_params=self._exec_params) + raise + finally: + snapshot = self._exec_params.workflow_snapshot + if ( + not self._execution_finalized + and self._validated_snapshot + and snapshot is not None + and snapshot.execution_id is not None + ): + WorkflowAgentRetirementService.finish_execution( + sessions=self._session_factory, + tenant_id=snapshot.tenant_id, + app_id=snapshot.app_id, + workflow_id=snapshot.id, + execution_id=snapshot.execution_id, + account_id=snapshot.created_by, + ) + + def _run(self): exec_params = self._exec_params + completed_run: WorkflowRun | None = None + completed_creator: Account | EndUser | None = None with self._session() as session: + snapshot = exec_params.workflow_snapshot + reserved_run = None + if snapshot is not None and snapshot.execution_id is not None: + reserved_run = session.scalar( + select(WorkflowRun) + .where( + WorkflowRun.id == exec_params.workflow_run_id, + WorkflowRun.tenant_id == exec_params.tenant_id, + WorkflowRun.app_id == exec_params.app_id, + WorkflowRun.workflow_id == exec_params.workflow_id, + ) + .with_for_update() + ) + if reserved_run is None or ( + snapshot.execution_id, + snapshot.tenant_id, + snapshot.app_id, + snapshot.id, + ) != (reserved_run.id, reserved_run.tenant_id, reserved_run.app_id, reserved_run.workflow_id): + raise ValueError("Workflow snapshot does not belong to this debugger invocation") workflow = session.get(Workflow, exec_params.workflow_id) if workflow is None: logger.warning("Workflow %s not found for execution", exec_params.workflow_id) @@ -163,42 +240,86 @@ def run(self): logger.warning("App %s not found for workflow %s", workflow.app_id, exec_params.workflow_id) return None - pause_config = PauseStateLayerConfig( - session_factory=self._session_factory, - state_owner_user_id=workflow.created_by, - ) + snapshot = exec_params.workflow_snapshot + if snapshot is not None: + if ( + (snapshot.execution_id is not None and snapshot.execution_id != exec_params.workflow_run_id) + or exec_params.invoke_from != InvokeFrom.DEBUGGER + or exec_params.app_mode != AppMode.WORKFLOW + or app.mode != AppMode.WORKFLOW + or (workflow.id, workflow.tenant_id, workflow.app_id) + != (exec_params.workflow_id, exec_params.tenant_id, exec_params.app_id) + or (snapshot.id, snapshot.tenant_id, snapshot.app_id) != (workflow.id, app.tenant_id, app.id) + ): + raise ValueError("Workflow snapshot does not belong to this debugger invocation") + workflow = workflow_from_snapshot(snapshot) + if reserved_run is not None: + workflow.graph = reserved_run.graph or snapshot.graph + if WorkflowDebugReservationRepository.claim(session, reserved_run, naive_utc_now()): + self._validated_snapshot = True + else: + completed_run = reserved_run + self._mark_execution_finalized() + if exec_params.streaming: + completed_creator = _resolve_user_for_run(session, reserved_run) + + if completed_run is not None: + # Redelivery never claims a lease, runs the engine, or repeats cleanup. + # Replay the stored outcome after releasing the read transaction, + # including failures recorded by expiry recovery before worker startup. + if completed_creator is not None and completed_run.finished_at is not None: + response = WorkflowResponseConverter.workflow_run_result_to_finish_response( + task_id=exec_params.workflow_run_id, + workflow_run=completed_run, + creator_user=completed_creator, + ) + topic = WorkflowEventStream.get_response_topic(exec_params.app_mode, exec_params.workflow_run_id) + topic.publish(json.dumps(response.model_dump(mode="json"), ensure_ascii=False).encode()) + return None - user = self._resolve_user() + with contextlib.nullcontext() if self._validated_snapshot else contextlib.nullcontext() as self._cancellation: + pause_config = PauseStateLayerConfig( + session_factory=self._session_factory, + state_owner_user_id=workflow.created_by, + ) - with self._setup_flask_context(user), self._session_factory(expire_on_commit=False) as session: - try: - response = self._run_app( - app=app, - workflow=workflow, - user=user, - pause_state_config=pause_config, - session=session, - ) - except Exception as exc: - if exec_params.streaming: - _publish_failed_workflow_terminal_events( - exc=exc, - exec_params=exec_params, + user = self._resolve_user() + + with self._setup_flask_context(user): + try: + response = self._run_app( + app=app, + workflow=workflow, + user=user, + pause_state_config=pause_config, ) - raise - - if not exec_params.streaming: - return response - - assert isinstance(response, Generator) - _publish_streaming_response( - response, - exec_params.workflow_run_id, - exec_params.app_mode, - exec_params.workflow_id, - exec_params.args.get("inputs", {}), - WorkflowStartReason.INITIAL, - ) + except Exception as exc: + if exec_params.streaming: + _publish_failed_workflow_terminal_events( + exc=exc, + exec_params=exec_params, + ) + raise + + if not exec_params.streaming: + return response + + assert isinstance(response, Generator) + _publish_streaming_response( + response, + exec_params.workflow_run_id, + exec_params.app_mode, + exec_params.workflow_id, + exec_params.args.get("inputs", {}), + WorkflowStartReason.INITIAL, + sessions=self._session_factory, + tenant_id=app.tenant_id, + app_id=app.id, + agent_snapshot=exec_params.workflow_snapshot is not None + and exec_params.workflow_snapshot.execution_id is not None, + on_finalized=self._mark_execution_finalized, + on_terminal=self._cancellation.finish if self._cancellation is not None else None, + ) def _run_app( self, @@ -207,11 +328,15 @@ def _run_app( workflow: Workflow, user: Account | EndUser, pause_state_config: PauseStateLayerConfig, - session: Session, ): + variables = self._variables exec_params = self._exec_params if exec_params.app_mode == AppMode.ADVANCED_CHAT: - return AdvancedChatAppGenerator().generate( + return AdvancedChatAppGenerator( + runtime=self._runtime, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ).generate( app_model=app, workflow=workflow, user=user, @@ -220,10 +345,14 @@ def _run_app( streaming=exec_params.streaming, workflow_run_id=exec_params.workflow_run_id, pause_state_config=pause_state_config, - session=session, ) if exec_params.app_mode == AppMode.WORKFLOW: - return WorkflowAppGenerator().generate( + return WorkflowAppGenerator( + runtime=self._runtime, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + cancellation=self._cancellation, + ).generate( app_model=app, workflow=workflow, user=user, @@ -276,7 +405,7 @@ def _publish_failed_workflow_terminal_events(exc: Exception, exec_params: AppExe timestamp = to_timestamp(naive_utc_now()) assert timestamp is not None - topic = MessageBasedAppGenerator.get_response_topic(exec_params.app_mode, exec_params.workflow_run_id) + topic = WorkflowEventStream.get_response_topic(exec_params.app_mode, exec_params.workflow_run_id) started_payload = WorkflowStartStreamResponse( task_id=exec_params.workflow_run_id, workflow_run_id=exec_params.workflow_run_id, @@ -358,13 +487,24 @@ def _publish_streaming_response( workflow_id: str, inputs: Mapping[str, Any], started_reason: WorkflowStartReason, + *, + sessions: sessionmaker[Session], + tenant_id: str, + app_id: str, + agent_snapshot: bool, + on_finalized: Callable[[], None] | None = None, + on_terminal: Callable[[], None] | None = None, ) -> None: - """Publish workflow stream events and close broken streams with a failed terminal event. + """Publish engine outcomes, using synthetic failures only before an outcome is known. - `_AppRunner.run()` only handles failures before the generator is returned. - Once we start iterating the runtime stream, this helper becomes the last - place that can guarantee SSE consumers eventually see a terminal workflow - lifecycle event. + `_AppRunner.run()` provides fallback execution cleanup until this helper + commits the execution state. Resource cleanup runs after terminal delivery + and remains independently retryable. An observed pause or completion remains authoritative even when + delivery fails; retry its actual event once without inventing a failed result. + + Notify the initial runner after execution finalization succeeds, even when + streaming raises. Its fallback must not infer a second outcome from database + state that asynchronous persistence may have changed in the meantime. """ normalized_workflow_run_id = str(workflow_run_id) @@ -414,74 +554,137 @@ def _publish_failed_terminal_event(error_message: str, task_id: str, publish_sta terminal_events = {"workflow_finished", "workflow_paused"} unexpected_stream_end_message = "Workflow stream ended without a terminal event" - topic = MessageBasedAppGenerator.get_response_topic(app_mode, normalized_workflow_run_id) + topic = WorkflowEventStream.get_response_topic(app_mode, normalized_workflow_run_id) started_published = False terminal_published = False last_task_id = normalized_workflow_run_id stream_error_message: str | None = None + terminal_status: WorkflowExecutionStatus | None = None + terminal_payload: str | None = None + finalized = False + + def finalize() -> None: + nonlocal finalized + if agent_snapshot and not finalized: + WorkflowExecutionWriteRepository(sessions).finish( + tenant_id=tenant_id, + app_id=app_id, + workflow_id=workflow_id, + execution_id=normalized_workflow_run_id, + status=terminal_status, + ) + finalized = True + if on_finalized is not None: + on_finalized() + try: - for event in response_stream: - event_name = _get_event_name(event) - task_id = _get_task_id(event) - if task_id is not None: - last_task_id = task_id + try: + for event in response_stream: + event_name = _get_event_name(event) + task_id = _get_task_id(event) + if task_id is not None: + last_task_id = task_id + + # Engine outcomes own execution state. Serialization and Redis + # delivery can fail while a paused execution still needs its Agents. + if event_name == "workflow_paused": + terminal_status = WorkflowExecutionStatus.PAUSED + elif event_name == "workflow_finished": + data = _get_event_data(event) or {} + terminal_status = WorkflowExecutionStatus(data.get("data", {}).get("status", "failed")) + + if event_name in terminal_events: + if on_terminal is not None: + on_terminal() + # A subscriber can resume immediately on receiving PAUSED. + # Commit and acknowledge this worker's outcome first; neither + # this helper nor the outer runner may finalize it again. + finalize() + + try: + if isinstance(event, BaseModel): + payload = json.dumps(event.model_dump(mode="json"), ensure_ascii=False) + else: + payload = json.dumps(event, ensure_ascii=False, default=str) + except (TypeError, ValueError): + logger.exception("error while encoding event") + continue + + if event_name in terminal_events: + terminal_payload = payload + topic.publish(payload.encode()) + + if event_name == "workflow_started": + started_published = True + elif event_name in terminal_events: + terminal_published = True + elif event_name == "error": + stream_error_message = _get_error_message(event) or stream_error_message + except Exception as exc: + if not terminal_published and terminal_payload is not None: + logger.exception("Retrying engine terminal event publication for run %s", normalized_workflow_run_id) + topic.publish(terminal_payload.encode()) + elif not terminal_published and terminal_status is None: + logger.exception( + "Workflow stream for run %s failed before terminal event; publishing fallback terminal event", + normalized_workflow_run_id, + ) + _publish_failed_terminal_event( + error_message=str(exc) or exc.__class__.__name__, + task_id=last_task_id, + publish_started=not started_published, + ) + raise - try: - if isinstance(event, BaseModel): - payload = json.dumps(event.model_dump(mode="json"), ensure_ascii=False) - else: - payload = json.dumps(event, ensure_ascii=False, default=str) - except (TypeError, ValueError): - logger.exception("error while encoding event") - continue - - topic.publish(payload.encode()) - - if event_name == "workflow_started": - started_published = True - elif event_name in terminal_events: - terminal_published = True - elif event_name == "error": - stream_error_message = _get_error_message(event) or stream_error_message - except Exception as exc: - if not terminal_published: - logger.exception( - "Workflow stream for run %s failed before terminal event; publishing fallback terminal event", + if not terminal_published and terminal_status is None: + logger.warning( + "Workflow stream for run %s ended without a terminal event; publishing fallback terminal event", normalized_workflow_run_id, ) _publish_failed_terminal_event( - error_message=str(exc) or exc.__class__.__name__, + error_message=stream_error_message or unexpected_stream_end_message, task_id=last_task_id, publish_started=not started_published, ) - raise - - if not terminal_published: - logger.warning( - "Workflow stream for run %s ended without a terminal event; publishing fallback terminal event", - normalized_workflow_run_id, - ) - _publish_failed_terminal_event( - error_message=stream_error_message or unexpected_stream_end_message, - task_id=last_task_id, - publish_started=not started_published, - ) + finally: + try: + if isinstance(response_stream, Generator): + response_stream.close() + finally: + finalize() + if agent_snapshot and finalized: + # Cleanup dispatch is independent of the engine outcome and SSE. + # Retain durable retry evidence if the resource broker is down. + try: + WorkflowAgentRetirementService.retire_finished_execution( + delete_unstarted=False, + sessions=sessions, + tenant_id=tenant_id, + app_id=app_id, + workflow_id=workflow_id, + execution_id=normalized_workflow_run_id, + account_id=None, + ) + except Exception: + logger.exception("Failed to dispatch Agent cleanup for run %s", normalized_workflow_run_id) @shared_task(queue=WORKFLOW_BASED_APP_EXECUTION_QUEUE) def workflow_based_app_execution_task( payload: str, ) -> Mapping[str, Any] | None: + from extensions.ext_application_services import application_services + exec_params = AppExecutionParams.model_validate_json(payload) logger.info("workflow_based_app_execution_task run with params: %s", exec_params) - runner = _AppRunner(db.engine, exec_params=exec_params) + runner = _AppRunner(db.engine, exec_params=exec_params, variables=application_services().workflow_variables) return runner.run() -def _resume_app_execution(payload: dict[str, Any]) -> None: +def _resume_app_execution(payload: dict[str, Any], *, variables: WorkflowExecutionVariables) -> None: workflow_run_id = payload["workflow_run_id"] session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) @@ -551,6 +754,11 @@ def _resume_app_execution(payload: dict[str, Any]) -> None: logger.warning("Message not found for workflow run %s", workflow_run_id) return + binding_snapshot = workflow_run.graph_dict.get("_agent_bindings", {}) + if binding_snapshot.get("execution_id") == workflow_run.id and workflow_run.graph is not None: + # The resumed graph and its Agent generations belong to the original execution. + workflow.graph = workflow_run.graph + if not isinstance(generate_entity, (AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity)): logger.error( "Unsupported resumption entity for workflow run %s (found %s)", @@ -576,24 +784,24 @@ def _resume_app_execution(payload: dict[str, Any]) -> None: case AdvancedChatAppGenerateEntity(): assert conversation is not None assert message is not None - with session_factory() as session: - _resume_advanced_chat( - app_model=app_model, - workflow=workflow, - user=user, - conversation=conversation, - message=message, - generate_entity=generate_entity, - graph_runtime_state=graph_runtime_state, - response_stream_filter=response_stream_filter, - session_factory=session_factory, - pause_state_config=pause_config, - workflow_run_id=workflow_run_id, - workflow_run=workflow_run, - session=session, - ) + _resume_advanced_chat( + variables=variables, + app_model=app_model, + workflow=workflow, + user=user, + conversation=conversation, + message=message, + generate_entity=generate_entity, + graph_runtime_state=graph_runtime_state, + response_stream_filter=response_stream_filter, + session_factory=session_factory, + pause_state_config=pause_config, + workflow_run_id=workflow_run_id, + workflow_run=workflow_run, + ) case WorkflowAppGenerateEntity(): _resume_workflow( + variables=variables, app_model=app_model, workflow=workflow, user=user, @@ -611,6 +819,7 @@ def _resume_app_execution(payload: dict[str, Any]) -> None: def _resume_advanced_chat( *, + variables: WorkflowExecutionVariables, app_model: App, workflow: Workflow, user: Account | EndUser, @@ -623,7 +832,6 @@ def _resume_advanced_chat( pause_state_config: PauseStateLayerConfig, workflow_run_id: str, workflow_run: WorkflowRun, - session: Session, ) -> None: resumed_generate_entity = generate_entity.model_copy(update={"stream": True}) @@ -647,7 +855,11 @@ def _resume_advanced_chat( triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, ) - generator = AdvancedChatAppGenerator() + generator = AdvancedChatAppGenerator( + runtime=build_workflow_execution_dependencies(session_factory), + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ) try: response = generator.resume( @@ -662,7 +874,6 @@ def _resume_advanced_chat( graph_runtime_state=graph_runtime_state, pause_state_config=pause_state_config, response_stream_filter=response_stream_filter, - session=session, ) except Exception: logger.exception("Failed to resume chatflow execution for workflow run %s", workflow_run_id) @@ -676,11 +887,16 @@ def _resume_advanced_chat( workflow.id, generate_entity.inputs, WorkflowStartReason.RESUMPTION, + sessions=session_factory, + tenant_id=workflow.tenant_id, + app_id=workflow.app_id, + agent_snapshot="_agent_bindings" in workflow_run.graph_dict, ) def _resume_workflow( *, + variables: WorkflowExecutionVariables, app_model: App, workflow: Workflow, user: Account | EndUser, @@ -716,33 +932,47 @@ def _resume_workflow( triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, ) - generator = WorkflowAppGenerator() - - try: - response = generator.resume( - app_model=app_model, - workflow=workflow, - user=user, - application_generate_entity=resumed_generate_entity, - graph_runtime_state=graph_runtime_state, - workflow_execution_repository=workflow_execution_repository, - workflow_node_execution_repository=workflow_node_execution_repository, - pause_state_config=pause_state_config, - response_stream_filter=response_stream_filter, + with ( + contextlib.nullcontext() + if workflow_run.graph_dict.get("_agent_bindings", {}).get("execution_id") == workflow_run_id + else contextlib.nullcontext() + ) as cancellation: + generator = WorkflowAppGenerator( + runtime=build_workflow_execution_dependencies(session_factory), + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + cancellation=cancellation, ) - except Exception: - logger.exception("Failed to resume workflow execution for workflow run %s", workflow_run_id) - raise + try: + response = generator.resume( + app_model=app_model, + workflow=workflow, + user=user, + application_generate_entity=resumed_generate_entity, + graph_runtime_state=graph_runtime_state, + workflow_execution_repository=workflow_execution_repository, + workflow_node_execution_repository=workflow_node_execution_repository, + pause_state_config=pause_state_config, + response_stream_filter=response_stream_filter, + ) + except Exception: + logger.exception("Failed to resume workflow execution for workflow run %s", workflow_run_id) + raise - assert isinstance(response, Generator) - _publish_streaming_response( - response, - workflow_run_id, - AppMode.WORKFLOW, - workflow.id, - generate_entity.inputs, - WorkflowStartReason.RESUMPTION, - ) + assert isinstance(response, Generator) + _publish_streaming_response( + response, + workflow_run_id, + AppMode.WORKFLOW, + workflow.id, + generate_entity.inputs, + WorkflowStartReason.RESUMPTION, + sessions=session_factory, + tenant_id=workflow.tenant_id, + app_id=workflow.app_id, + agent_snapshot="_agent_bindings" in workflow_run.graph_dict, + on_terminal=cancellation.finish if cancellation is not None else None, + ) try: workflow_run_repo.delete_workflow_pause(pause_entity) @@ -757,4 +987,6 @@ def _resume_workflow( @shared_task(queue=WORKFLOW_BASED_APP_EXECUTION_QUEUE, name="resume_app_execution") def resume_app_execution(payload: dict[str, Any]) -> None: - _resume_app_execution(payload) + from extensions.ext_application_services import application_services + + _resume_app_execution(payload, variables=application_services().workflow_variables) diff --git a/api/tasks/async_workflow_tasks.py b/api/tasks/async_workflow_tasks.py index a3187b148c8512..2c2c1d09be0a6f 100644 --- a/api/tasks/async_workflow_tasks.py +++ b/api/tasks/async_workflow_tasks.py @@ -15,13 +15,13 @@ from typing_extensions import TypedDict from configs import dify_config -from core.app.apps.workflow.app_generator import WorkflowAppGenerator from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig, WorkflowResumptionContext from core.app.layers.timeslice_layer import TimeSliceLayer from core.app.layers.trigger_post_layer import TriggerPostLayer from core.db.session_factory import session_factory from core.repositories import DifyCoreRepositoryFactory +from extensions.application_services.workflow import build_workflow_execution_dependencies from extensions.ext_database import db from graphon.runtime import GraphRuntimeState from models.account import Account @@ -37,6 +37,7 @@ WorkflowResumeTaskData, WorkflowTaskData, ) +from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator from tasks.workflow_cfs_scheduler.cfs_scheduler import AsyncWorkflowCFSPlanEntity, AsyncWorkflowCFSPlanScheduler from tasks.workflow_cfs_scheduler.entities import AsyncWorkflowQueue, AsyncWorkflowSystemStrategy @@ -145,7 +146,9 @@ def _execute_workflow_common( user = _get_user(session, trigger_log) # Execute workflow using WorkflowAppGenerator - generator = WorkflowAppGenerator() + generator = WorkflowAppGenerator( + runtime=build_workflow_execution_dependencies(session_factory.get_session_maker()), + ) # Adapt trigger inputs and files for the generator. args = _build_generator_args(trigger_data) @@ -258,7 +261,9 @@ def resume_workflow_execution(task_data_dict: dict[str, Any]) -> None: state_owner_user_id=workflow.created_by, ) - generator = WorkflowAppGenerator() + generator = WorkflowAppGenerator( + runtime=build_workflow_execution_dependencies(session_factory), + ) start_time = datetime.now(UTC) graph_engine_layers = [] trigger_log = _query_trigger_log_info(session_factory, task_data.workflow_run_id) diff --git a/api/tasks/human_input_timeout_tasks.py b/api/tasks/human_input_timeout_tasks.py index 0f556d196d2a81..afd68b0846555f 100644 --- a/api/tasks/human_input_timeout_tasks.py +++ b/api/tasks/human_input_timeout_tasks.py @@ -6,14 +6,17 @@ from sqlalchemy.orm import sessionmaker from configs import dify_config -from core.repositories.human_input_repository import HumanInputFormSubmissionRepository from enums.human_input import HumanInputFormKind, HumanInputFormStatus from extensions.ext_database import db from extensions.ext_storage import storage from graphon.enums import WorkflowExecutionStatus from libs.datetime_utils import ensure_naive_utc, naive_utc_now +from models.enums import CreatorUserRole from models.human_input import HumanInputForm from models.workflow import WorkflowPause, WorkflowRun +from repositories.human_input.form_repository import HumanInputFormSubmissionRepository +from repositories.workflow.execution_write_repository import sync_debug_lease_status +from services.agent.retirement_service import WorkflowAgentRetirementService from services.human_input_service import HumanInputService logger = logging.getLogger(__name__) @@ -32,25 +35,48 @@ def _is_global_timeout(form_model: HumanInputForm, global_timeout_seconds: int, def _handle_global_timeout(*, form_id: str, workflow_run_id: str, node_id: str, session_factory: sessionmaker) -> None: now = naive_utc_now() with session_factory() as session, session.begin(): - workflow_run = session.get(WorkflowRun, workflow_run_id) - if workflow_run is not None: + workflow_run = session.scalar( + select(WorkflowRun) + .join( + HumanInputForm, + (HumanInputForm.workflow_run_id == WorkflowRun.id) + & (HumanInputForm.tenant_id == WorkflowRun.tenant_id) + & (HumanInputForm.app_id == WorkflowRun.app_id), + ) + .where(HumanInputForm.id == form_id, WorkflowRun.id == workflow_run_id) + .with_for_update(of=WorkflowRun) + ) + if workflow_run is None: + return + if not workflow_run.status.is_ended(): workflow_run.status = WorkflowExecutionStatus.STOPPED workflow_run.error = f"Human input global timeout at node {node_id}" workflow_run.finished_at = now session.add(workflow_run) + sync_debug_lease_status(session, workflow_run) pause_model = session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == workflow_run_id)) + state_object_key = None if pause_model is not None: - try: - storage.delete(pause_model.state_object_key) - except Exception: - logger.exception( - "Failed to delete pause state object for workflow_run_id=%s, pause_id=%s", - workflow_run_id, - pause_model.id, - ) + state_object_key = pause_model.state_object_key pause_model.resumed_at = now session.add(pause_model) + tenant_id, app_id, workflow_id = workflow_run.tenant_id, workflow_run.app_id, workflow_run.workflow_id + account_id = workflow_run.created_by if workflow_run.created_by_role == CreatorUserRole.ACCOUNT else None + + if state_object_key is not None: + try: + storage.delete(state_object_key) + except Exception: + logger.exception("Failed to delete pause state object for workflow_run_id=%s", workflow_run_id) + WorkflowAgentRetirementService.finish_execution( + sessions=session_factory, + tenant_id=tenant_id, + app_id=app_id, + workflow_id=workflow_id, + execution_id=workflow_run_id, + account_id=account_id, + ) @shared_task(name="human_input_form_timeout.check_and_resume", queue="schedule_executor") diff --git a/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py b/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py index c76edab1b4f7db..47ce3c50a386a5 100644 --- a/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py +++ b/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py @@ -14,13 +14,13 @@ from sqlalchemy.orm import Session, sessionmaker from configs import dify_config -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom, RagPipelineGenerateEntity from core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntity from core.db.session_factory import get_session_maker from core.rag.pipeline.queue import TenantIsolatedTaskQueue from core.repositories.factory import DifyCoreRepositoryFactory from extensions.application_services.data_sources import build_data_source_credentials +from extensions.application_services.workflow import build_workflow_execution_dependencies from extensions.ext_database import db from models import Account, Tenant from models.dataset import Pipeline @@ -28,6 +28,7 @@ from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository from services.file_service import FileService +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator logger = logging.getLogger(__name__) @@ -176,25 +177,24 @@ def run_single_rag_pipeline_task(rag_pipeline_invoke_entity: Mapping[str, Any], # Since we're already in a thread pool, no need for nested threading pipeline_generator = PipelineGenerator( + runtime=build_workflow_execution_dependencies(session_factory), documents=SQLAlchemyDocumentRepository(session_factory=session_factory), datasource_providers=build_data_source_credentials(database_client=get_session_maker()).providers, ) # Using protected method intentionally for async execution - with Session(db.engine, expire_on_commit=False) as session: - pipeline_generator._generate( # type: ignore[attr-defined] - session=session, - flask_app=flask_app, - context=context, - pipeline=pipeline, - workflow_id=workflow_id, - user=account, - application_generate_entity=entity, - invoke_from=InvokeFrom.PUBLISHED_PIPELINE, - workflow_execution_repository=workflow_execution_repository, - workflow_node_execution_repository=workflow_node_execution_repository, - streaming=streaming, - workflow_thread_pool_id=workflow_thread_pool_id, - ) + pipeline_generator._generate( # type: ignore[attr-defined] + flask_app=flask_app, + context=context, + pipeline=pipeline, + workflow_id=workflow_id, + user=account, + application_generate_entity=entity, + invoke_from=InvokeFrom.PUBLISHED_PIPELINE, + workflow_execution_repository=workflow_execution_repository, + workflow_node_execution_repository=workflow_node_execution_repository, + streaming=streaming, + workflow_thread_pool_id=workflow_thread_pool_id, + ) except Exception: logging.exception("Error in priority pipeline task") raise diff --git a/api/tasks/rag_pipeline/rag_pipeline_run_task.py b/api/tasks/rag_pipeline/rag_pipeline_run_task.py index 9774be64e9fcff..3f33bb8b5c1ebd 100644 --- a/api/tasks/rag_pipeline/rag_pipeline_run_task.py +++ b/api/tasks/rag_pipeline/rag_pipeline_run_task.py @@ -15,13 +15,13 @@ from sqlalchemy.orm import sessionmaker from configs import dify_config -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom, RagPipelineGenerateEntity from core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntity from core.db.session_factory import get_session_maker from core.rag.pipeline.queue import TenantIsolatedTaskQueue from core.repositories.factory import DifyCoreRepositoryFactory from extensions.application_services.data_sources import build_data_source_credentials +from extensions.application_services.workflow import build_workflow_execution_dependencies from extensions.ext_database import db from models import Account, Tenant from models.dataset import Pipeline @@ -29,6 +29,7 @@ from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository from services.file_service import FileService +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator logger = logging.getLogger(__name__) @@ -154,60 +155,58 @@ def run_single_rag_pipeline_task(rag_pipeline_invoke_entity: Mapping[str, Any], if not workflow: raise ValueError(f"Workflow {pipeline.workflow_id} not found") - if workflow_execution_id is None: - workflow_execution_id = str(uuid.uuid4()) - - # Create application generate entity from dict - entity = RagPipelineGenerateEntity.model_validate(application_generate_entity) - - # Create workflow repositories - session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) - workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository( - session_factory=session_factory, - tenant_id=pipeline.tenant_id, - user=account, - app_id=entity.app_config.app_id, - triggered_from=WorkflowRunTriggeredFrom.RAG_PIPELINE_RUN, - ) - - workflow_node_execution_repository = ( - DifyCoreRepositoryFactory.create_workflow_node_execution_repository( - session_factory=session_factory, - tenant_id=pipeline.tenant_id, - user=account, - app_id=entity.app_config.app_id, - triggered_from=WorkflowNodeExecutionTriggeredFrom.RAG_PIPELINE_RUN, - ) - ) - - # Set the user directly in g for preserve_flask_contexts - g._login_user = account - - # Copy context for passing to pipeline generator - context = contextvars.copy_context() - - # Direct execution without creating another thread - # Since we're already in a thread pool, no need for nested threading - - pipeline_generator = PipelineGenerator( - documents=SQLAlchemyDocumentRepository(session_factory=session_factory), - datasource_providers=build_data_source_credentials(database_client=get_session_maker()).providers, - ) - # Using protected method intentionally for async execution - pipeline_generator._generate( # type: ignore[attr-defined] - session=session, - flask_app=flask_app, - context=context, - pipeline=pipeline, - workflow_id=workflow_id, - user=account, - application_generate_entity=entity, - invoke_from=InvokeFrom.PUBLISHED_PIPELINE, - workflow_execution_repository=workflow_execution_repository, - workflow_node_execution_repository=workflow_node_execution_repository, - streaming=streaming, - workflow_thread_pool_id=workflow_thread_pool_id, - ) + if workflow_execution_id is None: + workflow_execution_id = str(uuid.uuid4()) + + # Create application generate entity from dict + entity = RagPipelineGenerateEntity.model_validate(application_generate_entity) + + # Create workflow repositories + session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) + workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository( + session_factory=session_factory, + tenant_id=pipeline.tenant_id, + user=account, + app_id=entity.app_config.app_id, + triggered_from=WorkflowRunTriggeredFrom.RAG_PIPELINE_RUN, + ) + + workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( + session_factory=session_factory, + tenant_id=pipeline.tenant_id, + user=account, + app_id=entity.app_config.app_id, + triggered_from=WorkflowNodeExecutionTriggeredFrom.RAG_PIPELINE_RUN, + ) + + # Set the user directly in g for preserve_flask_contexts + g._login_user = account + + # Copy context for passing to pipeline generator + context = contextvars.copy_context() + + # Direct execution without creating another thread + # Since we're already in a thread pool, no need for nested threading + + pipeline_generator = PipelineGenerator( + runtime=build_workflow_execution_dependencies(session_factory), + documents=SQLAlchemyDocumentRepository(session_factory=session_factory), + datasource_providers=build_data_source_credentials(database_client=get_session_maker()).providers, + ) + # Using protected method intentionally for async execution + pipeline_generator._generate( # type: ignore[attr-defined] + flask_app=flask_app, + context=context, + pipeline=pipeline, + workflow_id=workflow_id, + user=account, + application_generate_entity=entity, + invoke_from=InvokeFrom.PUBLISHED_PIPELINE, + workflow_execution_repository=workflow_execution_repository, + workflow_node_execution_repository=workflow_node_execution_repository, + streaming=streaming, + workflow_thread_pool_id=workflow_thread_pool_id, + ) except Exception: logging.exception("Error in pipeline task") raise diff --git a/api/tasks/retry_document_indexing_task.py b/api/tasks/retry_document_indexing_task.py index 13d4e72e1ea677..3703da1f06f9d0 100644 --- a/api/tasks/retry_document_indexing_task.py +++ b/api/tasks/retry_document_indexing_task.py @@ -6,17 +6,14 @@ from sqlalchemy import delete, select from configs import dify_config -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator -from core.db.session_factory import get_session_maker, session_factory +from core.db.session_factory import session_factory from core.rag.index_processor.index_processor import IndexProcessorFactory from enums import DeploymentEdition -from extensions.application_services.data_sources import build_data_source_credentials from extensions.ext_redis import redis_client from libs.datetime_utils import naive_utc_now from models import Account, Tenant from models.dataset import Dataset, Document, DocumentSegment from models.enums import IndexingStatus -from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository from services.feature_service import FeatureService from services.knowledge.indexing.adapters.execution import build_document_indexing_service from services.knowledge.resource_scope import DatasetRef @@ -119,20 +116,15 @@ def retry_document_indexing_task(dataset_id: str, document_ids: list[str], user_ session.commit() if runtime_mode == "rag_pipeline": + from extensions.ext_application_services import application_services + with session_factory.create_session() as rag_session: rag_pipeline_service = RagPipelineService(rag_session) rag_pipeline_service.retry_error_document( dataset, document, user, - generator=PipelineGenerator( - documents=SQLAlchemyDocumentRepository( - session_factory=session_factory.get_session_maker() - ), - datasource_providers=build_data_source_credentials( - database_client=get_session_maker() - ).providers, - ), + generator=application_services().knowledge.pipeline_generator, ) else: indexing_service = build_document_indexing_service( diff --git a/api/tasks/workflow_draft_var_tasks.py b/api/tasks/workflow_draft_var_tasks.py index 01aab2ac561826..dd62dba2f9fc40 100644 --- a/api/tasks/workflow_draft_var_tasks.py +++ b/api/tasks/workflow_draft_var_tasks.py @@ -1,30 +1,14 @@ -""" -Celery tasks for asynchronous workflow execution storage operations. - -These tasks provide asynchronous storage capabilities for workflow execution data, -improving performance by offloading storage operations to background workers. -""" +"""Retry reclamation of uploads from deleted variables or failed variable saves.""" import logging from celery import shared_task # type: ignore[import-untyped] from core.db.session_factory import session_factory -from services.workflow_draft_variable_service import DraftVarFileDeletion, WorkflowDraftVariableService logger = logging.getLogger(__name__) -@shared_task(queue="workflow_draft_var", bind=True, max_retries=3, default_retry_delay=60) -def save_workflow_execution_task( - self, - deletions: list[DraftVarFileDeletion], -): - with session_factory.create_session() as session, session.begin(): - srv = WorkflowDraftVariableService(session=session) - srv.delete_workflow_draft_variable_file(deletions=deletions) - - @shared_task( queue="workflow_storage", autoretry_for=(Exception,), diff --git a/api/tests/integration_tests/workflow/nodes/test_code.py b/api/tests/integration_tests/workflow/nodes/test_code.py index 8bbb0957b3c4b9..d2ae209728cad8 100644 --- a/api/tests/integration_tests/workflow/nodes/test_code.py +++ b/api/tests/integration_tests/workflow/nodes/test_code.py @@ -7,7 +7,6 @@ from configs import dify_config from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom -from core.workflow.node_factory import DifyNodeFactory from core.workflow.system_variables import build_system_variables from graphon.enums import WorkflowNodeExecutionStatus from graphon.graph import Graph @@ -16,6 +15,7 @@ from graphon.nodes.code.entities import CodeNodeData from graphon.nodes.code.limits import CodeNodeLimits from graphon.runtime import GraphRuntimeState, VariablePool +from services.workflow.execution.adapters.node_factory import DifyNodeFactory from tests.workflow_test_utils import build_test_graph_init_params pytest_plugins = ("tests.integration_tests.workflow.nodes.__mock.code_executor",) diff --git a/api/tests/integration_tests/workflow/nodes/test_http.py b/api/tests/integration_tests/workflow/nodes/test_http.py index 86b0cd0fd71c0f..fafb7aba5a2d41 100644 --- a/api/tests/integration_tests/workflow/nodes/test_http.py +++ b/api/tests/integration_tests/workflow/nodes/test_http.py @@ -8,14 +8,14 @@ from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom from core.helper.ssrf_proxy import ssrf_proxy from core.tools.tool_file_manager import ToolFileManager -from core.workflow.node_factory import DifyNodeFactory -from core.workflow.node_runtime import DifyFileReferenceFactory from core.workflow.system_variables import build_system_variables from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatus from graphon.file.file_manager import file_manager from graphon.graph import Graph from graphon.nodes.http_request import HttpRequestNode, HttpRequestNodeConfig, HttpRequestNodeData from graphon.runtime import GraphRuntimeState, VariablePool +from services.workflow.execution.adapters.node_factory import DifyNodeFactory +from services.workflow.execution.adapters.node_runtime import DifyFileReferenceFactory from tests.workflow_test_utils import build_test_graph_init_params pytest_plugins = ("tests.integration_tests.workflow.nodes.__mock.http",) diff --git a/api/tests/test_containers_integration_tests/conftest.py b/api/tests/test_containers_integration_tests/conftest.py index 7b588b7fa006a4..16e9bf4a01a24b 100644 --- a/api/tests/test_containers_integration_tests/conftest.py +++ b/api/tests/test_containers_integration_tests/conftest.py @@ -19,7 +19,7 @@ from flask import Flask from flask.testing import FlaskClient from sqlalchemy import Engine, text -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from testcontainers.core.container import DockerContainer from testcontainers.core.network import Network from testcontainers.core.wait_strategies import LogMessageWaitStrategy @@ -28,6 +28,7 @@ from app_factory import create_app from extensions.ext_database import db +from services.workflow.variable_contracts import WorkflowExecutionVariables # Configure logging for test containers logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") @@ -592,3 +593,11 @@ def _fake_request(method: str, url: str, **_kwargs: object) -> httpx.Response: patch("core.helper.ssrf_proxy.head", side_effect=lambda url, **kw: _fake_request("HEAD", url, **kw)), ): yield + + +@pytest.fixture +def workflow_variables(app: Flask) -> WorkflowExecutionVariables: + from extensions.application_services.workflow_variables import build_workflow_variable_service + + with app.app_context(): + return build_workflow_variable_service(database_client=sessionmaker(bind=db.engine, expire_on_commit=False)) diff --git a/api/tests/test_containers_integration_tests/controllers/openapi/test_app_dsl.py b/api/tests/test_containers_integration_tests/controllers/openapi/test_app_dsl.py index f65f78919169b0..e3d3936ab63c27 100644 --- a/api/tests/test_containers_integration_tests/controllers/openapi/test_app_dsl.py +++ b/api/tests/test_containers_integration_tests/controllers/openapi/test_app_dsl.py @@ -49,6 +49,7 @@ def external_deps() -> Generator[dict[str, object], None, None]: service and DB writes real.""" with ( patch("services.app_dsl_service.WorkflowService") as mock_workflow_service, + patch("extensions.application_services.workflow.build_workflow_draft_service") as mock_drafts, patch("services.app_dsl_service.DependenciesAnalysisService") as mock_dependencies_service, patch("services.app_dsl_service.app_was_created") as mock_app_was_created, patch("services.app_service.ModelManager.for_tenant") as mock_model_manager, @@ -56,7 +57,7 @@ def external_deps() -> Generator[dict[str, object], None, None]: patch("services.app_service.EnterpriseService") as mock_enterprise_service, ): mock_workflow_service.return_value.get_draft_workflow.return_value = None - mock_workflow_service.return_value.sync_draft_workflow.return_value = None + mock_drafts.return_value.sync.return_value = None mock_dependencies_service.generate_latest_dependencies.return_value = [] # type: ignore[assignment] mock_dependencies_service.get_leaked_dependencies.return_value = [] # type: ignore[assignment] mock_dependencies_service.generate_dependencies.return_value = [] # type: ignore[assignment] diff --git a/api/tests/test_containers_integration_tests/core/workflow/test_human_input_resume_node_execution.py b/api/tests/test_containers_integration_tests/core/workflow/test_human_input_resume_node_execution.py index c7674b9d401da5..fecd5e9be18a55 100644 --- a/api/tests/test_containers_integration_tests/core/workflow/test_human_input_resume_node_execution.py +++ b/api/tests/test_containers_integration_tests/core/workflow/test_human_input_resume_node_execution.py @@ -9,13 +9,8 @@ from core.app.app_config.entities import WorkflowUIBasedAppConfig from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity -from core.app.workflow.layers import PersistenceWorkflowInfo, WorkflowPersistenceLayer -from core.repositories.human_input_repository import HumanInputFormEntity, HumanInputFormRepository from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository -from core.workflow.nodes.human_input.callback import ( - DifyHITLCallback, -) from core.workflow.system_variables import build_system_variables from enums.human_input import HumanInputFormStatus from graphon.enums import WorkflowType @@ -32,9 +27,14 @@ from models import Account from models.account import AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatus from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom +from models.human_input_contracts import HumanInputFormEntity, HumanInputFormRepository from models.human_input_entities import HumanInputNodeData, UserActionConfig from models.model import App, AppMode, IconType from models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowRun +from services.workflow.execution.adapters.human_input import ( + DifyHITLCallback, +) +from services.workflow.execution.adapters.persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer from tests.workflow_test_utils import build_test_graph_init_params @@ -42,7 +42,6 @@ def _mock_form_repository_without_submission() -> HumanInputFormRepository: repo = MagicMock(spec=HumanInputFormRepository) form_entity = MagicMock(spec=HumanInputFormEntity) form_entity.id = "test-form-id" - form_entity.submission_token = "test-form-token" form_entity.recipients = [] form_entity.rendered_content = "rendered" form_entity.submitted = False @@ -55,7 +54,6 @@ def _mock_form_repository_with_submission(action_id: str) -> HumanInputFormRepos repo = MagicMock(spec=HumanInputFormRepository) form_entity = MagicMock(spec=HumanInputFormEntity) form_entity.id = "test-form-id" - form_entity.submission_token = "test-form-token" form_entity.recipients = [] form_entity.rendered_content = "rendered" form_entity.submitted = True diff --git a/api/tests/test_containers_integration_tests/services/dataset_collection_binding.py b/api/tests/test_containers_integration_tests/services/dataset_collection_binding.py index 706382b4b9fea4..58685375a171cd 100644 --- a/api/tests/test_containers_integration_tests/services/dataset_collection_binding.py +++ b/api/tests/test_containers_integration_tests/services/dataset_collection_binding.py @@ -1,7 +1,7 @@ """ -Comprehensive unit tests for DatasetCollectionBindingService. +Comprehensive unit tests for DatasetCollectionBindingRepository. -This module contains extensive unit tests for the DatasetCollectionBindingService class, +This module contains extensive unit tests for the DatasetCollectionBindingRepository class, which handles dataset collection binding operations for vector database collections. """ @@ -13,7 +13,7 @@ from models.dataset import DatasetCollectionBinding from models.enums import CollectionBindingType -from services.knowledge.dataset_service import DatasetCollectionBindingService +from repositories.knowledge.collection_binding_repository import DatasetCollectionBindingRepository class DatasetCollectionBindingTestDataFactory: @@ -58,9 +58,9 @@ def create_collection_binding( return binding -class TestDatasetCollectionBindingServiceGetBinding: +class TestDatasetCollectionBindingRepositoryGetBinding: """ - Comprehensive unit tests for DatasetCollectionBindingService.get_dataset_collection_binding method. + Comprehensive unit tests for DatasetCollectionBindingRepository.get_dataset_collection_binding method. This test class covers the main collection binding retrieval/creation functionality, including various provider/model combinations, collection types, and edge cases. @@ -87,7 +87,7 @@ def test_get_dataset_collection_binding_existing_binding_success(self, db_sessio ) # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding( provider_name, model_name, session=db_session_with_containers, collection_type=collection_type ) @@ -108,7 +108,7 @@ def test_get_dataset_collection_binding_create_new_binding_success(self, db_sess collection_type = CollectionBindingType.DATASET # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding( provider_name, model_name, session=db_session_with_containers, collection_type=collection_type ) @@ -127,7 +127,7 @@ def test_get_dataset_collection_binding_different_collection_type(self, db_sessi collection_type = "custom_type" # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding( provider_name, model_name, session=db_session_with_containers, collection_type=collection_type ) @@ -143,7 +143,7 @@ def test_get_dataset_collection_binding_default_collection_type(self, db_session model_name = "text-embedding-ada-002" # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding( provider_name, model_name, session=db_session_with_containers ) @@ -164,7 +164,7 @@ def test_get_dataset_collection_binding_different_provider_model_combination( ] # Act - results = list(starmap(DatasetCollectionBindingService.get_dataset_collection_binding, combinations)) + results = list(starmap(DatasetCollectionBindingRepository.get_dataset_collection_binding, combinations)) # Assert assert len(results) == 3 @@ -173,9 +173,9 @@ def test_get_dataset_collection_binding_different_provider_model_combination( assert result.model_name == model -class TestDatasetCollectionBindingServiceGetBindingByIdAndType: +class TestDatasetCollectionBindingRepositoryGetBindingByIdAndType: """ - Comprehensive unit tests for DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type method. + Verify collection binding lookup by ID and type through the shared repository. This test class covers retrieval of specific collection bindings by ID and type, including successful retrieval and error handling for missing bindings. @@ -193,7 +193,7 @@ def test_get_dataset_collection_binding_by_id_and_type_success(self, db_session_ ) # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( binding.id, session=db_session_with_containers, collection_type=CollectionBindingType.DATASET ) @@ -211,7 +211,7 @@ def test_get_dataset_collection_binding_by_id_and_type_not_found_error(self, db_ # Act & Assert with pytest.raises(ValueError, match="Dataset collection binding not found"): - DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( non_existent_id, session=db_session_with_containers, collection_type=CollectionBindingType.DATASET ) @@ -229,7 +229,7 @@ def test_get_dataset_collection_binding_by_id_and_type_different_collection_type ) # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( binding.id, session=db_session_with_containers, collection_type="custom_type" ) @@ -251,7 +251,7 @@ def test_get_dataset_collection_binding_by_id_and_type_default_collection_type( ) # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( binding.id, session=db_session_with_containers ) @@ -272,6 +272,6 @@ def test_get_dataset_collection_binding_by_id_and_type_wrong_type_error(self, db # Act & Assert with pytest.raises(ValueError, match="Dataset collection binding not found"): - DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( binding.id, session=db_session_with_containers, collection_type="wrong_type" ) diff --git a/api/tests/test_containers_integration_tests/services/test_agent_service.py b/api/tests/test_containers_integration_tests/services/test_agent_service.py deleted file mode 100644 index ab7c658be0803f..00000000000000 --- a/api/tests/test_containers_integration_tests/services/test_agent_service.py +++ /dev/null @@ -1,1028 +0,0 @@ -import json -from unittest.mock import MagicMock, create_autospec, patch - -import pytest -from faker import Faker -from sqlalchemy.orm import Session - -from core.plugin.impl.exc import PluginDaemonClientSideError -from models import Account, AppMode, CreatorUserRole -from models.enums import ConversationFromSource, EndUserType, MessageFileBelongsTo -from models.model import AppModelConfig, Conversation, EndUser, Message, MessageAgentThought -from services.agent_service import AgentService -from services.app_service import AppService, CreateAppParams -from tests.test_containers_integration_tests.helpers import accounts as account_fixtures -from tests.test_containers_integration_tests.helpers import generate_valid_password - - -class TestAgentService: - """Integration tests for AgentService using testcontainers.""" - - @pytest.fixture - def mock_external_service_dependencies(self): - """Mock setup for external service dependencies.""" - with ( - patch("services.agent_service.PluginAgentClient", autospec=True) as mock_plugin_agent_client, - patch("services.agent_service.ToolManager", autospec=True) as mock_tool_manager, - patch("services.agent_service.AgentConfigManager", autospec=True) as mock_agent_config_manager, - patch("services.agent_service.current_user", create_autospec(Account, instance=True)) as mock_current_user, - patch("services.app_service.SystemFeatureService", autospec=True) as mock_feature_service, - patch("services.app_service.EnterpriseService", autospec=True) as mock_enterprise_service, - patch("services.app_service.ModelManager.for_tenant", autospec=True) as mock_model_manager, - patch( - "services.account.login_adapters.SystemFeatureService", autospec=True - ) as mock_account_feature_service, - ): - # Setup default mock returns for agent service - mock_plugin_agent_client_instance = mock_plugin_agent_client.return_value - mock_plugin_agent_client_instance.fetch_agent_strategy_providers.return_value = [ - MagicMock( - plugin_id="test_plugin", - declaration=MagicMock( - identity=MagicMock(name="test_provider"), - strategies=[MagicMock(identity=MagicMock(name="test_strategy"))], - ), - ) - ] - mock_plugin_agent_client_instance.fetch_agent_strategy_provider.return_value = MagicMock( - plugin_id="test_plugin", - declaration=MagicMock( - identity=MagicMock(name="test_provider"), - strategies=[MagicMock(identity=MagicMock(name="test_strategy"))], - ), - ) - - # Setup ToolManager mocks - mock_tool_manager.get_tool_icon.return_value = "test_icon" - mock_tool_manager.get_tool_label.return_value = MagicMock( - to_dict=lambda: {"en_US": "Test Tool", "zh_Hans": "测试工具"} - ) - - # Setup AgentConfigManager mocks - mock_agent_config = MagicMock() - mock_agent_config.tools = [ - MagicMock(tool_name="test_tool", provider_type="test_provider", provider_id="test_id") - ] - mock_agent_config_manager.convert.return_value = mock_agent_config - - # Setup current_user mock - mock_current_user.timezone = "UTC" - - # Setup default mock returns for app service - mock_feature_service.is_webapp_auth_enabled.return_value = False - mock_enterprise_service.WebAppAuth.update_app_access_mode.return_value = None - mock_enterprise_service.WebAppAuth.cleanup_webapp.return_value = None - - # Setup default mock returns for account service - mock_account_feature_service.is_registration_allowed.return_value = True - - # Mock ModelManager for model configuration - mock_model_instance = mock_model_manager.return_value - mock_model_instance.get_default_model_instance.return_value = None - mock_model_instance.get_default_provider_model_name.return_value = ("openai", "gpt-3.5-turbo") - - yield { - "plugin_agent_client": mock_plugin_agent_client, - "tool_manager": mock_tool_manager, - "agent_config_manager": mock_agent_config_manager, - "current_user": mock_current_user, - "feature_service": mock_feature_service, - "enterprise_service": mock_enterprise_service, - "model_manager": mock_model_manager, - "account_feature_service": mock_account_feature_service, - } - - def _create_test_app_and_account(self, db_session_with_containers: Session, mock_external_service_dependencies): - """ - Helper method to create a test app and account for testing. - - Args: - db_session_with_containers: Database session from testcontainers infrastructure - mock_external_service_dependencies: Mock dependencies - - Returns: - tuple: (app, account) - Created app and account instances - """ - fake = Faker() - - # Setup mocks for account creation - mock_external_service_dependencies["account_feature_service"].is_registration_allowed.return_value = True - - # Create account and tenant - account = account_fixtures.create_account( - email=fake.email(), - name=fake.name(), - interface_language="en-US", - password=generate_valid_password(fake), - session=db_session_with_containers, - ) - account_fixtures.create_owner_workspace(account, name=fake.company(), session=db_session_with_containers) - tenant = account.current_tenant - - # Create app with realistic data - app_args = CreateAppParams( - name=fake.company(), - description=fake.text(max_nb_chars=100), - mode="agent-chat", - icon_type="emoji", - icon="🤖", - icon_background="#FF6B6B", - api_rph=100, - api_rpm=10, - ) - - app_service = AppService() - app = app_service.create_app(tenant.id, app_args, account, session=db_session_with_containers) - - # Update the app model config to set agent_mode for agent-chat mode - app_model_config = app.app_model_config_with_session(session=db_session_with_containers) - if app.mode == AppMode.AGENT_CHAT and app_model_config: - app_model_config.agent_mode = json.dumps({"enabled": True, "strategy": "react", "tools": []}) - - db_session_with_containers.commit() - - return app, account - - def _create_test_conversation_and_message(self, db_session_with_containers: Session, app, account): - """ - Helper method to create a test conversation and message with agent thoughts. - - Args: - db_session_with_containers: Database session from testcontainers infrastructure - app: App instance - account: Account instance - - Returns: - tuple: (conversation, message) - Created conversation and message instances - """ - fake = Faker() - - # Create conversation - conversation = Conversation( - id=fake.uuid4(), - app_id=app.id, - from_account_id=account.id, - from_end_user_id=None, - name=fake.sentence(), - inputs={}, - status="normal", - mode="chat", - from_source=ConversationFromSource.API, - ) - db_session_with_containers.add(conversation) - db_session_with_containers.commit() - - # Create app model config - app_model_config = AppModelConfig( - app_id=app.id, - provider="openai", - model_id="gpt-3.5-turbo", - configs={}, - model="gpt-3.5-turbo", - agent_mode=json.dumps({"enabled": True, "strategy": "react", "tools": []}), - ) - app_model_config.id = fake.uuid4() - db_session_with_containers.add(app_model_config) - db_session_with_containers.commit() - - # Update conversation with app model config - conversation.app_model_config_id = app_model_config.id - db_session_with_containers.commit() - - # Create message - message = Message( - id=fake.uuid4(), - conversation_id=conversation.id, - app_id=app.id, - from_account_id=account.id, - from_end_user_id=None, - inputs={}, - query=fake.text(max_nb_chars=100), - message=[{"role": "user", "text": fake.text(max_nb_chars=100)}], - answer=fake.text(max_nb_chars=200), - message_tokens=100, - message_unit_price=0.001, - answer_tokens=200, - answer_unit_price=0.001, - provider_response_latency=1.5, - currency="USD", - from_source=ConversationFromSource.API, - ) - db_session_with_containers.add(message) - db_session_with_containers.commit() - - return conversation, message - - def _create_test_agent_thoughts(self, db_session_with_containers: Session, message): - """ - Helper method to create test agent thoughts for a message. - - Args: - db_session_with_containers: Database session from testcontainers infrastructure - message: Message instance - - Returns: - list: Created agent thoughts - """ - fake = Faker() - - agent_thoughts = [] - - # Create first agent thought - thought1 = MessageAgentThought( - message_id=message.id, - position=1, - thought="I need to analyze the user's request", - tool="test_tool", - tool_labels_str=json.dumps({"test_tool": {"en_US": "Test Tool", "zh_Hans": "测试工具"}}), - tool_meta_str=json.dumps( - { - "test_tool": { - "error": None, - "time_cost": 0.5, - "tool_config": {"tool_provider_type": "test_provider", "tool_provider": "test_id"}, - "tool_parameters": {}, - } - } - ), - tool_input=json.dumps({"test_tool": {"input": "test_input"}}), - observation=json.dumps({"test_tool": {"output": "test_output"}}), - tokens=50, - created_by_role=CreatorUserRole.ACCOUNT, - created_by=message.from_account_id, - ) - db_session_with_containers.add(thought1) - agent_thoughts.append(thought1) - - # Create second agent thought - thought2 = MessageAgentThought( - message_id=message.id, - position=2, - thought="Based on the analysis, I can provide a response", - tool="dataset_tool", - tool_labels_str=json.dumps({"dataset_tool": {"en_US": "Dataset Tool", "zh_Hans": "数据集工具"}}), - tool_meta_str=json.dumps( - { - "dataset_tool": { - "error": None, - "time_cost": 0.3, - "tool_config": {"tool_provider_type": "dataset-retrieval", "tool_provider": "dataset_id"}, - "tool_parameters": {}, - } - } - ), - tool_input=json.dumps({"dataset_tool": {"query": "test_query"}}), - observation=json.dumps({"dataset_tool": {"results": "test_results"}}), - tokens=30, - created_by_role=CreatorUserRole.ACCOUNT, - created_by=message.from_account_id, - ) - db_session_with_containers.add(thought2) - agent_thoughts.append(thought2) - - db_session_with_containers.commit() - - return agent_thoughts - - def test_get_agent_logs_success(self, db_session_with_containers: Session, mock_external_service_dependencies): - """ - Test successful retrieval of agent logs with complete data. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - conversation, message = self._create_test_conversation_and_message(db_session_with_containers, app, account) - agent_thoughts = self._create_test_agent_thoughts(db_session_with_containers, message) - - # Execute the method under test - result = AgentService.get_agent_logs(app, conversation.id, message.id, db_session_with_containers) - - # Verify the result structure - assert result is not None - assert "meta" in result - assert "iterations" in result - assert "files" in result - - # Verify meta information - meta = result["meta"] - assert meta["status"] == "success" - assert meta["executor"] == account.name - assert meta["iterations"] == 2 - assert meta["agent_mode"] == "react" - assert meta["total_tokens"] == 300 # 100 + 200 - assert meta["elapsed_time"] == 1.5 - - # Verify iterations - iterations = result["iterations"] - assert len(iterations) == 2 - - # Verify first iteration - first_iteration = iterations[0] - assert first_iteration["tokens"] == 50 - assert first_iteration["thought"] == "I need to analyze the user's request" - assert len(first_iteration["tool_calls"]) == 1 - - tool_call = first_iteration["tool_calls"][0] - assert tool_call["tool_name"] == "test_tool" - assert tool_call["tool_label"] == {"en_US": "Test Tool", "zh_Hans": "测试工具"} - assert tool_call["status"] == "success" - assert tool_call["time_cost"] == 0.5 - assert tool_call["tool_icon"] == "test_icon" - - # Verify second iteration - second_iteration = iterations[1] - assert second_iteration["tokens"] == 30 - assert second_iteration["thought"] == "Based on the analysis, I can provide a response" - assert len(second_iteration["tool_calls"]) == 1 - - dataset_tool_call = second_iteration["tool_calls"][0] - assert dataset_tool_call["tool_name"] == "dataset_tool" - assert dataset_tool_call["tool_label"] == {"en_US": "Dataset Tool", "zh_Hans": "数据集工具"} - assert dataset_tool_call["status"] == "success" - assert dataset_tool_call["time_cost"] == 0.3 - assert dataset_tool_call["tool_icon"] == "" # dataset-retrieval tools have empty icon - - def test_get_agent_logs_conversation_not_found( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test error handling when conversation is not found. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - - # Execute the method under test with non-existent conversation - with pytest.raises(ValueError, match="Conversation not found"): - AgentService.get_agent_logs(app, fake.uuid4(), fake.uuid4(), db_session_with_containers) - - def test_get_agent_logs_message_not_found( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test error handling when message is not found. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - conversation, message = self._create_test_conversation_and_message(db_session_with_containers, app, account) - - # Execute the method under test with non-existent message - with pytest.raises(ValueError, match="Message not found"): - AgentService.get_agent_logs(app, conversation.id, fake.uuid4(), db_session_with_containers) - - def test_get_agent_logs_with_end_user( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test agent logs retrieval when conversation is from end user. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - - # Create end user - end_user = EndUser( - id=fake.uuid4(), - tenant_id=app.tenant_id, - app_id=app.id, - type=EndUserType.BROWSER, - is_anonymous=False, - session_id=fake.uuid4(), - name=fake.name(), - ) - db_session_with_containers.add(end_user) - db_session_with_containers.commit() - - # Create conversation with end user - conversation = Conversation( - id=fake.uuid4(), - app_id=app.id, - from_account_id=None, - from_end_user_id=end_user.id, - name=fake.sentence(), - inputs={}, - status="normal", - mode="chat", - from_source=ConversationFromSource.API, - ) - db_session_with_containers.add(conversation) - db_session_with_containers.commit() - - # Create app model config - app_model_config = AppModelConfig( - app_id=app.id, - provider="openai", - model_id="gpt-3.5-turbo", - configs={}, - model="gpt-3.5-turbo", - agent_mode=json.dumps({"enabled": True, "strategy": "react", "tools": []}), - ) - app_model_config.id = fake.uuid4() - db_session_with_containers.add(app_model_config) - db_session_with_containers.commit() - - # Update conversation with app model config - conversation.app_model_config_id = app_model_config.id - db_session_with_containers.commit() - - # Create message - message = Message( - id=fake.uuid4(), - conversation_id=conversation.id, - app_id=app.id, - from_account_id=None, - from_end_user_id=end_user.id, - inputs={}, - query=fake.text(max_nb_chars=100), - message=[{"role": "user", "text": fake.text(max_nb_chars=100)}], - answer=fake.text(max_nb_chars=200), - message_tokens=100, - message_unit_price=0.001, - answer_tokens=200, - answer_unit_price=0.001, - provider_response_latency=1.5, - currency="USD", - from_source=ConversationFromSource.API, - ) - db_session_with_containers.add(message) - db_session_with_containers.commit() - - # Execute the method under test - result = AgentService.get_agent_logs(app, conversation.id, message.id, db_session_with_containers) - - # Verify the result - assert result is not None - assert result["meta"]["executor"] == end_user.name - - def test_get_agent_logs_with_unknown_executor( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test agent logs retrieval when executor is unknown. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - - # Create conversation with non-existent account - conversation = Conversation( - id=fake.uuid4(), - app_id=app.id, - from_account_id=fake.uuid4(), # Non-existent account - from_end_user_id=None, - name=fake.sentence(), - inputs={}, - status="normal", - mode="chat", - from_source=ConversationFromSource.API, - ) - db_session_with_containers.add(conversation) - db_session_with_containers.commit() - - # Create app model config - app_model_config = AppModelConfig( - app_id=app.id, - provider="openai", - model_id="gpt-3.5-turbo", - configs={}, - model="gpt-3.5-turbo", - agent_mode=json.dumps({"enabled": True, "strategy": "react", "tools": []}), - ) - app_model_config.id = fake.uuid4() - db_session_with_containers.add(app_model_config) - db_session_with_containers.commit() - - # Update conversation with app model config - conversation.app_model_config_id = app_model_config.id - db_session_with_containers.commit() - - # Create message - message = Message( - id=fake.uuid4(), - conversation_id=conversation.id, - app_id=app.id, - from_account_id=fake.uuid4(), # Non-existent account - from_end_user_id=None, - inputs={}, - query=fake.text(max_nb_chars=100), - message=[{"role": "user", "text": fake.text(max_nb_chars=100)}], - answer=fake.text(max_nb_chars=200), - message_tokens=100, - message_unit_price=0.001, - answer_tokens=200, - answer_unit_price=0.001, - provider_response_latency=1.5, - currency="USD", - from_source=ConversationFromSource.API, - ) - db_session_with_containers.add(message) - db_session_with_containers.commit() - - # Execute the method under test - result = AgentService.get_agent_logs(app, conversation.id, message.id, db_session_with_containers) - - # Verify the result - assert result is not None - assert result["meta"]["executor"] == "Unknown" - - def test_get_agent_logs_with_tool_error( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test agent logs retrieval with tool errors. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - conversation, message = self._create_test_conversation_and_message(db_session_with_containers, app, account) - - # Create agent thought with tool error - thought_with_error = MessageAgentThought( - message_id=message.id, - position=1, - thought="I need to analyze the user's request", - tool="error_tool", - tool_labels_str=json.dumps({"error_tool": {"en_US": "Error Tool", "zh_Hans": "错误工具"}}), - tool_meta_str=json.dumps( - { - "error_tool": { - "error": "Tool execution failed", - "time_cost": 0.5, - "tool_config": {"tool_provider_type": "test_provider", "tool_provider": "test_id"}, - "tool_parameters": {}, - } - } - ), - tool_input=json.dumps({"error_tool": {"input": "test_input"}}), - observation=json.dumps({"error_tool": {"output": "error_output"}}), - tokens=50, - created_by_role=CreatorUserRole.ACCOUNT, - created_by=message.from_account_id, - ) - db_session_with_containers.add(thought_with_error) - db_session_with_containers.commit() - - # Execute the method under test - result = AgentService.get_agent_logs(app, conversation.id, message.id, db_session_with_containers) - - # Verify the result - assert result is not None - iterations = result["iterations"] - assert len(iterations) == 1 - - tool_call = iterations[0]["tool_calls"][0] - assert tool_call["status"] == "error" - assert tool_call["error"] == "Tool execution failed" - - def test_get_agent_logs_without_agent_thoughts( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test agent logs retrieval when message has no agent thoughts. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - conversation, message = self._create_test_conversation_and_message(db_session_with_containers, app, account) - - # Execute the method under test - result = AgentService.get_agent_logs(app, conversation.id, message.id, db_session_with_containers) - - # Verify the result - assert result is not None - assert result["meta"]["iterations"] == 0 - assert len(result["iterations"]) == 0 - - def test_get_agent_logs_app_model_config_not_found( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test error handling when app model config is not found. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - - # Remove app model config to test error handling - app.app_model_config_id = None - db_session_with_containers.commit() - - # Create conversation without app model config - conversation = Conversation( - id=fake.uuid4(), - app_id=app.id, - from_account_id=account.id, - from_end_user_id=None, - name=fake.sentence(), - inputs={}, - status="normal", - mode="chat", - from_source=ConversationFromSource.API, - app_model_config_id=None, # Explicitly set to None - ) - db_session_with_containers.add(conversation) - db_session_with_containers.commit() - - # Create message - message = Message( - id=fake.uuid4(), - conversation_id=conversation.id, - app_id=app.id, - from_account_id=account.id, - from_end_user_id=None, - inputs={}, - query=fake.text(max_nb_chars=100), - message=[{"role": "user", "text": fake.text(max_nb_chars=100)}], - answer=fake.text(max_nb_chars=200), - message_tokens=100, - message_unit_price=0.001, - answer_tokens=200, - answer_unit_price=0.001, - provider_response_latency=1.5, - currency="USD", - from_source=ConversationFromSource.API, - ) - db_session_with_containers.add(message) - db_session_with_containers.commit() - - # Execute the method under test - with pytest.raises(ValueError, match="App model config not found"): - AgentService.get_agent_logs(app, conversation.id, message.id, db_session_with_containers) - - def test_get_agent_logs_agent_config_not_found( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test error handling when agent config is not found. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - conversation, message = self._create_test_conversation_and_message(db_session_with_containers, app, account) - - # Mock AgentConfigManager to return None - mock_external_service_dependencies["agent_config_manager"].convert.return_value = None - - # Execute the method under test - with pytest.raises(ValueError, match="Agent config not found"): - AgentService.get_agent_logs(app, conversation.id, message.id, db_session_with_containers) - - def test_list_agent_providers_success( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test successful listing of agent providers. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - - # Execute the method under test - result = AgentService.list_agent_providers(account.id, app.tenant_id) - - # Verify the result - assert result is not None - assert len(result) == 1 - assert result[0].plugin_id == "test_plugin" - - # Verify the mock was called correctly - mock_plugin_client = mock_external_service_dependencies["plugin_agent_client"].return_value - mock_plugin_client.fetch_agent_strategy_providers.assert_called_once_with(app.tenant_id) - - def test_get_agent_provider_success(self, db_session_with_containers: Session, mock_external_service_dependencies): - """ - Test successful retrieval of specific agent provider. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - - provider_name = "test_provider" - - # Execute the method under test - result = AgentService.get_agent_provider(account.id, app.tenant_id, provider_name) - - # Verify the result - assert result is not None - assert result.plugin_id == "test_plugin" - - # Verify the mock was called correctly - mock_plugin_client = mock_external_service_dependencies["plugin_agent_client"].return_value - mock_plugin_client.fetch_agent_strategy_provider.assert_called_once_with(app.tenant_id, provider_name) - - def test_get_agent_provider_plugin_error( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test error handling when plugin daemon client raises an error. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - - provider_name = "test_provider" - error_message = "Plugin not found" - - # Mock PluginAgentClient to raise an error - mock_plugin_client = mock_external_service_dependencies["plugin_agent_client"].return_value - mock_plugin_client.fetch_agent_strategy_provider.side_effect = PluginDaemonClientSideError(error_message) - - # Execute the method under test - with pytest.raises(ValueError, match=error_message): - AgentService.get_agent_provider(account.id, app.tenant_id, provider_name) - - def test_get_agent_logs_with_complex_tool_data( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test agent logs retrieval with complex tool data and multiple tools. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - conversation, message = self._create_test_conversation_and_message(db_session_with_containers, app, account) - - # Create agent thought with multiple tools - complex_thought = MessageAgentThought( - message_id=message.id, - position=1, - thought="I need to use multiple tools to complete this task", - tool="tool1;tool2;tool3", - tool_labels_str=json.dumps( - { - "tool1": {"en_US": "First Tool", "zh_Hans": "第一个工具"}, - "tool2": {"en_US": "Second Tool", "zh_Hans": "第二个工具"}, - "tool3": {"en_US": "Third Tool", "zh_Hans": "第三个工具"}, - } - ), - tool_meta_str=json.dumps( - { - "tool1": { - "error": None, - "time_cost": 0.5, - "tool_config": {"tool_provider_type": "test_provider", "tool_provider": "test_id"}, - "tool_parameters": {"param1": "value1"}, - }, - "tool2": { - "error": "Tool 2 failed", - "time_cost": 0.3, - "tool_config": {"tool_provider_type": "another_provider", "tool_provider": "another_id"}, - "tool_parameters": {"param2": "value2"}, - }, - "tool3": { - "error": None, - "time_cost": 0.7, - "tool_config": {"tool_provider_type": "dataset-retrieval", "tool_provider": "dataset_id"}, - "tool_parameters": {"param3": "value3"}, - }, - } - ), - tool_input=json.dumps( - {"tool1": {"input1": "data1"}, "tool2": {"input2": "data2"}, "tool3": {"input3": "data3"}} - ), - observation=json.dumps( - {"tool1": {"output1": "result1"}, "tool2": {"output2": "result2"}, "tool3": {"output3": "result3"}} - ), - tokens=100, - created_by_role=CreatorUserRole.ACCOUNT, - created_by=message.from_account_id, - ) - db_session_with_containers.add(complex_thought) - db_session_with_containers.commit() - - # Execute the method under test - result = AgentService.get_agent_logs(app, conversation.id, message.id, db_session_with_containers) - - # Verify the result - assert result is not None - iterations = result["iterations"] - assert len(iterations) == 1 - - tool_calls = iterations[0]["tool_calls"] - assert len(tool_calls) == 3 - - # Verify first tool - assert tool_calls[0]["tool_name"] == "tool1" - assert tool_calls[0]["tool_label"] == {"en_US": "First Tool", "zh_Hans": "第一个工具"} - assert tool_calls[0]["status"] == "success" - assert tool_calls[0]["tool_parameters"] == {"param1": "value1"} - - # Verify second tool (with error) - assert tool_calls[1]["tool_name"] == "tool2" - assert tool_calls[1]["tool_label"] == {"en_US": "Second Tool", "zh_Hans": "第二个工具"} - assert tool_calls[1]["status"] == "error" - assert tool_calls[1]["error"] == "Tool 2 failed" - - # Verify third tool (dataset tool) - assert tool_calls[2]["tool_name"] == "tool3" - assert tool_calls[2]["tool_label"] == {"en_US": "Third Tool", "zh_Hans": "第三个工具"} - assert tool_calls[2]["status"] == "success" - assert tool_calls[2]["tool_icon"] == "" # dataset-retrieval tools have empty icon - - def test_get_agent_logs_with_files(self, db_session_with_containers: Session, mock_external_service_dependencies): - """ - Test agent logs retrieval with message files and agent thought files. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - conversation, message = self._create_test_conversation_and_message(db_session_with_containers, app, account) - - from graphon.file import FileTransferMethod, FileType - - # Add files to message - from models.model import MessageFile - - assert message.from_account_id is not None - message_file1 = MessageFile( - message_id=message.id, - type=FileType.IMAGE, - transfer_method=FileTransferMethod.REMOTE_URL, - url="http://example.com/file1.jpg", - belongs_to=MessageFileBelongsTo.USER, - created_by_role=CreatorUserRole.ACCOUNT, - created_by=message.from_account_id, - ) - message_file2 = MessageFile( - message_id=message.id, - type=FileType.IMAGE, - transfer_method=FileTransferMethod.REMOTE_URL, - url="http://example.com/file2.png", - belongs_to=MessageFileBelongsTo.USER, - created_by_role=CreatorUserRole.ACCOUNT, - created_by=message.from_account_id, - ) - db_session_with_containers.add(message_file1) - db_session_with_containers.add(message_file2) - db_session_with_containers.commit() - - # Create agent thought with files - thought_with_files = MessageAgentThought( - message_id=message.id, - position=1, - thought="I need to process some files", - tool="file_tool", - tool_labels_str=json.dumps({"file_tool": {"en_US": "File Tool", "zh_Hans": "文件工具"}}), - tool_meta_str=json.dumps( - { - "file_tool": { - "error": None, - "time_cost": 0.5, - "tool_config": {"tool_provider_type": "test_provider", "tool_provider": "test_id"}, - "tool_parameters": {}, - } - } - ), - tool_input=json.dumps({"file_tool": {"input": "test_input"}}), - observation=json.dumps({"file_tool": {"output": "test_output"}}), - message_files=json.dumps(["file1", "file2"]), - tokens=50, - created_by_role=CreatorUserRole.ACCOUNT, - created_by=message.from_account_id, - ) - db_session_with_containers.add(thought_with_files) - db_session_with_containers.commit() - - # Execute the method under test - result = AgentService.get_agent_logs(app, conversation.id, message.id, db_session_with_containers) - - # Verify the result - assert result is not None - assert len(result["files"]) == 2 - - iterations = result["iterations"] - assert len(iterations) == 1 - assert len(iterations[0]["files"]) == 2 - assert "file1" in iterations[0]["files"] - assert "file2" in iterations[0]["files"] - - def test_get_agent_logs_with_different_timezone( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test agent logs retrieval with different timezone settings. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - conversation, message = self._create_test_conversation_and_message(db_session_with_containers, app, account) - - # Mock current_user with different timezone - mock_external_service_dependencies["current_user"].timezone = "Asia/Shanghai" - - # Execute the method under test - result = AgentService.get_agent_logs(app, conversation.id, message.id, db_session_with_containers) - - # Verify the result - assert result is not None - assert "start_time" in result["meta"] - - # Verify the timezone conversion - start_time = result["meta"]["start_time"] - assert "T" in start_time # ISO format - assert "+08:00" in start_time or "Z" in start_time # Timezone offset - - def test_get_agent_logs_with_empty_tool_data( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test agent logs retrieval with empty tool data. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - conversation, message = self._create_test_conversation_and_message(db_session_with_containers, app, account) - - # Create agent thought with empty tool data - empty_thought = MessageAgentThought( - message_id=message.id, - position=1, - thought="I need to analyze the user's request", - tool="", # Empty tool - tool_labels_str="{}", # Empty labels - tool_meta_str="{}", # Empty meta - tool_input="", # Empty input - observation="", # Empty observation - tokens=50, - created_by_role=CreatorUserRole.ACCOUNT, - created_by=message.from_account_id, - ) - db_session_with_containers.add(empty_thought) - db_session_with_containers.commit() - - # Execute the method under test - result = AgentService.get_agent_logs(app, conversation.id, message.id, db_session_with_containers) - - # Verify the result - assert result is not None - iterations = result["iterations"] - assert len(iterations) == 1 - - # Verify empty tool calls - tool_calls = iterations[0]["tool_calls"] - assert len(tool_calls) == 0 # No tools to process - - def test_get_agent_logs_with_malformed_json( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test agent logs retrieval with malformed JSON data in tool fields. - """ - fake = Faker() - - # Create test data - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - conversation, message = self._create_test_conversation_and_message(db_session_with_containers, app, account) - - # Create agent thought with malformed JSON - malformed_thought = MessageAgentThought( - message_id=message.id, - position=1, - thought="I need to analyze the user's request", - tool="test_tool", - tool_labels_str="invalid json", # Malformed JSON - tool_meta_str="invalid json", # Malformed JSON - tool_input="invalid json", # Malformed JSON - observation="invalid json", # Malformed JSON - tokens=50, - created_by_role=CreatorUserRole.ACCOUNT, - created_by=message.from_account_id, - ) - db_session_with_containers.add(malformed_thought) - db_session_with_containers.commit() - - # Execute the method under test - result = AgentService.get_agent_logs(app, conversation.id, message.id, db_session_with_containers) - - # Verify the result - should handle malformed JSON gracefully - assert result is not None - iterations = result["iterations"] - assert len(iterations) == 1 - - tool_calls = iterations[0]["tool_calls"] - assert len(tool_calls) == 1 - - # Verify default values for malformed JSON - tool_call = tool_calls[0] - assert tool_call["tool_name"] == "test_tool" - assert tool_call["tool_label"] == "test_tool" # Default to tool name - assert tool_call["tool_input"] == {} - assert tool_call["tool_output"] == "invalid json" # Raw observation value - assert tool_call["tool_parameters"] == {} diff --git a/api/tests/test_containers_integration_tests/services/test_annotation_service.py b/api/tests/test_containers_integration_tests/services/test_annotation_service.py index ddf1fe25c97ed4..dfaa175c10d0f4 100644 --- a/api/tests/test_containers_integration_tests/services/test_annotation_service.py +++ b/api/tests/test_containers_integration_tests/services/test_annotation_service.py @@ -3,12 +3,12 @@ import pytest from faker import Faker from sqlalchemy.orm import Session -from werkzeug.exceptions import NotFound from enums import DeploymentEdition from models import Account from models.enums import ConversationFromSource, InvokeFrom -from models.model import MessageAnnotation +from models.model import AppAnnotationHitHistory, MessageAnnotation +from services.annotation.errors import AnnotationResourceNotFoundError from services.annotation_service import AppAnnotationService from services.app_ref_service import AnnotationRef, AppRef from services.app_service import AppService, CreateAppParams @@ -263,7 +263,7 @@ def test_insert_app_annotation_directly_app_not_found( } # Try to insert annotation with non-existent app - with pytest.raises(NotFound, match="App not found"): + with pytest.raises(AnnotationResourceNotFoundError, match="App not found"): AppAnnotationService.insert_app_annotation_directly( annotation_args, non_existent_app_id, session=db_session_with_containers ) @@ -412,7 +412,7 @@ def test_up_insert_app_annotation_from_message_app_not_found( } # Try to insert annotation with non-existent app - with pytest.raises(NotFound, match="App not found"): + with pytest.raises(AnnotationResourceNotFoundError, match="App not found"): AppAnnotationService.up_insert_app_annotation_from_message( annotation_args, non_existent_app_id, session=db_session_with_containers ) @@ -575,7 +575,7 @@ def test_get_annotation_list_by_app_id_app_not_found( self._mock_current_user(mock_external_service_dependencies, fake.uuid4(), fake.uuid4()) # Try to get annotation list with non-existent app - with pytest.raises(NotFound, match="App not found"): + with pytest.raises(AnnotationResourceNotFoundError, match="App not found"): AppAnnotationService.get_annotation_list_by_app_id( non_existent_app_id, page=1, limit=10, keyword="", session=db_session_with_containers ) @@ -631,7 +631,7 @@ def test_delete_app_annotation_annotation_not_found_for_wrong_app( self._mock_current_user(mock_external_service_dependencies, fake.uuid4(), fake.uuid4()) # Try to delete annotation with a ref that cannot match any annotation row - with pytest.raises(NotFound, match="Annotation not found"): + with pytest.raises(AnnotationResourceNotFoundError, match="Annotation not found"): AppAnnotationService.delete_app_annotation(app_ref, db_session_with_containers) def test_delete_app_annotation_annotation_not_found( @@ -645,7 +645,7 @@ def test_delete_app_annotation_annotation_not_found( non_existent_annotation_id = fake.uuid4() # Try to delete non-existent annotation - with pytest.raises(NotFound, match="Annotation not found"): + with pytest.raises(AnnotationResourceNotFoundError, match="Annotation not found"): AppAnnotationService.delete_app_annotation( self._annotation_ref(app, non_existent_annotation_id), db_session_with_containers, @@ -756,18 +756,20 @@ def test_get_annotation_hit_histories_success( # Add some hit histories for i in range(3): - AppAnnotationService.add_annotation_history( - annotation_id=annotation.id, - app_id=app.id, - annotation_question=annotation.question, - annotation_content=annotation.content, - query=f"Query {i}: {fake.sentence()}", - user_id=account.id, - message_id=fake.uuid4(), - from_source=ConversationFromSource.CONSOLE, - score=0.8 + (i * 0.1), - session=db_session_with_containers, + db_session_with_containers.add( + AppAnnotationHitHistory( + annotation_id=annotation.id, + app_id=app.id, + annotation_question=annotation.question, + annotation_content=annotation.content, + question=f"Query {i}: {fake.sentence()}", + account_id=account.id, + message_id=fake.uuid4(), + source=ConversationFromSource.CONSOLE, + score=0.8 + (i * 0.1), + ) ) + db_session_with_containers.flush() # Get hit histories hit_histories, total = AppAnnotationService.get_annotation_hit_histories( @@ -787,99 +789,6 @@ def test_get_annotation_hit_histories_success( assert history.app_id == app.id assert history.account_id == account.id - def test_add_annotation_history_success( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test successful addition of annotation history. - """ - fake = Faker() - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - - # Create an annotation first - annotation_args = { - "question": fake.sentence(), - "answer": fake.text(max_nb_chars=200), - } - annotation = AppAnnotationService.insert_app_annotation_directly( - annotation_args, app.id, session=db_session_with_containers - ) - - # Get initial hit count - initial_hit_count = annotation.hit_count - - # Add annotation history - query = fake.sentence() - message_id = fake.uuid4() - score = 0.85 - - AppAnnotationService.add_annotation_history( - annotation_id=annotation.id, - app_id=app.id, - annotation_question=annotation.question, - annotation_content=annotation.content, - query=query, - user_id=account.id, - message_id=message_id, - from_source=ConversationFromSource.CONSOLE, - score=score, - session=db_session_with_containers, - ) - - # Verify hit count was incremented - - db_session_with_containers.refresh(annotation) - assert annotation.hit_count == initial_hit_count + 1 - - # Verify history was created - from models.model import AppAnnotationHitHistory - - history = ( - db_session_with_containers.query(AppAnnotationHitHistory) - .where( - AppAnnotationHitHistory.annotation_id == annotation.id, AppAnnotationHitHistory.message_id == message_id - ) - .first() - ) - - assert history is not None - assert history.app_id == app.id - assert history.account_id == account.id - assert history.question == query - assert history.score == score - assert history.source == "console" - - def test_get_annotation_by_id_success( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test successful retrieval of annotation by ID. - """ - fake = Faker() - app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) - - # Create an annotation - annotation_args = { - "question": fake.sentence(), - "answer": fake.text(max_nb_chars=200), - } - created_annotation = AppAnnotationService.insert_app_annotation_directly( - annotation_args, app.id, session=db_session_with_containers - ) - - # Get annotation by ID - retrieved_annotation = AppAnnotationService.get_annotation_by_id( - created_annotation.id, session=db_session_with_containers - ) - - # Verify annotation was retrieved correctly - assert retrieved_annotation is not None - assert retrieved_annotation.id == created_annotation.id - assert retrieved_annotation.app_id == app.id - assert retrieved_annotation.question == annotation_args["question"] - assert retrieved_annotation.content == annotation_args["answer"] - assert retrieved_annotation.account_id == account.id - @config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) def test_batch_import_app_annotations_success( self, db_session_with_containers: Session, mock_external_service_dependencies @@ -1167,7 +1076,7 @@ def test_export_annotation_list_by_app_id_app_not_found( self._mock_current_user(mock_external_service_dependencies, fake.uuid4(), fake.uuid4()) # Try to export annotation list with non-existent app - with pytest.raises(NotFound, match="App not found"): + with pytest.raises(AnnotationResourceNotFoundError, match="App not found"): AppAnnotationService.export_annotation_list_by_app_id(non_existent_app_id, db_session_with_containers) def test_insert_app_annotation_directly_with_setting_success( diff --git a/api/tests/test_containers_integration_tests/services/test_app_dsl_service.py b/api/tests/test_containers_integration_tests/services/test_app_dsl_service.py index 786d0659370024..5bcb0028462282 100644 --- a/api/tests/test_containers_integration_tests/services/test_app_dsl_service.py +++ b/api/tests/test_containers_integration_tests/services/test_app_dsl_service.py @@ -19,6 +19,8 @@ TRIGGER_SCHEDULE_NODE_TYPE, TRIGGER_WEBHOOK_NODE_TYPE, ) +from enums.agent import WorkflowAgentBindingType +from extensions.application_services.workflow import build_app_dsl_service from extensions.ext_redis import redis_client from graphon.enums import BuiltinNodeTypes from models import Account, App, AppMode @@ -31,16 +33,15 @@ AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import AgentSoulConfig from models.model import AppModelConfig, IconType from models.workflow import Workflow, WorkflowType +from repositories.app.creation_records import create_site_record from services import app_dsl_service from services.agent.dsl_entities import AGENT_PACKAGE_REF_KEY, make_portable_agent_package from services.agent.dsl_service import AgentDslService -from services.app_creation_records import create_site_record from services.app_dsl_service import ( CHECK_DEPENDENCIES_REDIS_KEY_PREFIX, CURRENT_DSL_VERSION, @@ -119,6 +120,7 @@ def mock_external_service_dependencies(self): """Mock setup for external service dependencies.""" with ( patch("services.app_dsl_service.WorkflowService") as mock_workflow_service, + patch("extensions.application_services.workflow.build_workflow_draft_service") as mock_drafts, patch("services.app_dsl_service.DependenciesAnalysisService") as mock_dependencies_service, patch("services.app_dsl_service.app_was_created") as mock_app_was_created, patch("services.app_service.ModelManager.for_tenant") as mock_model_manager, @@ -126,7 +128,7 @@ def mock_external_service_dependencies(self): patch("services.app_service.EnterpriseService") as mock_enterprise_service, ): mock_workflow_service.return_value.get_draft_workflow.return_value = None - mock_workflow_service.return_value.sync_draft_workflow.return_value = MagicMock() + mock_drafts.return_value.sync.return_value = None mock_dependencies_service.generate_latest_dependencies.return_value = [] mock_dependencies_service.get_leaked_dependencies.return_value = [] mock_dependencies_service.generate_dependencies.return_value = [] @@ -144,6 +146,7 @@ def mock_external_service_dependencies(self): mock_enterprise_service.WebAppAuth.cleanup_webapp.return_value = None yield { "workflow_service": mock_workflow_service, + "drafts": mock_drafts, "dependencies_service": mock_dependencies_service, "app_was_created": mock_app_was_created, "model_manager": mock_model_manager, @@ -233,7 +236,7 @@ def test_check_version_compatibility_equal_returns_completed(self): # ── Import: Validation ──────────────────────────────────────────── def test_import_app_invalid_import_mode_raises_value_error(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with pytest.raises(ValueError, match="Invalid import_mode"): service.import_app( account=_account(), @@ -242,7 +245,7 @@ def test_import_app_invalid_import_mode_raises_value_error(self, db_session_with ) def test_import_app_missing_yaml_content(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -252,7 +255,7 @@ def test_import_app_missing_yaml_content(self, db_session_with_containers: Sessi assert "yaml_content is required" in result.error def test_import_app_missing_yaml_url(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_URL, @@ -262,7 +265,7 @@ def test_import_app_missing_yaml_url(self, db_session_with_containers: Session): assert "yaml_url is required" in result.error def test_import_app_yaml_not_mapping_returns_failed(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -272,7 +275,7 @@ def test_import_app_yaml_not_mapping_returns_failed(self, db_session_with_contai assert "content must be a mapping" in result.error def test_import_app_version_not_str_returns_failed(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) yaml_content = _yaml_dump({"version": 1, "kind": "app", "app": {"name": "x", "mode": "workflow"}}) result = service.import_app( account=_account(), @@ -283,7 +286,7 @@ def test_import_app_version_not_str_returns_failed(self, db_session_with_contain assert "Invalid version type" in result.error def test_import_app_missing_app_data_returns_failed(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -300,7 +303,7 @@ def bad_safe_load(_content: str): monkeypatch.setattr(app_dsl_service.yaml, "safe_load", bad_safe_load) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -318,7 +321,7 @@ def test_import_app_unexpected_error_returns_failed( lambda *_args, **_kwargs: (_ for _ in ()).throw(ValueError("oops")), ) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -338,7 +341,7 @@ def test_import_app_yaml_url_fetch_error_returns_failed( lambda _method, _url, **_kw: (_ for _ in ()).throw(RuntimeError("boom")), ) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_URL, @@ -355,7 +358,7 @@ def test_import_app_yaml_url_empty_content_returns_failed( response.raise_for_status.return_value = None monkeypatch.setattr(app_dsl_service.remote_fetcher, "make_request", lambda _method, _url, **_kw: response) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_URL, @@ -372,7 +375,7 @@ def test_import_app_yaml_url_file_too_large_returns_failed( response.raise_for_status.return_value = None monkeypatch.setattr(app_dsl_service.remote_fetcher, "make_request", lambda _method, _url, **_kw: response) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_URL, @@ -399,7 +402,7 @@ def fake_make_request(method: str, url: str, **kwargs): monkeypatch.setattr(app_dsl_service.remote_fetcher, "make_request", fake_make_request) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_URL, @@ -430,7 +433,7 @@ def fake_make_request(method: str, url: str, **kwargs): monkeypatch.setattr(app_dsl_service.remote_fetcher, "make_request", fake_make_request) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_URL, @@ -443,7 +446,7 @@ def fake_make_request(method: str, url: str, **kwargs): # ── Import: App ID checks ──────────────────────────────────────── def test_import_app_app_id_not_found_returns_failed(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -459,7 +462,7 @@ def test_import_app_overwrite_only_allows_workflow_and_advanced_chat( app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) assert app.mode == "chat" - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=account, import_mode=ImportMode.YAML_CONTENT, @@ -472,7 +475,7 @@ def test_import_app_overwrite_only_allows_workflow_and_advanced_chat( # ── Import: Flow ────────────────────────────────────────────────── def test_import_app_pending_stores_import_info_in_redis(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -508,7 +511,7 @@ def test_import_app_completed_uses_declared_dependencies( } ] - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=account, import_mode=ImportMode.YAML_CONTENT, @@ -557,13 +560,6 @@ def test_import_app_legacy_versions_extract_dependencies( lambda *_args, **_kwargs: created_app, ) - draft_var_service = MagicMock() - monkeypatch.setattr( - app_dsl_service, - "WorkflowDraftVariableService", - lambda *args, **kwargs: draft_var_service, - ) - data: dict = { "version": "0.1.5", "kind": "app", @@ -574,19 +570,18 @@ def test_import_app_legacy_versions_extract_dependencies( else: data["model_config"] = {"model": {"provider": "openai"}} - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, yaml_content=_yaml_dump(data), ) assert result.status == ImportStatus.COMPLETED_WITH_WARNINGS - draft_var_service.delete_app_workflow_variables.assert_called_once_with(app_id=created_app.id) # ── Confirm Import ──────────────────────────────────────────────── def test_confirm_import_expired_returns_failed(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.confirm_import(import_id=str(uuid4()), account=_account()) assert result.status == ImportStatus.FAILED assert "expired" in result.error @@ -622,7 +617,7 @@ def test_confirm_import_success_deletes_redis_key( lambda *_args, **_kwargs: created_app, ) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.confirm_import(import_id=import_id, account=_account()) assert result.status == ImportStatus.COMPLETED assert result.app_id == created_app.id @@ -633,7 +628,7 @@ def test_confirm_import_invalid_pending_data_type_returns_failed(self, db_sessio redis_key = f"{IMPORT_INFO_REDIS_KEY_PREFIX}{import_id}" redis_client.setex(redis_key, IMPORT_INFO_REDIS_EXPIRY, "123") - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.confirm_import(import_id=import_id, account=_account()) assert result.status == ImportStatus.FAILED assert "validation error" in result.error @@ -643,14 +638,14 @@ def test_confirm_import_exception_returns_failed(self, db_session_with_container redis_key = f"{IMPORT_INFO_REDIS_KEY_PREFIX}{import_id}" redis_client.setex(redis_key, IMPORT_INFO_REDIS_EXPIRY, "not-valid-json") - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.confirm_import(import_id=import_id, account=_account()) assert result.status == ImportStatus.FAILED # ── Check Dependencies ──────────────────────────────────────────── def test_check_dependencies_returns_empty_when_no_redis_data(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) app_model = _app_record() result = service.check_dependencies(app_model=app_model) assert result.leaked_dependencies == [] @@ -681,7 +676,7 @@ def test_check_dependencies_calls_analysis_service( lambda *, tenant_id, dependencies: [dep], ) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.check_dependencies(app_model=_app_record(id=app_id)) assert len(result.leaked_dependencies) == 1 @@ -697,14 +692,14 @@ def test_check_dependencies_with_real_app( mock_dependencies_json, ) - dsl_service = AppDslService(db_session_with_containers) + dsl_service = build_app_dsl_service(db_session_with_containers) result = dsl_service.check_dependencies(app_model=app) assert result.leaked_dependencies == [] # ── Create/Update App ───────────────────────────────────────────── def test_create_or_update_app_missing_mode_raises(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with pytest.raises(ValueError, match="loss app mode"): service._create_or_update_app(app=None, data={"app": {}}, account=_account()) @@ -738,7 +733,7 @@ def test_create_or_update_app_existing_app_updates_fields( updated_by=None, updated_at=None, ) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) updated = service._create_or_update_app( app=app, data={ @@ -765,7 +760,7 @@ def test_create_or_update_app_existing_app_updates_fields( def test_create_or_update_app_new_app_requires_tenant(self, db_session_with_containers: Session): account = _account() account._current_tenant = None - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with pytest.raises(ValueError, match="Current tenant is not set"): service._create_or_update_app( app=None, @@ -781,7 +776,7 @@ def test_create_or_update_app_creates_workflow_app_and_saves_dependencies( mock_wf_svc = mock_external_service_dependencies["workflow_service"] mock_wf_svc.return_value.get_draft_workflow.return_value = MagicMock(unique_hash="uh") - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) deps = [ app_dsl_service.PluginDependency.model_validate( { @@ -805,13 +800,13 @@ def test_create_or_update_app_creates_workflow_app_and_saves_dependencies( assert app.tenant_id == account.current_tenant_id mock_external_service_dependencies["app_was_created"].send.assert_called_once() - mock_wf_svc.return_value.sync_draft_workflow.assert_called_once() + mock_external_service_dependencies["drafts"].return_value.sync.assert_called_once() stored = redis_client.get(f"{CHECK_DEPENDENCIES_REDIS_KEY_PREFIX}{app.id}") assert stored is not None def test_create_or_update_app_workflow_missing_workflow_data_raises(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with pytest.raises(ValueError, match="Missing workflow data"): service._create_or_update_app( app=_app_record(mode=AppMode.WORKFLOW), @@ -820,7 +815,7 @@ def test_create_or_update_app_workflow_missing_workflow_data_raises(self, db_ses ) def test_create_or_update_app_chat_requires_model_config(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with pytest.raises(ValueError, match="Missing model_config"): service._create_or_update_app( app=_app_record(mode=AppMode.CHAT), @@ -835,7 +830,7 @@ def test_create_or_update_app_chat_creates_model_config_and_sends_event( app.app_model_config_id = None db_session_with_containers.commit() - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with patch("services.app_dsl_service.app_model_config_was_updated") as signal: service._create_or_update_app( app=app, @@ -851,7 +846,7 @@ def test_create_or_update_app_chat_creates_model_config_and_sends_event( assert signal.send.call_args.kwargs["session"] is db_session_with_containers def test_create_or_update_app_invalid_mode_raises(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with pytest.raises(ValueError, match="Invalid app mode"): service._create_or_update_app( app=_app_record(mode=AppMode.RAG_PIPELINE), @@ -1202,7 +1197,7 @@ def test_agent_app_dsl_round_trip_creates_unpublished_imported_agent( mock_external_service_dependencies["app_was_created"].send.side_effect = ( lambda app, *, account, session, **_kwargs: create_site_record(app=app, account=account, session=session) ) - result = AppDslService(db_session_with_containers).import_app( + result = build_app_dsl_service(db_session_with_containers).import_app( account=account, import_mode=ImportMode.YAML_CONTENT, yaml_content=yaml_content, diff --git a/api/tests/test_containers_integration_tests/services/test_app_generate_service.py b/api/tests/test_containers_integration_tests/services/test_app_generate_service.py index a7eec53a182d33..b0c0f6d315ddd3 100644 --- a/api/tests/test_containers_integration_tests/services/test_app_generate_service.py +++ b/api/tests/test_containers_integration_tests/services/test_app_generate_service.py @@ -6,7 +6,7 @@ import pytest import sqlalchemy as sa from faker import Faker -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from core.app.entities.app_invoke_entities import InvokeFrom from enums import DeploymentEdition @@ -16,9 +16,17 @@ from models.workflow import Workflow from services.app_generate_service import AppGenerateService from services.errors.app import WorkflowIdFormatError, WorkflowNotFoundError +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.test_containers_integration_tests.helpers import generate_valid_password +@pytest.fixture +def workflow_runtime(db_session_with_containers): + from extensions.application_services.workflow import build_workflow_execution_dependencies + + return build_workflow_execution_dependencies(sessionmaker(bind=db_session_with_containers.get_bind())) + + class TestAppGenerateService: """Integration tests for AppGenerateService using testcontainers.""" @@ -27,7 +35,17 @@ def mock_external_service_dependencies(self): """Mock setup for external service dependencies.""" with ( patch("services.billing_service.BillingService", autospec=True) as mock_billing_service, - patch("services.app_generate_service.WorkflowService", autospec=True) as mock_workflow_service, + patch( + "repositories.workflow.definition_repository.WorkflowDefinitionStore.get_draft_workflow", autospec=True + ) as mock_draft_workflow, + patch( + "repositories.workflow.definition_repository.WorkflowDefinitionStore.get_published_workflow", + autospec=True, + ) as mock_published_workflow, + patch( + "repositories.workflow.definition_repository.WorkflowDefinitionStore.get_published_workflow_by_id", + autospec=True, + ) as mock_workflow_by_id, patch("services.app_generate_service.RateLimit", autospec=True) as mock_rate_limit, patch("services.app_generate_service.CompletionAppGenerator", autospec=True) as mock_completion_generator, patch("services.app_generate_service.ChatAppGenerator", autospec=True) as mock_chat_generator, @@ -58,8 +76,7 @@ def mock_external_service_dependencies(self): "refunded": 0, } - # Setup default mock returns for workflow service - mock_workflow_service_instance = mock_workflow_service.return_value + # Supply persisted workflow reads while keeping execution selection intact. graph = json.dumps( { "nodes": [ @@ -70,10 +87,10 @@ def mock_external_service_dependencies(self): } ) published_workflow = Workflow(id=str(uuid.uuid4()), version="1", graph=graph) - mock_workflow_service_instance.get_published_workflow.return_value = published_workflow + mock_published_workflow.return_value = published_workflow draft_workflow = Workflow(id=str(uuid.uuid4()), version="draft", graph=graph) - mock_workflow_service_instance.get_draft_workflow.return_value = draft_workflow - mock_workflow_service_instance.get_published_workflow_by_id.return_value = published_workflow + mock_draft_workflow.return_value = draft_workflow + mock_workflow_by_id.return_value = published_workflow # Setup default mock returns for rate limiting mock_rate_limit_instance = mock_rate_limit.return_value @@ -131,7 +148,9 @@ def mock_external_service_dependencies(self): yield { "billing_service": mock_billing_service, - "workflow_service": mock_workflow_service, + "get_draft_workflow": mock_draft_workflow, + "get_published_workflow": mock_published_workflow, + "get_published_workflow_by_id": mock_workflow_by_id, "rate_limit": mock_rate_limit, "completion_generator": mock_completion_generator, "chat_generator": mock_chat_generator, @@ -228,7 +247,12 @@ def _create_test_workflow(self, db_session_with_containers: Session, app: App): return workflow def test_generate_completion_mode_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful generation for completion mode app. @@ -243,12 +267,14 @@ def test_generate_completion_mode_success( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -262,7 +288,14 @@ def test_generate_completion_mode_success( mock_external_service_dependencies["completion_generator"].return_value.generate.assert_called_once() mock_external_service_dependencies["completion_generator"].convert_to_event_stream.assert_called_once() - def test_generate_chat_mode_success(self, db_session_with_containers: Session, mock_external_service_dependencies): + def test_generate_chat_mode_success( + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, + ): """ Test successful generation for chat mode app. """ @@ -276,12 +309,14 @@ def test_generate_chat_mode_success(self, db_session_with_containers: Session, m # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -292,7 +327,12 @@ def test_generate_chat_mode_success(self, db_session_with_containers: Session, m mock_external_service_dependencies["chat_generator"].convert_to_event_stream.assert_called_once() def test_generate_agent_chat_mode_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful generation for agent chat mode app. @@ -307,12 +347,14 @@ def test_generate_agent_chat_mode_success( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -323,7 +365,12 @@ def test_generate_agent_chat_mode_success( mock_external_service_dependencies["agent_chat_generator"].convert_to_event_stream.assert_called_once() def test_generate_advanced_chat_mode_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful generation for advanced chat mode app. @@ -338,12 +385,14 @@ def test_generate_advanced_chat_mode_success( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -356,7 +405,12 @@ def test_generate_advanced_chat_mode_success( ].return_value.convert_to_event_stream.assert_called_once() def test_generate_workflow_mode_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful generation for workflow mode app. @@ -371,12 +425,14 @@ def test_generate_workflow_mode_success( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -387,7 +443,12 @@ def test_generate_workflow_mode_success( mock_external_service_dependencies["workflow_generator"].convert_to_event_stream.assert_called_once() def test_generate_with_specific_workflow_id( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with a specific workflow ID. @@ -408,24 +469,29 @@ def test_generate_with_specific_workflow_id( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result assert result == ["test_response"] - # Verify workflow service was called with specific workflow ID - mock_external_service_dependencies[ - "workflow_service" - ].return_value.get_published_workflow_by_id.assert_called_once() + # Verify the repository selected the specific workflow ID + mock_external_service_dependencies["get_published_workflow_by_id"].assert_called_once() def test_generate_with_debugger_invoke_from( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with debugger invoke from. @@ -440,22 +506,29 @@ def test_generate_with_debugger_invoke_from( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result assert result == ["test_response"] # Verify draft workflow was fetched for debugger - mock_external_service_dependencies["workflow_service"].return_value.get_draft_workflow.assert_called_once() + mock_external_service_dependencies["get_draft_workflow"].assert_called_once() def test_generate_with_non_streaming_mode( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with non-streaming mode. @@ -470,12 +543,14 @@ def test_generate_with_non_streaming_mode( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -484,7 +559,14 @@ def test_generate_with_non_streaming_mode( # Verify rate limit exit was called for non-streaming mode mock_external_service_dependencies["rate_limit"].return_value.exit.assert_called_once() - def test_generate_with_end_user(self, db_session_with_containers: Session, mock_external_service_dependencies): + def test_generate_with_end_user( + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, + ): """ Test generation with EndUser instead of Account. """ @@ -512,19 +594,26 @@ def test_generate_with_end_user(self, db_session_with_containers: Session, mock_ # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=end_user, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result assert result == ["test_response"] def test_generate_in_cloud_sandbox_plan( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation in the Cloud edition with a sandbox plan. @@ -544,12 +633,14 @@ def test_generate_in_cloud_sandbox_plan( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -561,7 +652,12 @@ def test_generate_in_cloud_sandbox_plan( billing.quota_commit.assert_called_once() def test_generate_with_invalid_app_mode( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with invalid app mode. @@ -583,16 +679,23 @@ def test_generate_with_invalid_app_mode( # StatementError (from EnumText validation during autoflush) with pytest.raises((ValueError, sa.exc.StatementError)): AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) def test_generate_with_workflow_id_format_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with invalid workflow ID format. @@ -612,19 +715,26 @@ def test_generate_with_workflow_id_format_error( # Execute the method under test and expect WorkflowIdFormatError with pytest.raises(WorkflowIdFormatError) as exc_info: AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify error message assert "Invalid workflow_id format" in str(exc_info.value) def test_generate_with_workflow_not_found_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation when workflow is not found. @@ -636,10 +746,8 @@ def test_generate_with_workflow_not_found_error( workflow_id = str(uuid.uuid4()) - # Setup workflow service to return None (workflow not found) - mock_external_service_dependencies[ - "workflow_service" - ].return_value.get_published_workflow_by_id.return_value = None + # Return no persisted workflow for the requested ID + mock_external_service_dependencies["get_published_workflow_by_id"].return_value = None # Setup test arguments args = { @@ -651,19 +759,26 @@ def test_generate_with_workflow_not_found_error( # Execute the method under test and expect WorkflowNotFoundError with pytest.raises(WorkflowNotFoundError) as exc_info: AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify error message assert f"Workflow not found with id: {workflow_id}" in str(exc_info.value) def test_generate_with_workflow_not_initialized_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation when workflow is not initialized for debugger. @@ -674,7 +789,7 @@ def test_generate_with_workflow_not_initialized_error( ) # Setup workflow service to return None (workflow not initialized) - mock_external_service_dependencies["workflow_service"].return_value.get_draft_workflow.return_value = None + mock_external_service_dependencies["get_draft_workflow"].return_value = None # Setup test arguments args = {"inputs": {"query": fake.text(max_nb_chars=50)}, "response_mode": "streaming"} @@ -682,19 +797,26 @@ def test_generate_with_workflow_not_initialized_error( # Execute the method under test and expect ValueError with pytest.raises(ValueError) as exc_info: AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify error message assert "Workflow not initialized" in str(exc_info.value) def test_generate_with_workflow_not_published_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation when workflow is not published for non-debugger. @@ -705,7 +827,7 @@ def test_generate_with_workflow_not_published_error( ) # Setup workflow service to return None (workflow not published) - mock_external_service_dependencies["workflow_service"].return_value.get_published_workflow.return_value = None + mock_external_service_dependencies["get_published_workflow"].return_value = None # Setup test arguments args = {"inputs": {"query": fake.text(max_nb_chars=50)}, "response_mode": "streaming"} @@ -713,19 +835,26 @@ def test_generate_with_workflow_not_published_error( # Execute the method under test and expect ValueError with pytest.raises(ValueError) as exc_info: AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify error message assert "Workflow not published" in str(exc_info.value) def test_generate_single_iteration_advanced_chat_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful single iteration generation for advanced chat mode. @@ -740,12 +869,14 @@ def test_generate_single_iteration_advanced_chat_success( # Execute the method under test result = AppGenerateService.generate_single_iteration( + runtime=workflow_runtime, + workflow=mock_external_service_dependencies["get_draft_workflow"].return_value, app_model=app, user=account, node_id=node_id, args=args, streaming=True, - session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -757,7 +888,12 @@ def test_generate_single_iteration_advanced_chat_success( ].return_value.single_iteration_generate.assert_called_once() def test_generate_single_iteration_workflow_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful single iteration generation for workflow mode. @@ -772,12 +908,14 @@ def test_generate_single_iteration_workflow_success( # Execute the method under test result = AppGenerateService.generate_single_iteration( + runtime=workflow_runtime, + workflow=mock_external_service_dependencies["get_draft_workflow"].return_value, app_model=app, user=account, node_id=node_id, args=args, streaming=True, - session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -789,7 +927,12 @@ def test_generate_single_iteration_workflow_success( ].return_value.single_iteration_generate.assert_called_once() def test_generate_single_iteration_invalid_mode( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test single iteration generation with invalid app mode. @@ -805,19 +948,26 @@ def test_generate_single_iteration_invalid_mode( # Execute the method under test and expect ValueError with pytest.raises(ValueError) as exc_info: AppGenerateService.generate_single_iteration( + runtime=workflow_runtime, + workflow=mock_external_service_dependencies["get_draft_workflow"].return_value, app_model=app, user=account, node_id=node_id, args=args, streaming=True, - session=db_session_with_containers, + variables=workflow_variables, ) # Verify error message assert "Invalid app mode" in str(exc_info.value) def test_generate_single_loop_advanced_chat_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful single loop generation for advanced chat mode. @@ -832,12 +982,14 @@ def test_generate_single_loop_advanced_chat_success( # Execute the method under test result = AppGenerateService.generate_single_loop( + runtime=workflow_runtime, + workflow=mock_external_service_dependencies["get_draft_workflow"].return_value, app_model=app, user=account, node_id=node_id, args=args, streaming=True, - session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -849,7 +1001,12 @@ def test_generate_single_loop_advanced_chat_success( ].return_value.single_loop_generate.assert_called_once() def test_generate_single_loop_workflow_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful single loop generation for workflow mode. @@ -864,12 +1021,14 @@ def test_generate_single_loop_workflow_success( # Execute the method under test result = AppGenerateService.generate_single_loop( + runtime=workflow_runtime, + workflow=mock_external_service_dependencies["get_draft_workflow"].return_value, app_model=app, user=account, node_id=node_id, args=args, streaming=True, - session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -879,7 +1038,12 @@ def test_generate_single_loop_workflow_success( mock_external_service_dependencies["workflow_generator"].return_value.single_loop_generate.assert_called_once() def test_generate_single_loop_invalid_mode( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test single loop generation with invalid app mode. @@ -895,19 +1059,21 @@ def test_generate_single_loop_invalid_mode( # Execute the method under test and expect ValueError with pytest.raises(ValueError) as exc_info: AppGenerateService.generate_single_loop( + runtime=workflow_runtime, + workflow=mock_external_service_dependencies["get_draft_workflow"].return_value, app_model=app, user=account, node_id=node_id, args=args, streaming=True, - session=db_session_with_containers, + variables=workflow_variables, ) # Verify error message assert "Invalid app mode" in str(exc_info.value) def test_generate_more_like_this_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, workflow_runtime, db_session_with_containers: Session, mock_external_service_dependencies ): """ Test successful more like this generation. @@ -921,6 +1087,9 @@ def test_generate_more_like_this_success( # Execute the method under test result = AppGenerateService.generate_more_like_this( + retrieval=workflow_runtime.retrieval, + records=workflow_runtime.chat_records, + annotations=workflow_runtime.annotation_replies, session=db_session_with_containers, app_model=app, user=account, @@ -938,7 +1107,7 @@ def test_generate_more_like_this_success( ].return_value.generate_more_like_this.assert_called_once() def test_generate_more_like_this_with_end_user( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, workflow_runtime, db_session_with_containers: Session, mock_external_service_dependencies ): """ Test more like this generation with EndUser. @@ -966,6 +1135,9 @@ def test_generate_more_like_this_with_end_user( # Execute the method under test result = AppGenerateService.generate_more_like_this( + retrieval=workflow_runtime.retrieval, + records=workflow_runtime.chat_records, + annotations=workflow_runtime.annotation_replies, session=db_session_with_containers, app_model=app, user=end_user, @@ -1039,7 +1211,12 @@ def test_get_max_active_requests_with_zero_limits( assert result == 100 # dify_config.APP_MAX_ACTIVE_REQUESTS def test_generate_with_exception_cleanup( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test that rate limit exit is called when an exception occurs. @@ -1060,12 +1237,14 @@ def test_generate_with_exception_cleanup( # Execute the method under test and expect exception with pytest.raises(Exception) as exc_info: AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify exception message @@ -1075,7 +1254,12 @@ def test_generate_with_exception_cleanup( mock_external_service_dependencies["rate_limit"].return_value.exit.assert_called_once() def test_generate_with_agent_mode_detection( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with agent mode detection based on app configuration. @@ -1093,12 +1277,14 @@ def test_generate_with_agent_mode_detection( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -1109,7 +1295,12 @@ def test_generate_with_agent_mode_detection( mock_external_service_dependencies["agent_chat_generator"].convert_to_event_stream.assert_called_once() def test_generate_with_different_invoke_from_values( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with different invoke from values. @@ -1133,18 +1324,27 @@ def test_generate_with_different_invoke_from_values( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=invoke_from, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result assert result == ["test_response"] - def test_generate_with_complex_args(self, db_session_with_containers: Session, mock_external_service_dependencies): + def test_generate_with_complex_args( + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, + ): """ Test generation with complex arguments including files and external trace ID. """ @@ -1176,12 +1376,14 @@ def test_generate_with_complex_args(self, db_session_with_containers: Session, m mock_exec_params.new.return_value = mock_payload result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result diff --git a/api/tests/test_containers_integration_tests/services/test_conversation_variable_updater.py b/api/tests/test_containers_integration_tests/services/test_conversation_variable_updater.py index 638a962f18bd9f..b17450d1b85f89 100644 --- a/api/tests/test_containers_integration_tests/services/test_conversation_variable_updater.py +++ b/api/tests/test_containers_integration_tests/services/test_conversation_variable_updater.py @@ -1,4 +1,4 @@ -"""Testcontainers integration tests for ConversationVariableUpdater.""" +"""Testcontainers integration tests for WorkflowConversationVariableRepository.""" from uuid import uuid4 @@ -8,10 +8,13 @@ from extensions.ext_database import db from graphon.variables import StringVariable from models.workflow import ConversationVariable -from services.conversation_variable_updater import ConversationVariableNotFoundError, ConversationVariableUpdater +from repositories.workflow.conversation_variable_repository import ( + ConversationVariableNotFoundError, + WorkflowConversationVariableRepository, +) -class TestConversationVariableUpdater: +class TestWorkflowConversationVariableRepository: def _create_conversation_variable( self, db_session_with_containers: Session, @@ -38,7 +41,7 @@ def test_should_update_conversation_variable_data_and_commit(self, db_session_wi ) updated_variable = StringVariable(id=variable.id, name="topic", value="new value") - updater = ConversationVariableUpdater(sessionmaker(bind=db.engine)) + updater = WorkflowConversationVariableRepository(sessionmaker(bind=db.engine)) updater.update(conversation_id=conversation_id, variable=updated_variable) @@ -50,14 +53,7 @@ def test_should_update_conversation_variable_data_and_commit(self, db_session_wi def test_should_raise_not_found_when_variable_missing(self, db_session_with_containers: Session): conversation_id = str(uuid4()) variable = StringVariable(id=str(uuid4()), name="topic", value="value") - updater = ConversationVariableUpdater(sessionmaker(bind=db.engine)) + updater = WorkflowConversationVariableRepository(sessionmaker(bind=db.engine)) with pytest.raises(ConversationVariableNotFoundError, match="conversation variable not found in the database"): updater.update(conversation_id=conversation_id, variable=variable) - - def test_should_do_nothing_when_flush_is_called(self, db_session_with_containers: Session): - updater = ConversationVariableUpdater(sessionmaker(bind=db.engine)) - - result = updater.flush() - - assert result is None diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service_document.py b/api/tests/test_containers_integration_tests/services/test_dataset_service_document.py index e221eaf4be3bb0..1ef9540116e8cb 100644 --- a/api/tests/test_containers_integration_tests/services/test_dataset_service_document.py +++ b/api/tests/test_containers_integration_tests/services/test_dataset_service_document.py @@ -7,7 +7,6 @@ import pytest from sqlalchemy.orm import Session -from werkzeug.exceptions import Forbidden, NotFound from core.rag.index_processor.constant.index_type import IndexStructureType from extensions.storage.storage_type import StorageType @@ -17,7 +16,10 @@ from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus from models.model import UploadFile from repositories.knowledge.dataset_read_repository import get_dataset_doc_form +from repositories.knowledge.document_repository import next_document_position from services.errors.base import NoPermissionError +from services.errors.dataset import DatasetNotFoundError +from services.errors.document import DocumentAccessDeniedError, DocumentNotFoundError, DocumentSourceNotFoundError from services.knowledge.dataset_service import DocumentService from services.knowledge.resource_scope import DatasetRef @@ -239,7 +241,7 @@ def test_get_upload_file_id_for_upload_file_document_rejects_invalid_source_type data_source_info={"url": "https://example.com"}, ) - with pytest.raises(NotFound, match="invalid source"): + with pytest.raises(DocumentSourceNotFoundError, match="invalid source"): DocumentService._get_upload_file_id_for_upload_file_document( document, invalid_source_message="invalid source", @@ -257,7 +259,7 @@ def test_get_upload_file_id_for_upload_file_document_rejects_missing_upload_file data_source_info={}, ) - with pytest.raises(NotFound, match="missing file"): + with pytest.raises(DocumentSourceNotFoundError, match="missing file"): DocumentService._get_upload_file_id_for_upload_file_document( document, invalid_source_message="invalid source", @@ -293,7 +295,7 @@ def test_get_upload_file_for_upload_file_document_raises_when_file_service_retur ) with patch("services.knowledge.dataset_service.FileService.get_upload_files_by_ids", return_value={}): - with pytest.raises(NotFound, match="Uploaded file not found"): + with pytest.raises(DocumentSourceNotFoundError, match="Uploaded file not found"): DocumentService._get_upload_file_for_upload_file_document(document, session=db_session_with_containers) @@ -320,7 +322,7 @@ def test_get_upload_files_by_document_id_for_zip_download_raises_for_missing_doc ): dataset = DocumentServiceIntegrationFactory.create_dataset(db_session_with_containers) - with pytest.raises(NotFound, match="Document not found"): + with pytest.raises(DocumentNotFoundError, match="Document not found"): DocumentService._get_upload_files_by_document_id_for_zip_download( dataset_id=dataset.id, document_ids=[str(uuid4())], @@ -345,7 +347,7 @@ def test_get_upload_files_by_document_id_for_zip_download_hides_cross_tenant_doc data_source_info={"upload_file_id": upload_file.id}, ) - with pytest.raises(NotFound, match="Document not found"): + with pytest.raises(DocumentNotFoundError, match="Document not found"): DocumentService._get_upload_files_by_document_id_for_zip_download( dataset_id=dataset.id, document_ids=[document.id], @@ -364,7 +366,7 @@ def test_get_upload_files_by_document_id_for_zip_download_rejects_missing_upload data_source_info={"upload_file_id": str(uuid4())}, ) - with pytest.raises(NotFound, match="Only uploaded-file documents can be downloaded as ZIP"): + with pytest.raises(DocumentSourceNotFoundError, match="Only uploaded-file documents can be downloaded as ZIP"): DocumentService._get_upload_files_by_document_id_for_zip_download( dataset_id=dataset.id, document_ids=[document.id], @@ -416,7 +418,7 @@ def test_prepare_document_batch_download_zip_raises_not_found_for_missing_datase current_user_mock, flask_app_with_containers, db_session_with_containers: Session ): with flask_app_with_containers.app_context(): - with pytest.raises(NotFound, match="Dataset not found"): + with pytest.raises(DatasetNotFoundError, match="Dataset not found"): DocumentService.prepare_document_batch_download_zip( dataset_id=str(uuid4()), document_ids=[str(uuid4())], @@ -440,7 +442,7 @@ def test_prepare_document_batch_download_zip_translates_permission_error_to_forb "services.knowledge.dataset_service.DatasetService.check_dataset_permission", side_effect=NoPermissionError("denied"), ): - with pytest.raises(Forbidden, match="denied"): + with pytest.raises(DocumentAccessDeniedError, match="denied"): DocumentService.prepare_document_batch_download_zip( dataset_id=dataset.id, document_ids=[], @@ -692,10 +694,10 @@ def test_get_documents_position_returns_next_position_when_documents_exist(db_se dataset = DocumentServiceIntegrationFactory.create_dataset(db_session_with_containers) DocumentServiceIntegrationFactory.create_document(db_session_with_containers, dataset=dataset, position=3) - assert DocumentService.get_documents_position(dataset.id, session=db_session_with_containers) == 4 + assert next_document_position(dataset.id, session=db_session_with_containers) == 4 def test_get_documents_position_defaults_to_one_when_dataset_is_empty(db_session_with_containers: Session): dataset = DocumentServiceIntegrationFactory.create_dataset(db_session_with_containers) - assert DocumentService.get_documents_position(dataset.id, session=db_session_with_containers) == 1 + assert next_document_position(dataset.id, session=db_session_with_containers) == 1 diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service_permissions.py b/api/tests/test_containers_integration_tests/services/test_dataset_service_permissions.py index 194bca1d8a7cc6..88c41e8e23fa75 100644 --- a/api/tests/test_containers_integration_tests/services/test_dataset_service_permissions.py +++ b/api/tests/test_containers_integration_tests/services/test_dataset_service_permissions.py @@ -21,8 +21,9 @@ DatasetPermissionEnum, ) from models.enums import DataSourceType +from repositories.knowledge.collection_binding_repository import DatasetCollectionBindingRepository from services.errors.base import NoPermissionError -from services.knowledge.dataset_service import DatasetCollectionBindingService, DatasetPermissionService, DatasetService +from services.knowledge.dataset_service import DatasetPermissionService, DatasetService from services.knowledge.resource_scope import DatasetRef from tests.unit_tests.config_override import config_overrides_context @@ -449,7 +450,7 @@ def test_get_dataset_auto_disable_logs_returns_recent_document_ids(self, db_sess assert len(result["document_ids"]) == 2 -class TestDatasetCollectionBindingServiceIntegration: +class TestDatasetCollectionBindingRepositoryIntegration: def test_get_dataset_collection_binding_returns_existing_binding(self, db_session_with_containers: Session): binding = DatasetPermissionIntegrationFactory.create_collection_binding( db_session_with_containers, @@ -457,14 +458,14 @@ def test_get_dataset_collection_binding_returns_existing_binding(self, db_sessio model_name="model", ) - result = DatasetCollectionBindingService.get_dataset_collection_binding( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding( "provider", "model", session=db_session_with_containers ) assert result.id == binding.id def test_get_dataset_collection_binding_creates_binding_when_missing(self, db_session_with_containers: Session): - result = DatasetCollectionBindingService.get_dataset_collection_binding( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding( "provider", "missing-model", session=db_session_with_containers ) @@ -480,7 +481,7 @@ def test_get_dataset_collection_binding_by_id_and_type_raises_when_missing( ): with flask_app_with_containers.app_context(): with pytest.raises(ValueError, match="Dataset collection binding not found"): - DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( str(uuid4()), session=db_session_with_containers ) @@ -491,7 +492,7 @@ def test_get_dataset_collection_binding_by_id_and_type_returns_binding(self, db_ model_name="model", ) - result = DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( binding.id, session=db_session_with_containers ) diff --git a/api/tests/test_containers_integration_tests/services/test_dataset_service_update_dataset.py b/api/tests/test_containers_integration_tests/services/test_dataset_service_update_dataset.py index ebf6fa0f54d20f..667b88a848ca77 100644 --- a/api/tests/test_containers_integration_tests/services/test_dataset_service_update_dataset.py +++ b/api/tests/test_containers_integration_tests/services/test_dataset_service_update_dataset.py @@ -411,7 +411,7 @@ def test_update_internal_dataset_indexing_technique_to_high_quality(self, db_ses patch("services.knowledge.dataset_service.current_user", user), patch("services.knowledge.dataset_service.ModelManager.for_tenant") as mock_model_manager, patch( - "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding" + "services.knowledge.dataset_service.DatasetCollectionBindingRepository.get_dataset_collection_binding" ) as mock_get_binding, patch("services.knowledge.dataset_service.deal_dataset_vector_index_task") as mock_task, ): @@ -506,7 +506,7 @@ def test_update_internal_dataset_embedding_model_update(self, db_session_with_co patch("services.knowledge.dataset_service.current_user", user), patch("services.knowledge.dataset_service.ModelManager.for_tenant") as mock_model_manager, patch( - "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding" + "services.knowledge.dataset_service.DatasetCollectionBindingRepository.get_dataset_collection_binding" ) as mock_get_binding, patch("services.knowledge.dataset_service.deal_dataset_vector_index_task") as mock_task, patch("services.knowledge.dataset_service.regenerate_summary_index_task") as mock_regenerate_task, diff --git a/api/tests/test_containers_integration_tests/services/test_human_input_delivery_test.py b/api/tests/test_containers_integration_tests/services/test_human_input_delivery_test.py index 6dae35ec379b4a..e53a292683fcf8 100644 --- a/api/tests/test_containers_integration_tests/services/test_human_input_delivery_test.py +++ b/api/tests/test_containers_integration_tests/services/test_human_input_delivery_test.py @@ -7,18 +7,19 @@ import pytest from flask.testing import FlaskClient from sqlalchemy import select -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker import services.remote_file_service as remote_file_service_module from enums.human_input import HumanInputFormKind +from extensions.application_services.human_input import build_human_input_debug_service from graphon.enums import BuiltinNodeTypes +from machinery.context import RequestContext from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole from models.human_input import HumanInputForm, HumanInputFormRecipient, HumanInputFormUploadFile from models.human_input_delivery import EmailDeliveryConfig, EmailDeliveryMethod, EmailRecipients, ExternalRecipient from models.human_input_entities import FileInputConfig, HumanInputNodeData from models.model import App, AppMode, UploadFile from models.workflow import Workflow, WorkflowType -from services.workflow_service import WorkflowService def _create_app_with_draft_workflow( @@ -97,6 +98,17 @@ def _create_app_with_draft_workflow( return app, account +def _test_delivery( + *, app_model: App, account: Account, node_id: str, delivery_method_id: str, session: Session +) -> None: + gateway = build_human_input_debug_service(database_client=sessionmaker(session.get_bind(), expire_on_commit=False)) + context = RequestContext("delivery-test", None, account.id, app_model.tenant_id) + app_id = app_model.id + session.expunge_all() + session.rollback() + gateway.test_delivery(context, app_id, node_id, delivery_method_id=delivery_method_id, inputs={}) + + def test_human_input_delivery_test_sends_email( db_session_with_containers, monkeypatch: pytest.MonkeyPatch, @@ -108,8 +120,7 @@ def test_human_input_delivery_test_sends_email( monkeypatch.setattr("services.human_input_delivery_test_service.mail.is_inited", lambda: True) monkeypatch.setattr("services.human_input_delivery_test_service.mail.send", send_mock) - service = WorkflowService() - service.test_human_input_delivery( + _test_delivery( app_model=app, account=account, node_id="human-node", @@ -136,7 +147,7 @@ def test_human_input_delivery_test_form_accepts_file_upload( monkeypatch.setattr("services.human_input_delivery_test_service.mail.is_inited", lambda: True) monkeypatch.setattr("services.human_input_delivery_test_service.mail.send", MagicMock()) - WorkflowService().test_human_input_delivery( + _test_delivery( app_model=app, account=account, node_id="human-node", @@ -205,7 +216,7 @@ def test_human_input_delivery_test_form_accepts_remote_file_upload( monkeypatch.setattr("services.human_input_delivery_test_service.mail.is_inited", lambda: True) monkeypatch.setattr("services.human_input_delivery_test_service.mail.send", MagicMock()) - WorkflowService().test_human_input_delivery( + _test_delivery( app_model=app, account=account, node_id="human-node", diff --git a/api/tests/test_containers_integration_tests/services/test_schedule_service.py b/api/tests/test_containers_integration_tests/services/test_schedule_service.py index aeab080a6a2e59..8d69541cbe82fb 100644 --- a/api/tests/test_containers_integration_tests/services/test_schedule_service.py +++ b/api/tests/test_containers_integration_tests/services/test_schedule_service.py @@ -1,3 +1,5 @@ +from repositories.trigger.workflow_repository import WorkflowTriggerRepository + """Testcontainers integration tests for schedule service SQL-backed behavior.""" from datetime import datetime @@ -10,11 +12,12 @@ from core.workflow.nodes.trigger_schedule.entities import ScheduleConfig, SchedulePlanUpdate from core.workflow.nodes.trigger_schedule.exc import ScheduleNotFoundError -from events.event_handlers.sync_workflow_schedule_when_app_published import sync_schedule_from_workflow from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole from models.trigger import WorkflowSchedulePlan from services.account_errors import AccountNotFoundError from services.trigger.schedule_service import ScheduleService +from services.trigger.workflow_policy import schedule_config +from tests.unit_tests.model_factories import make_app class ScheduleServiceIntegrationFactory: @@ -117,10 +120,10 @@ def test_create_schedule_persists_schedule(self, db_session_with_containers: Ses with pytest.MonkeyPatch.context() as monkeypatch: monkeypatch.setattr( - "services.trigger.schedule_service.calculate_next_run_at", + "repositories.trigger.workflow_repository.calculate_next_run_at", lambda *_args, **_kwargs: expected_next_run, ) - schedule = ScheduleService.create_schedule( + schedule = WorkflowTriggerRepository.create_schedule( session=db_session_with_containers, tenant_id=tenant.id, app_id=str(uuid4()), @@ -147,10 +150,10 @@ def test_update_schedule_updates_fields_and_recomputes_next_run(self, db_session with pytest.MonkeyPatch.context() as monkeypatch: monkeypatch.setattr( - "services.trigger.schedule_service.calculate_next_run_at", + "repositories.trigger.workflow_repository.calculate_next_run_at", lambda *_args, **_kwargs: expected_next_run, ) - updated = ScheduleService.update_schedule( + updated = WorkflowTriggerRepository.update_schedule( session=db_session_with_containers, schedule_id=schedule.id, updates=SchedulePlanUpdate( @@ -180,8 +183,8 @@ def _track[**P](*args: P.args, **kwargs: P.kwargs): calls.append((args, kwargs)) return datetime(2026, 1, 9, 10, 0, 0) - monkeypatch.setattr("services.trigger.schedule_service.calculate_next_run_at", _track) - updated = ScheduleService.update_schedule( + monkeypatch.setattr("repositories.trigger.workflow_repository.calculate_next_run_at", _track) + updated = WorkflowTriggerRepository.update_schedule( session=db_session_with_containers, schedule_id=schedule.id, updates=SchedulePlanUpdate(node_id="node-new"), @@ -194,33 +197,31 @@ def _track[**P](*args: P.args, **kwargs: P.kwargs): def test_update_schedule_not_found_raises(self, db_session_with_containers: Session): with pytest.raises(ScheduleNotFoundError, match="Schedule not found"): - ScheduleService.update_schedule( + WorkflowTriggerRepository.update_schedule( session=db_session_with_containers, schedule_id=str(uuid4()), updates=SchedulePlanUpdate(node_id="node-new"), ) - def test_delete_schedule_removes_row(self, db_session_with_containers: Session): + def test_sync_schedule_without_config_removes_row(self, db_session_with_containers: Session): _account, tenant = ScheduleServiceIntegrationFactory.create_account_with_tenant(db_session_with_containers) schedule = ScheduleServiceIntegrationFactory.create_schedule_plan( - db_session_with_containers, - tenant_id=tenant.id, - ) - - ScheduleService.delete_schedule( - session=db_session_with_containers, - schedule_id=schedule.id, + db_session_with_containers, tenant_id=tenant.id ) + app = make_app(app_id=schedule.app_id, tenant_id=tenant.id) + WorkflowTriggerRepository.sync_schedule(db_session_with_containers, app, None) db_session_with_containers.commit() - assert db_session_with_containers.get(WorkflowSchedulePlan, schedule.id) is None - def test_delete_schedule_not_found_raises(self, db_session_with_containers: Session): - with pytest.raises(ScheduleNotFoundError, match="Schedule not found"): - ScheduleService.delete_schedule( - session=db_session_with_containers, - schedule_id=str(uuid4()), - ) + def test_sync_schedule_without_config_is_idempotent_when_missing(self, db_session_with_containers: Session): + app = make_app(app_id=str(uuid4()), tenant_id=str(uuid4())) + WorkflowTriggerRepository.sync_schedule(db_session_with_containers, app, None) + WorkflowTriggerRepository.sync_schedule(db_session_with_containers, app, None) + db_session_with_containers.commit() + assert ( + db_session_with_containers.scalar(select(WorkflowSchedulePlan).where(WorkflowSchedulePlan.app_id == app.id)) + is None + ) def test_get_tenant_owner_returns_owner_account(self, db_session_with_containers: Session): owner, tenant = ScheduleServiceIntegrationFactory.create_account_with_tenant( @@ -272,35 +273,6 @@ def test_get_tenant_owner_raises_when_no_owner_or_admin_found(self, db_session_w with pytest.raises(AccountNotFoundError, match=tenant.id): ScheduleService.get_tenant_owner(session=db_session_with_containers, tenant_id=tenant.id) - def test_update_next_run_at_updates_persisted_value(self, db_session_with_containers: Session): - _account, tenant = ScheduleServiceIntegrationFactory.create_account_with_tenant(db_session_with_containers) - schedule = ScheduleServiceIntegrationFactory.create_schedule_plan( - db_session_with_containers, - tenant_id=tenant.id, - ) - expected_next_run = datetime(2026, 1, 3, 10, 30, 0) - - with pytest.MonkeyPatch.context() as monkeypatch: - monkeypatch.setattr( - "services.trigger.schedule_service.calculate_next_run_at", - lambda *_args, **_kwargs: expected_next_run, - ) - result = ScheduleService.update_next_run_at( - session=db_session_with_containers, - schedule_id=schedule.id, - ) - - db_session_with_containers.refresh(schedule) - assert result == expected_next_run - assert schedule.next_run_at == expected_next_run - - def test_update_next_run_at_raises_when_schedule_not_found(self, db_session_with_containers: Session): - with pytest.raises(ScheduleNotFoundError, match="Schedule not found"): - ScheduleService.update_next_run_at( - session=db_session_with_containers, - schedule_id=str(uuid4()), - ) - class TestSyncScheduleFromWorkflowIntegration: def test_sync_schedule_create_new(self, db_session_with_containers: Session): @@ -310,16 +282,16 @@ def test_sync_schedule_create_new(self, db_session_with_containers: Session): with pytest.MonkeyPatch.context() as monkeypatch: monkeypatch.setattr( - "services.trigger.schedule_service.calculate_next_run_at", + "repositories.trigger.workflow_repository.calculate_next_run_at", lambda *_args, **_kwargs: expected_next_run, ) - result = sync_schedule_from_workflow( - tenant_id=tenant.id, - app_id=app_id, - workflow=_cron_workflow(), + WorkflowTriggerRepository.sync_schedule( + db_session_with_containers, + make_app(tenant_id=tenant.id, app_id=app_id), + schedule_config(_cron_workflow().graph_dict), ) + db_session_with_containers.commit() - assert result is not None persisted = db_session_with_containers.execute( select(WorkflowSchedulePlan).where(WorkflowSchedulePlan.app_id == app_id) ).scalar_one() @@ -344,20 +316,22 @@ def test_sync_schedule_update_existing(self, db_session_with_containers: Session with pytest.MonkeyPatch.context() as monkeypatch: monkeypatch.setattr( - "services.trigger.schedule_service.calculate_next_run_at", + "repositories.trigger.workflow_repository.calculate_next_run_at", lambda *_args, **_kwargs: expected_next_run, ) - result = sync_schedule_from_workflow( - tenant_id=tenant.id, - app_id=app_id, - workflow=_cron_workflow( - node_id="start", - cron_expression="0 12 * * *", - timezone="America/New_York", + WorkflowTriggerRepository.sync_schedule( + db_session_with_containers, + make_app(tenant_id=tenant.id, app_id=app_id), + schedule_config( + _cron_workflow( + node_id="start", + cron_expression="0 12 * * *", + timezone="America/New_York", + ).graph_dict ), ) + db_session_with_containers.commit() - assert result is not None db_session_with_containers.expire_all() persisted = db_session_with_containers.get(WorkflowSchedulePlan, existing_id) assert persisted is not None @@ -376,12 +350,11 @@ def test_sync_schedule_remove_when_no_config(self, db_session_with_containers: S ) existing_id = existing.id - result = sync_schedule_from_workflow( - tenant_id=tenant.id, - app_id=app_id, - workflow=_no_schedule_workflow(), + WorkflowTriggerRepository.sync_schedule( + db_session_with_containers, + make_app(tenant_id=tenant.id, app_id=app_id), + schedule_config(_no_schedule_workflow().graph_dict), ) - - assert result is None + db_session_with_containers.commit() db_session_with_containers.expire_all() assert db_session_with_containers.get(WorkflowSchedulePlan, existing_id) is None diff --git a/api/tests/test_containers_integration_tests/services/test_webhook_service_relationships.py b/api/tests/test_containers_integration_tests/services/test_webhook_service_relationships.py index acb786fc7585e7..0f4126b13fd240 100644 --- a/api/tests/test_containers_integration_tests/services/test_webhook_service_relationships.py +++ b/api/tests/test_containers_integration_tests/services/test_webhook_service_relationships.py @@ -20,6 +20,7 @@ from models.workflow import Workflow from services.errors.app import QuotaExceededError from services.trigger.webhook_service import WebhookService +from services.trigger.workflow_policy import MAX_WEBHOOK_NODES_PER_WORKFLOW class _EndUserServiceStub: @@ -457,7 +458,7 @@ def test_sync_webhook_relationships_raises_when_workflow_exceeds_node_limit( factory = WebhookServiceRelationshipFactory account, tenant = factory.create_account_and_tenant(db_session_with_containers) app = factory.create_app(db_session_with_containers, tenant, account) - node_ids = [f"node-{index}" for index in range(WebhookService.MAX_WEBHOOK_NODES_PER_WORKFLOW + 1)] + node_ids = [f"node-{index}" for index in range(MAX_WEBHOOK_NODES_PER_WORKFLOW + 1)] workflow = factory.create_workflow( db_session_with_containers, app=app, account=account, node_ids=node_ids, version=Workflow.VERSION_DRAFT ) @@ -465,23 +466,6 @@ def test_sync_webhook_relationships_raises_when_workflow_exceeds_node_limit( with pytest.raises(ValueError, match="maximum webhook node limit"): WebhookService.sync_webhook_relationships(app, workflow) - def test_sync_webhook_relationships_raises_when_lock_not_acquired( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - workflow = factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=["node-1"], version=Workflow.VERSION_DRAFT - ) - lock = MagicMock() - lock.acquire.return_value = False - - with patch("services.trigger.webhook_service.redis_client.lock", return_value=lock): - with pytest.raises(RuntimeError, match="Failed to acquire lock"): - WebhookService.sync_webhook_relationships(app, workflow) - def test_sync_webhook_relationships_creates_missing_records_and_deletes_stale_records( self, db_session_with_containers: Session, flask_app_with_containers: Flask ): @@ -506,7 +490,8 @@ def test_sync_webhook_relationships_creates_missing_records_and_deletes_stale_re ) with patch( - "services.trigger.webhook_service.WebhookService.generate_webhook_id", return_value="new-webhook-id-000001" + "repositories.trigger.workflow_repository.WorkflowTriggerRepository.generate_webhook_id", + return_value="new-webhook-id-000001", ): WebhookService.sync_webhook_relationships(app, workflow) @@ -518,66 +503,3 @@ def test_sync_webhook_relationships_creates_missing_records_and_deletes_stale_re assert [record.node_id for record in records] == ["node-new"] assert records[0].webhook_id == "new-webhook-id-000001" assert db_session_with_containers.get(WorkflowWebhookTrigger, stale_trigger_id) is None - - def test_sync_webhook_relationships_sets_redis_cache_for_new_record( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - workflow = factory.create_workflow( - db_session_with_containers, - app=app, - account=account, - node_ids=["node-cache"], - version=Workflow.VERSION_DRAFT, - ) - cache_key = f"{WebhookService.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:node-cache" - - with patch( - "services.trigger.webhook_service.WebhookService.generate_webhook_id", return_value="cache-webhook-id-00001" - ): - WebhookService.sync_webhook_relationships(app, workflow) - - cached_payload = WebhookServiceRelationshipFactory._read_cache(cache_key) - assert cached_payload is not None - assert cached_payload["node_id"] == "node-cache" - assert cached_payload["webhook_id"] == "cache-webhook-id-00001" - - def test_sync_webhook_relationships_logs_when_lock_release_fails( - self, - db_session_with_containers: Session, - flask_app_with_containers: Flask, - caplog: pytest.LogCaptureFixture, - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - workflow = factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=[], version=Workflow.VERSION_DRAFT - ) - lock = MagicMock() - lock.acquire.return_value = True - lock.release.side_effect = RuntimeError("release failed") - caplog.set_level(logging.ERROR, logger="services.trigger.webhook_service") - - with patch("services.trigger.webhook_service.redis_client.lock", return_value=lock): - WebhookService.sync_webhook_relationships(app, workflow) - - assert caplog.messages.count(f"Failed to release lock for webhook sync, app {app.id}") == 1 - - -def _read_cache(cache_key: str) -> dict[str, str] | None: - from extensions.ext_redis import redis_client - - cached = redis_client.get(cache_key) - if not cached: - return None - if isinstance(cached, bytes): - cached = cached.decode("utf-8") - return json.loads(cached) - - -WebhookServiceRelationshipFactory._read_cache = staticmethod(_read_cache) diff --git a/api/tests/test_containers_integration_tests/services/test_workflow_app_service.py b/api/tests/test_containers_integration_tests/services/test_workflow_app_service.py index caaa7ea54c6d66..b4b47b804fb155 100644 --- a/api/tests/test_containers_integration_tests/services/test_workflow_app_service.py +++ b/api/tests/test_containers_integration_tests/services/test_workflow_app_service.py @@ -13,8 +13,8 @@ from models import EndUser, Workflow, WorkflowAppLog, WorkflowRun from models.enums import CreatorUserRole, EndUserType from models.workflow import WorkflowAppLogCreatedFrom -from repositories.workflow_app_log_query_repository import WorkflowAppLogQueryRepository -from services.workflow_app_log_query_service import WorkflowAppLogQueryService +from repositories.workflow.app_log_repository import WorkflowAppLogRepository +from services.workflow.app_log_query_service import WorkflowAppLogQueryService from tests.test_containers_integration_tests.helpers import accounts as account_fixtures from tests.test_containers_integration_tests.helpers import generate_valid_password @@ -23,7 +23,7 @@ class _WorkflowAppLogTestClient: def __init__(self, session: Session) -> None: session_factory = sessionmaker(bind=session.get_bind(), expire_on_commit=False) self._service = WorkflowAppLogQueryService( - logs=WorkflowAppLogQueryRepository(session_factory=session_factory), + logs=WorkflowAppLogRepository(session_factory=session_factory), ) def get_paginate_workflow_app_logs(self, *, session: Session, app_model, **kwargs): diff --git a/api/tests/test_containers_integration_tests/services/test_workflow_draft_variable_service.py b/api/tests/test_containers_integration_tests/services/test_workflow_draft_variable_service.py index 82fe391b08af52..dfe4a750d1f3d2 100644 --- a/api/tests/test_containers_integration_tests/services/test_workflow_draft_variable_service.py +++ b/api/tests/test_containers_integration_tests/services/test_workflow_draft_variable_service.py @@ -1,15 +1,16 @@ import pytest from faker import Faker -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from core.workflow.variable_prefixes import CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID +from extensions.application_services.workflow_variables import build_workflow_variable_service from graphon.variables.segments import StringSegment from models import App, Workflow from models.enums import DraftVariableType from models.workflow import WorkflowDraftVariable -from services.workflow_draft_variable_service import ( +from repositories.workflow.draft_variable_repository import WorkflowDraftVariableRepository +from services.workflow.draft_variable_service import ( UpdateNotSupportedError, - WorkflowDraftVariableService, ) @@ -17,11 +18,11 @@ def _get_random_variable_name(fake: Faker): return "".join(fake.random_letters(length=10)) -class TestWorkflowDraftVariableService: +class TestWorkflowVariableService: """ - Comprehensive integration tests for WorkflowDraftVariableService using testcontainers. + Comprehensive integration tests for WorkflowVariableService using testcontainers. - This test class covers all major functionality of the WorkflowDraftVariableService: + This test class covers all major functionality of the WorkflowVariableService: - CRUD operations for workflow draft variables (Create, Read, Update, Delete) - Variable listing and filtering by type (conversation, system, node) - Variable updates and resets with proper validation @@ -38,11 +39,11 @@ def mock_external_service_dependencies(self): """ Mock setup for external service dependencies. - WorkflowDraftVariableService doesn't have external dependencies that need mocking, + WorkflowVariableService doesn't have external dependencies that need mocking, so this fixture returns an empty dictionary to maintain consistency with other test classes. This ensures the test structure remains consistent across different service test files. """ - # WorkflowDraftVariableService doesn't have external dependencies that need mocking + # WorkflowVariableService doesn't have external dependencies that need mocking return {} def _create_test_app( @@ -207,8 +208,10 @@ def test_get_variable_success(self, db_session_with_containers: Session, mock_ex user_id=app.created_by, fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) - retrieved_variable = service.get_variable(variable.id) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + retrieved_variable = service.get_variable(variable.id, app_id=app.id, user_id=app.created_by) assert retrieved_variable is not None assert retrieved_variable.id == variable.id assert retrieved_variable.name == "test_var" @@ -225,8 +228,10 @@ def test_get_variable_not_found(self, db_session_with_containers: Session, mock_ """ fake = Faker() non_existent_id = fake.uuid4() - service = WorkflowDraftVariableService(db_session_with_containers) - retrieved_variable = service.get_variable(non_existent_id) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + retrieved_variable = service.get_variable(non_existent_id, app_id="missing-app", user_id="missing-user") assert retrieved_variable is None def test_get_draft_variables_by_selectors_success( @@ -265,8 +270,12 @@ def test_get_draft_variables_by_selectors_success( [CONVERSATION_VARIABLE_NODE_ID, "var2"], ["test_node_1", "var3"], ] - service = WorkflowDraftVariableService(db_session_with_containers) - retrieved_variables = service.get_draft_variables_by_selectors(app.id, selectors, user_id=app.created_by) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + retrieved_variables = WorkflowDraftVariableRepository( + sessions=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ).get_draft_variables_by_selectors(app.id, selectors, user_id=app.created_by) assert len(retrieved_variables) == 3 var_names = [var.name for var in retrieved_variables] assert "var1" in var_names @@ -303,7 +312,9 @@ def test_list_variables_without_values_success( test_value, fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) result = service.list_variables_without_values(app.id, page=1, limit=3, user_id=app.created_by) assert result.total == 5 assert len(result.variables) == 3 @@ -354,7 +365,9 @@ def test_list_node_variables_success(self, db_session_with_containers: Session, variable_type=DraftVariableType.NODE, fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) result = service.list_node_variables(app.id, node_id, user_id=app.created_by) assert len(result.variables) == 2 for var in result.variables: @@ -396,7 +409,9 @@ def test_list_conversation_variables_success( variable_type=DraftVariableType.SYS, fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) result = service.list_conversation_variables(app.id, user_id=app.created_by) assert len(result.variables) == 2 for var in result.variables: @@ -428,16 +443,20 @@ def test_update_variable_success(self, db_session_with_containers: Session, mock original_value, fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) updated_variable = service.update_variable(variable, name="new_name", value=new_value) assert updated_variable.name == "new_name" assert updated_variable.get_value().value == new_value.value assert updated_variable.last_edited_at is not None - db_session_with_containers.refresh(variable) - assert variable.name == "new_name" - assert variable.get_value().value == new_value.value - assert variable.last_edited_at is not None + with Session(db_session_with_containers.get_bind()) as verify: + persisted = verify.get(WorkflowDraftVariable, variable.id) + assert persisted is not None + assert persisted.name == "new_name" + assert persisted.get_value().value == new_value.value + assert persisted.last_edited_at is not None def test_update_variable_not_editable( self, db_session_with_containers: Session, mock_external_service_dependencies @@ -464,7 +483,9 @@ def test_update_variable_not_editable( db_session_with_containers.add(variable) db_session_with_containers.commit() - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) with pytest.raises(UpdateNotSupportedError) as exc_info: service.update_variable(variable, name="new_name", value=new_value) assert "variable not support updating" in str(exc_info.value) @@ -506,14 +527,22 @@ def test_reset_conversation_variable_success( ) variable.last_edited_at = fake.date_time() db_session_with_containers.commit() - service = WorkflowDraftVariableService(db_session_with_containers) + db_session_with_containers.refresh(workflow) + db_session_with_containers.refresh(variable) + db_session_with_containers.expunge(workflow) + db_session_with_containers.expunge(variable) + db_session_with_containers.rollback() + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) reset_variable = service.reset_variable(workflow, variable) assert reset_variable is not None assert reset_variable.get_value().value == "default_value" assert reset_variable.last_edited_at is None - db_session_with_containers.refresh(variable) - assert variable.get_value().value == "default_value" - assert variable.last_edited_at is None + persisted = db_session_with_containers.get(WorkflowDraftVariable, variable.id) + assert persisted is not None + assert persisted.get_value().value == "default_value" + assert persisted.last_edited_at is None def test_delete_variable_success(self, db_session_with_containers: Session, mock_external_service_dependencies): """ @@ -531,7 +560,11 @@ def test_delete_variable_success(self, db_session_with_containers: Session, mock ) assert db_session_with_containers.query(WorkflowDraftVariable).filter_by(id=variable.id).first() is not None - service = WorkflowDraftVariableService(db_session_with_containers) + db_session_with_containers.expunge(variable) + db_session_with_containers.rollback() + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) service.delete_variable(variable) assert db_session_with_containers.query(WorkflowDraftVariable).filter_by(id=variable.id).first() is None @@ -574,7 +607,9 @@ def test_delete_workflow_variables_success( ) assert len(app_variables) == 3 assert len(other_app_variables) == 1 - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) service.delete_user_workflow_variables(app.id, user_id=app.created_by) app_variables_after = db_session_with_containers.query(WorkflowDraftVariable).filter_by(app_id=app.id).all() other_app_variables_after = ( @@ -628,7 +663,9 @@ def test_draft_variables_are_isolated_between_users( fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) user_a_vars = service.list_conversation_variables(app.id, user_id=user_a) user_b_vars = service.list_conversation_variables(app.id, user_id=user_b) @@ -705,7 +742,9 @@ def test_delete_node_variables_success( assert len(target_node_variables) == 2 assert len(other_node_variables) == 1 assert len(conv_variables) == 1 - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) service.delete_node_variables(app.id, node_id, user_id=app.created_by) target_node_variables_after = ( db_session_with_containers.query(WorkflowDraftVariable).filter_by(app_id=app.id, node_id=node_id).all() @@ -753,7 +792,9 @@ def test_prefill_conversation_variable_default_values_success( workflow.conversation_variables = [conv_var1, conv_var2] db_session_with_containers.commit() - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) service.prefill_conversation_variable_default_values(workflow, user_id="00000000-0000-0000-0000-000000000001") draft_variables = ( db_session_with_containers.query(WorkflowDraftVariable) @@ -793,7 +834,9 @@ def test_get_conversation_id_from_draft_variable_success( variable_type=DraftVariableType.SYS, fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) retrieved_conv_id = service._get_conversation_id_from_draft_variable(app.id, app.created_by) assert retrieved_conv_id == conversation_id @@ -809,7 +852,9 @@ def test_get_conversation_id_from_draft_variable_not_found( """ fake = Faker() app = self._create_test_app(db_session_with_containers, mock_external_service_dependencies, fake=fake) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) retrieved_conv_id = service._get_conversation_id_from_draft_variable(app.id, app.created_by) assert retrieved_conv_id is None @@ -850,7 +895,9 @@ def test_list_system_variables_success( self._create_test_variable( db_session_with_containers, app.id, CONVERSATION_VARIABLE_NODE_ID, "conv_var", conv_var_value, fake=fake ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) result = service.list_system_variables(app.id, user_id=app.created_by) assert len(result.variables) == 2 for var in result.variables: @@ -861,77 +908,3 @@ def test_list_system_variables_success( assert "sys_var1" in var_names assert "sys_var2" in var_names assert "conv_var" not in var_names - - def test_get_variable_by_name_success( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test getting variables by name successfully for different types. - - This test verifies that the service can retrieve variables by name - for different variable types (conversation, system, node). This - functionality is important for variable lookup operations during - workflow execution and user interactions. - """ - fake = Faker() - app = self._create_test_app(db_session_with_containers, mock_external_service_dependencies, fake=fake) - test_value = StringSegment(value=fake.word()) - conv_var = self._create_test_variable( - db_session_with_containers, app.id, CONVERSATION_VARIABLE_NODE_ID, "test_conv_var", test_value, fake=fake - ) - sys_var = self._create_test_variable( - db_session_with_containers, - app.id, - SYSTEM_VARIABLE_NODE_ID, - "test_sys_var", - test_value, - variable_type=DraftVariableType.SYS, - fake=fake, - ) - node_var = self._create_test_variable( - db_session_with_containers, - app.id, - "test_node", - "test_node_var", - test_value, - variable_type=DraftVariableType.NODE, - fake=fake, - ) - service = WorkflowDraftVariableService(db_session_with_containers) - retrieved_conv_var = service.get_conversation_variable(app.id, "test_conv_var", user_id=app.created_by) - assert retrieved_conv_var is not None - assert retrieved_conv_var.name == "test_conv_var" - assert retrieved_conv_var.node_id == CONVERSATION_VARIABLE_NODE_ID - retrieved_sys_var = service.get_system_variable(app.id, "test_sys_var", user_id=app.created_by) - assert retrieved_sys_var is not None - assert retrieved_sys_var.name == "test_sys_var" - assert retrieved_sys_var.node_id == SYSTEM_VARIABLE_NODE_ID - retrieved_node_var = service.get_node_variable(app.id, "test_node", "test_node_var", user_id=app.created_by) - assert retrieved_node_var is not None - assert retrieved_node_var.name == "test_node_var" - assert retrieved_node_var.node_id == "test_node" - - def test_get_variable_by_name_not_found( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test getting variables by name when they don't exist. - - This test verifies that the service returns None when trying to - retrieve variables by name that don't exist. This ensures proper - handling of missing variable scenarios for all variable types. - """ - fake = Faker() - app = self._create_test_app(db_session_with_containers, mock_external_service_dependencies, fake=fake) - service = WorkflowDraftVariableService(db_session_with_containers) - retrieved_conv_var = service.get_conversation_variable(app.id, "non_existent_conv_var", user_id=app.created_by) - assert retrieved_conv_var is None - retrieved_sys_var = service.get_system_variable(app.id, "non_existent_sys_var", user_id=app.created_by) - assert retrieved_sys_var is None - retrieved_node_var = service.get_node_variable( - app.id, - "test_node", - "non_existent_node_var", - user_id=app.created_by, - ) - assert retrieved_node_var is None diff --git a/api/tests/test_containers_integration_tests/services/test_workflow_service.py b/api/tests/test_containers_integration_tests/services/test_workflow_service.py index d8ce43c65e2776..92acb54cde309a 100644 --- a/api/tests/test_containers_integration_tests/services/test_workflow_service.py +++ b/api/tests/test_containers_integration_tests/services/test_workflow_service.py @@ -12,11 +12,15 @@ from faker import Faker from sqlalchemy.orm import Session +from extensions.application_services.workflow import build_app_dsl_service from graphon.enums import BuiltinNodeTypes, ErrorStrategy, WorkflowNodeExecutionStatus +from machinery.context import RequestContext from models import Account, AccountStatus, App, TenantStatus, Workflow from models.enums import CreatorUserRole from models.model import AppMode from models.workflow import WorkflowType +from repositories.workflow.definition_repository import WorkflowDefinitionStore +from services.workflow.contracts import DraftSyncCommand from services.workflow_ref_service import WorkflowRef from services.workflow_service import WorkflowService @@ -212,47 +216,6 @@ def test_get_node_last_run_not_found(self, db_session_with_containers: Session): # Assert assert result is None - def test_is_workflow_exist_true(self, db_session_with_containers: Session): - """ - Test workflow existence check when a draft workflow exists. - - This test verifies that the service correctly identifies when a draft workflow - exists for an application, which is important for workflow management operations. - """ - # Arrange - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - app = self._create_test_app(db_session_with_containers, fake) - workflow = self._create_test_workflow(db_session_with_containers, app, account, fake) - - workflow_service = WorkflowService() - - # Act - result = workflow_service.is_workflow_exist(app, session=db_session_with_containers) - - # Assert - assert result is True - - def test_is_workflow_exist_false(self, db_session_with_containers: Session): - """ - Test workflow existence check when no draft workflow exists. - - This test ensures that the service correctly identifies when no draft workflow - exists for an application, which is the initial state for new apps. - """ - # Arrange - fake = Faker() - app = self._create_test_app(db_session_with_containers, fake) - # Don't create any workflow - - workflow_service = WorkflowService() - - # Act - result = workflow_service.is_workflow_exist(app, session=db_session_with_containers) - - # Assert - assert result is False - def test_get_draft_workflow_success(self, db_session_with_containers: Session): """ Test successful retrieval of a draft workflow. @@ -453,7 +416,7 @@ def test_get_all_published_workflow_pagination(self, db_session_with_containers: workflow_service = WorkflowService() # Act - First page - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, @@ -466,7 +429,7 @@ def test_get_all_published_workflow_pagination(self, db_session_with_containers: assert has_more is True # Act - Second page - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=2, @@ -509,7 +472,7 @@ def test_get_all_published_workflow_user_filter(self, db_session_with_containers workflow_service = WorkflowService() # Act - Filter by account1 - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, limit=10, user_id=account1.id ) @@ -551,7 +514,7 @@ def test_get_all_published_workflow_named_only_filter(self, db_session_with_cont workflow_service = WorkflowService() # Act - Filter named only - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, limit=10, user_id=None, named_only=True ) @@ -570,7 +533,7 @@ def test_get_all_published_workflow_no_workflow_id(self, db_session_with_contain workflow_service = WorkflowService() # Act - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, limit=10, user_id=None ) @@ -596,7 +559,7 @@ def test_get_all_published_workflow_basic(self, db_session_with_containers: Sess workflow_service = WorkflowService() # Act - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, limit=10, user_id=None ) @@ -636,7 +599,7 @@ def test_get_all_published_workflow_combined_filters(self, db_session_with_conta workflow_service = WorkflowService() # Act - Filter by account1 + named_only - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, @@ -665,7 +628,7 @@ def test_get_all_published_workflow_empty_result(self, db_session_with_container workflow_service = WorkflowService() # Act - Filter by a user that has no workflows - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, @@ -706,18 +669,22 @@ def test_sync_draft_workflow_create_new(self, db_session_with_containers: Sessio environment_variables = [] conversation_variables = [] - workflow_service = WorkflowService() + drafts = build_app_dsl_service(db_session_with_containers)._drafts # Act - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=graph, - features=features, - unique_hash=unique_hash, - account=account, - environment_variables=environment_variables, - conversation_variables=conversation_variables, - session=db_session_with_containers, + result = drafts.sync( + RequestContext("test", None, account.id, app.tenant_id), + app.id, + DraftSyncCommand( + graph=graph, + features=features, + unique_hash=unique_hash, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=environment_variables, + conversation_variables=conversation_variables, + ), ) # Assert @@ -725,7 +692,7 @@ def test_sync_draft_workflow_create_new(self, db_session_with_containers: Sessio assert result.version == Workflow.VERSION_DRAFT assert result.app_id == app.id assert result.tenant_id == app.tenant_id - assert result.unique_hash is not None # Should have a hash generated + assert result.hash is not None # Should have a hash generated assert result.graph == json.dumps(graph) assert result.features == json.dumps(features) assert result.created_by == account.id @@ -771,18 +738,22 @@ def test_sync_draft_workflow_update_existing(self, db_session_with_containers: S environment_variables = [] conversation_variables = [] - workflow_service = WorkflowService() + drafts = build_app_dsl_service(db_session_with_containers)._drafts # Act - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=new_graph, - features=new_features, - unique_hash=original_hash, # Use original hash to allow update - account=account, - environment_variables=environment_variables, - conversation_variables=conversation_variables, - session=db_session_with_containers, + result = drafts.sync( + RequestContext("test", None, account.id, app.tenant_id), + app.id, + DraftSyncCommand( + graph=new_graph, + features=new_features, + unique_hash=original_hash, # Use original hash to allow update + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=environment_variables, + conversation_variables=conversation_variables, + ), ) # Assert @@ -790,7 +761,7 @@ def test_sync_draft_workflow_update_existing(self, db_session_with_containers: S assert result.id == existing_workflow.id # Same workflow updated assert result.version == Workflow.VERSION_DRAFT # Hash should be updated to reflect new content - assert result.unique_hash != original_hash # Hash should change after update + assert result.hash != original_hash # Hash should change after update assert result.graph == json.dumps(new_graph) assert result.features == json.dumps(new_features) assert result.updated_by == account.id @@ -829,177 +800,27 @@ def test_sync_draft_workflow_hash_mismatch_error(self, db_session_with_container environment_variables = [] conversation_variables = [] - workflow_service = WorkflowService() + drafts = build_app_dsl_service(db_session_with_containers)._drafts # Act & Assert from services.errors.app import WorkflowHashNotEqualError with pytest.raises(WorkflowHashNotEqualError): - workflow_service.sync_draft_workflow( - app_model=app, - graph=new_graph, - features=new_features, - unique_hash=mismatched_hash, - account=account, - environment_variables=environment_variables, - conversation_variables=conversation_variables, - session=db_session_with_containers, - ) - - def test_publish_workflow_success(self, db_session_with_containers: Session): - """ - Test successful workflow publishing. - - This test verifies that the service can correctly publish a draft - workflow, creating a new published version with proper versioning - and status management. - """ - # Arrange - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - app = self._create_test_app(db_session_with_containers, fake) - - # Create draft workflow - workflow = self._create_test_workflow(db_session_with_containers, app, account, fake) - workflow.version = Workflow.VERSION_DRAFT - - db_session_with_containers.commit() - - workflow_service = WorkflowService() - - # Act - Mock current_user context and pass session - from unittest.mock import patch - - with patch("flask_login.utils._get_user", return_value=account, autospec=True): - result = workflow_service.publish_workflow( - session=db_session_with_containers, app_model=app, account=account + drafts.sync( + RequestContext("test", None, account.id, app.tenant_id), + app.id, + DraftSyncCommand( + graph=new_graph, + features=new_features, + unique_hash=mismatched_hash, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=environment_variables, + conversation_variables=conversation_variables, + ), ) - # Assert - assert result is not None - assert result.version != Workflow.VERSION_DRAFT - # Version should be a timestamp format like '2025-08-22 00:10:24.722051' - assert isinstance(result.version, str) - assert len(result.version) > 10 # Should be a reasonable timestamp length - assert result.created_by == account.id - - def test_publish_workflow_no_draft_error(self, db_session_with_containers: Session): - """ - Test error when publishing workflow without draft. - - This test ensures that the service correctly prevents publishing - when no draft workflow exists, maintaining workflow state consistency. - """ - # Arrange - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - app = self._create_test_app(db_session_with_containers, fake) - - # Don't create any workflow - app should have no draft - - workflow_service = WorkflowService() - - # Act & Assert - with pytest.raises(ValueError, match="No valid workflow found"): - workflow_service.publish_workflow(session=db_session_with_containers, app_model=app, account=account) - - def test_publish_workflow_already_published_error(self, db_session_with_containers: Session): - """ - Test error when publishing already published workflow. - - This test ensures that the service correctly prevents re-publishing - of already published workflows, maintaining version control integrity. - """ - # Arrange - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - app = self._create_test_app(db_session_with_containers, fake) - - # Create already published workflow - workflow = self._create_test_workflow(db_session_with_containers, app, account, fake) - workflow.version = "2024.01.01.001" # Already published - - db_session_with_containers.commit() - - workflow_service = WorkflowService() - - # Act & Assert - with pytest.raises(ValueError, match="No valid workflow found"): - workflow_service.publish_workflow(session=db_session_with_containers, app_model=app, account=account) - - def test_restore_published_workflow_to_draft_does_not_persist_normalized_source_features( - self, db_session_with_containers: Session - ): - """Restore copies legacy feature JSON into draft without rewriting the source row.""" - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - app = self._create_test_app(db_session_with_containers, fake) - app.mode = AppMode.ADVANCED_CHAT - - legacy_features = { - "file_upload": { - "image": { - "enabled": True, - "number_limits": 6, - "transfer_methods": ["remote_url", "local_file"], - } - }, - "opening_statement": "", - "retriever_resource": {"enabled": True}, - "sensitive_word_avoidance": {"enabled": False}, - "speech_to_text": {"enabled": False}, - "suggested_questions": [], - "suggested_questions_after_answer": {"enabled": False}, - "text_to_speech": {"enabled": False, "language": "", "voice": ""}, - } - published_workflow = Workflow( - id=fake.uuid4(), - tenant_id=app.tenant_id, - app_id=app.id, - type=WorkflowType.WORKFLOW, - version="2026.03.19.001", - graph=json.dumps({"nodes": [], "edges": []}), - features=json.dumps(legacy_features), - created_by=account.id, - updated_by=account.id, - environment_variables=[], - conversation_variables=[], - ) - draft_workflow = Workflow( - id=fake.uuid4(), - tenant_id=app.tenant_id, - app_id=app.id, - type=WorkflowType.WORKFLOW, - version=Workflow.VERSION_DRAFT, - graph=json.dumps({"nodes": [], "edges": []}), - features=json.dumps({}), - created_by=account.id, - updated_by=account.id, - environment_variables=[], - conversation_variables=[], - ) - db_session_with_containers.add(published_workflow) - db_session_with_containers.add(draft_workflow) - db_session_with_containers.commit() - - workflow_service = WorkflowService() - - restored_workflow = workflow_service.restore_published_workflow_to_draft( - app_model=app, workflow_id=published_workflow.id, account=account, session=db_session_with_containers - ) - - db_session_with_containers.expire_all() - refreshed_published_workflow = ( - db_session_with_containers.query(Workflow).filter_by(id=published_workflow.id).first() - ) - refreshed_draft_workflow = db_session_with_containers.query(Workflow).filter_by(id=draft_workflow.id).first() - - assert restored_workflow.id == draft_workflow.id - assert refreshed_published_workflow is not None - assert refreshed_draft_workflow is not None - assert refreshed_published_workflow.serialized_features == json.dumps(legacy_features) - assert refreshed_draft_workflow.serialized_features == json.dumps(legacy_features) - def test_get_default_block_configs(self, db_session_with_containers: Session): """ Test retrieval of default block configurations for all node types. @@ -1084,156 +905,6 @@ def test_get_default_block_config_with_filters(self, db_session_with_containers: # Result might be None if filters don't match, but should not raise error assert result is None or isinstance(result, dict) - def test_convert_to_workflow_chat_mode_success(self, db_session_with_containers: Session): - """ - Test successful conversion from chat mode app to workflow mode. - - This test verifies that the service can correctly convert a chatbot - application to workflow mode, which is essential for app mode migration. - """ - # Arrange - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - - # Create chat mode app - app = self._create_test_app(db_session_with_containers, fake) - app.mode = AppMode.CHAT - - # Create app model config (required for conversion) - from models.model import AppModelConfig - - app_model_config = AppModelConfig( - app_id=app.id, - provider="openai", - model_id="gpt-3.5-turbo", - # Set the model field directly - this is what model_dict property returns - model=json.dumps( - { - "provider": "openai", - "name": "gpt-3.5-turbo", - "completion_params": {"max_tokens": 1000, "temperature": 0.7}, - } - ), - # Set pre_prompt for PromptTemplateConfigManager - pre_prompt="You are a helpful assistant.", - created_by=account.id, - updated_by=account.id, - ) - app_model_config.id = fake.uuid4() - - db_session_with_containers.add(app_model_config) - app.app_model_config_id = app_model_config.id - db_session_with_containers.commit() - - workflow_service = WorkflowService() - conversion_args = { - "name": "Converted Workflow App", - "icon_type": "emoji", - "icon": "🚀", - "icon_background": "#FF5733", - } - - # Act - result = workflow_service.convert_to_workflow( - app_model=app, account=account, args=conversion_args, session=db_session_with_containers - ) - - # Assert - assert result is not None - assert result.mode == AppMode.ADVANCED_CHAT # CHAT mode converts to ADVANCED_CHAT, not WORKFLOW - assert result.name == conversion_args["name"] - assert result.icon == conversion_args["icon"] - assert result.icon_type == conversion_args["icon_type"] - assert result.icon_background == conversion_args["icon_background"] - - def test_convert_to_workflow_completion_mode_success(self, db_session_with_containers: Session): - """ - Test successful conversion from completion mode app to workflow mode. - - This test verifies that the service can correctly convert a completion - application to workflow mode, supporting different app type migrations. - """ - # Arrange - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - - # Create completion mode app - app = self._create_test_app(db_session_with_containers, fake) - app.mode = AppMode.COMPLETION - - # Create app model config (required for conversion) - from models.model import AppModelConfig - - app_model_config = AppModelConfig( - app_id=app.id, - provider="openai", - model_id="gpt-3.5-turbo", - # Set the model field directly - this is what model_dict property returns - model=json.dumps( - { - "provider": "openai", - "name": "gpt-3.5-turbo", - "completion_params": {"max_tokens": 1000, "temperature": 0.7}, - } - ), - # Set pre_prompt for PromptTemplateConfigManager - pre_prompt="Complete the following text:", - created_by=account.id, - updated_by=account.id, - ) - app_model_config.id = fake.uuid4() - - db_session_with_containers.add(app_model_config) - app.app_model_config_id = app_model_config.id - db_session_with_containers.commit() - - workflow_service = WorkflowService() - conversion_args = { - "name": "Converted Workflow App", - "icon_type": "emoji", - "icon": "🚀", - "icon_background": "#FF5733", - } - - # Act - result = workflow_service.convert_to_workflow( - app_model=app, account=account, args=conversion_args, session=db_session_with_containers - ) - - # Assert - assert result is not None - assert result.mode == AppMode.WORKFLOW - assert result.name == conversion_args["name"] - assert result.icon == conversion_args["icon"] - assert result.icon_type == conversion_args["icon_type"] - assert result.icon_background == conversion_args["icon_background"] - - def test_convert_to_workflow_unsupported_mode_error(self, db_session_with_containers: Session): - """ - Test error when attempting to convert unsupported app mode. - - This test ensures that the service correctly prevents conversion - of apps that are not in supported modes for workflow conversion. - """ - # Arrange - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - - # Create workflow mode app (already in workflow mode) - app = self._create_test_app(db_session_with_containers, fake) - app.mode = AppMode.WORKFLOW - - db_session_with_containers.commit() - - workflow_service = WorkflowService() - conversion_args = {"name": "Test"} - - # Act & Assert - with pytest.raises(ValueError, match="Current App mode: workflow is not supported convert to workflow"): - workflow_service.convert_to_workflow( - app_model=app, account=account, args=conversion_args, session=db_session_with_containers - ) - def test_validate_features_structure_advanced_chat(self, db_session_with_containers: Session): """ Test feature structure validation for advanced chat mode apps. @@ -1328,7 +999,7 @@ def test_update_workflow_success(self, db_session_with_containers: Session): update_data = {"marked_name": "Updated Workflow Name", "marked_comment": "Updated workflow comment"} # Act - result = workflow_service.update_workflow( + result = WorkflowDefinitionStore.update_workflow( session=db_session_with_containers, account_id=account.id, data=update_data, @@ -1358,7 +1029,7 @@ def test_update_workflow_not_found(self, db_session_with_containers: Session): update_data = {"marked_name": "Test"} # Act - result = workflow_service.update_workflow( + result = WorkflowDefinitionStore.update_workflow( session=db_session_with_containers, account_id=account.id, data=update_data, @@ -1392,7 +1063,7 @@ def test_update_workflow_ignores_disallowed_fields(self, db_session_with_contain } # Act - result = workflow_service.update_workflow( + result = WorkflowDefinitionStore.update_workflow( session=db_session_with_containers, account_id=account.id, data=update_data, @@ -1425,10 +1096,8 @@ def test_delete_workflow_success(self, db_session_with_containers: Session): db_session_with_containers.commit() - workflow_service = WorkflowService() - # Act - result = workflow_service.delete_workflow( + result = WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=workflow.tenant_id, owner_id=app.id, workflow_id=workflow.id), ) @@ -1458,13 +1127,11 @@ def test_delete_workflow_draft_error(self, db_session_with_containers: Session): db_session_with_containers.commit() - workflow_service = WorkflowService() - # Act & Assert from services.errors.workflow_service import DraftWorkflowDeletionError with pytest.raises(DraftWorkflowDeletionError, match="Cannot delete draft workflow versions"): - workflow_service.delete_workflow( + WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=workflow.tenant_id, owner_id=app.id, workflow_id=workflow.id), ) @@ -1490,13 +1157,11 @@ def test_delete_workflow_in_use_error(self, db_session_with_containers: Session) db_session_with_containers.commit() - workflow_service = WorkflowService() - # Act & Assert from services.errors.workflow_service import WorkflowInUseError with pytest.raises(WorkflowInUseError, match="Cannot delete workflow that is currently in use by app"): - workflow_service.delete_workflow( + WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=workflow.tenant_id, owner_id=app.id, workflow_id=workflow.id), ) @@ -1513,11 +1178,9 @@ def test_delete_workflow_not_found_error(self, db_session_with_containers: Sessi app = self._create_test_app(db_session_with_containers, fake) non_existent_workflow_id = fake.uuid4() - workflow_service = WorkflowService() - # Act & Assert with pytest.raises(ValueError, match=f"Workflow with ID {non_existent_workflow_id} not found"): - workflow_service.delete_workflow( + WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef( tenant_id=app.tenant_id, @@ -1559,7 +1222,7 @@ def test_run_free_workflow_node_success(self, db_session_with_containers: Sessio from unittest.mock import patch from core.model_manager import ModelInstance - from core.workflow.node_factory import DifyNodeFactory + from services.workflow.execution.adapters.node_factory import DifyNodeFactory # Act with patch.object( diff --git a/api/tests/test_containers_integration_tests/services/tools/conftest.py b/api/tests/test_containers_integration_tests/services/tools/conftest.py new file mode 100644 index 00000000000000..187b5649fe7d00 --- /dev/null +++ b/api/tests/test_containers_integration_tests/services/tools/conftest.py @@ -0,0 +1,17 @@ +"""Tool metadata and management use the test database through explicit ports.""" + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from repositories.tools.workflow_repository import WorkflowToolRepository +from services.tools.workflow_tools_manage_service import WorkflowToolManageService + + +@pytest.fixture +def workflow_queries(db_session_with_containers: Session) -> WorkflowToolRepository: + return WorkflowToolRepository(sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False)) + + +@pytest.fixture +def workflow_tools(workflow_queries: WorkflowToolRepository) -> WorkflowToolManageService: + return WorkflowToolManageService(workflow_queries) diff --git a/api/tests/test_containers_integration_tests/services/tools/test_api_tools_manage_service.py b/api/tests/test_containers_integration_tests/services/tools/test_api_tools_manage_service.py index dc77f77a63f395..2196a1d04800e1 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_api_tools_manage_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_api_tools_manage_service.py @@ -10,10 +10,10 @@ from core.tools.entities.tool_entities import ApiProviderSchemaType from core.tools.errors import ApiToolProviderNotFoundError -from core.tools.tool_label_manager import ToolLabelManager from models import Account, AccountStatus, Tenant, TenantStatus from models.tools import ApiToolProvider -from services.tools.legacy_api_tools_manage_service import ApiToolManageService +from services.tools.api_tools_manage_service import ApiToolManageService +from services.tools.tool_label_manager import ToolLabelManager MockDependencies = dict[str, MagicMock] @@ -25,17 +25,15 @@ class TestApiToolManageService: def mock_external_service_dependencies(self) -> Iterator[MockDependencies]: """Mock setup for external service dependencies.""" with ( - patch("services.tools.legacy_api_tools_manage_service.ToolLabelManager") as mock_tool_label_manager, - patch("services.tools.legacy_api_tools_manage_service.create_tool_provider_encrypter") as mock_encrypter, - patch( - "services.tools.legacy_api_tools_manage_service.ApiToolProviderController" - ) as mock_provider_controller, + patch("services.tools.api_tools_manage_service.ToolLabelManager") as mock_tool_label_manager, + patch("services.tools.api_tools_manage_service.create_tool_provider_encrypter") as mock_encrypter, + patch("services.tools.api_tools_manage_service.ApiToolProviderController") as mock_provider_controller, ): # Setup default mock returns mock_tool_label_manager.update_tool_labels.return_value = None mock_encrypter.return_value = (mock_encrypter, None) mock_encrypter.encrypt.return_value = {"encrypted": "credentials"} - mock_provider_controller.from_db.return_value = mock_provider_controller + mock_provider_controller.from_provider.return_value = mock_provider_controller mock_provider_controller.load_bundled_tools.return_value = None yield { @@ -376,8 +374,7 @@ def test_create_api_tool_provider_success( # Verify mock interactions mock_external_service_dependencies["tool_label_manager"].update_tool_labels.assert_called_once() mock_external_service_dependencies["encrypter"].assert_called_once() - mock_external_service_dependencies["provider_controller"].from_db.assert_called_once() - mock_external_service_dependencies["provider_controller"].load_bundled_tools.assert_called_once() + mock_external_service_dependencies["provider_controller"].from_provider.assert_called_once() def test_create_api_tool_provider_duplicate_name( self, @@ -582,7 +579,7 @@ def test_create_api_tool_provider_with_api_key_auth( # Verify mock interactions mock_external_service_dependencies["encrypter"].assert_called_once() - mock_external_service_dependencies["provider_controller"].from_db.assert_called_once() + mock_external_service_dependencies["provider_controller"].from_provider.assert_called_once() def test_delete_api_tool_provider_success( self, @@ -682,7 +679,7 @@ def test_update_api_tool_provider_success( # Reset mock history so assertions focus on update path only mock_external_service_dependencies["encrypter"].reset_mock() - mock_external_service_dependencies["provider_controller"].from_db.reset_mock() + mock_external_service_dependencies["provider_controller"].from_provider.reset_mock() mock_external_service_dependencies["tool_label_manager"].update_tool_labels.reset_mock() # Act: Update the provider with new values @@ -739,7 +736,7 @@ def test_update_api_tool_provider_success( assert original_provider is None # Verify update flow calls critical collaborators - mock_external_service_dependencies["provider_controller"].from_db.assert_called_once() + mock_external_service_dependencies["provider_controller"].from_provider.assert_called_once() mock_external_service_dependencies["encrypter"].assert_called_once() mock_cache.delete.assert_called_once() @@ -793,7 +790,7 @@ def test_update_api_tool_provider_not_found( # Reset mock history so assertions focus on update failure path only mock_external_service_dependencies["tool_label_manager"].update_tool_labels.reset_mock() mock_external_service_dependencies["encrypter"].reset_mock() - mock_external_service_dependencies["provider_controller"].from_db.reset_mock() + mock_external_service_dependencies["provider_controller"].from_provider.reset_mock() # Act & Assert: Verify update fails with clear error message target_new_name = "new-provider-name" @@ -838,7 +835,7 @@ def test_update_api_tool_provider_not_found( # Assert: Early failure should skip all downstream external interactions mock_external_service_dependencies["tool_label_manager"].update_tool_labels.assert_not_called() mock_external_service_dependencies["encrypter"].assert_not_called() - mock_external_service_dependencies["provider_controller"].from_db.assert_not_called() + mock_external_service_dependencies["provider_controller"].from_provider.assert_not_called() def test_update_api_tool_provider_missing_auth_type( self, diff --git a/api/tests/test_containers_integration_tests/services/tools/test_mcp_tools_manage_service.py b/api/tests/test_containers_integration_tests/services/tools/test_mcp_tools_manage_service.py index f16aec54a3f5ca..ba2ae62a053743 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_mcp_tools_manage_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_mcp_tools_manage_service.py @@ -7,7 +7,7 @@ from core.tools.entities.tool_entities import ToolProviderType from models import Account, Tenant from models.tools import MCPToolProvider -from services.tools.legacy_mcp_tools_manage_service import UNCHANGED_SERVER_URL_PLACEHOLDER, MCPToolManageService +from services.tools.mcp_tools_manage_service import UNCHANGED_SERVER_URL_PLACEHOLDER, MCPToolManageService class TestMCPToolManageService: @@ -17,8 +17,8 @@ class TestMCPToolManageService: def mock_external_service_dependencies(self): """Mock setup for external service dependencies.""" with ( - patch("services.tools.legacy_mcp_tools_manage_service.encrypter") as mock_encrypter, - patch("services.tools.legacy_mcp_tools_manage_service.ToolTransformService") as mock_tool_transform_service, + patch("services.tools.mcp_tools_manage_service.encrypter") as mock_encrypter, + patch("services.tools.mcp_tools_manage_service.ToolTransformService") as mock_tool_transform_service, ): # Setup default mock returns from core.tools.entities.api_entities import ToolProviderApiEntity @@ -804,7 +804,7 @@ def test_list_mcp_tool_from_remote_server_success( )(), ] - with patch("services.tools.legacy_mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: + with patch("services.tools.mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: # Setup mock client mock_client_instance = mock_mcp_client.return_value.__enter__.return_value mock_client_instance.list_tools.return_value = mock_tools @@ -866,7 +866,7 @@ def test_list_mcp_tool_from_remote_server_auth_error( mock_decrypt.return_value = "https://example.com/mcp" # Mock MCPClient to raise authentication error - with patch("services.tools.legacy_mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: + with patch("services.tools.mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: from core.mcp.error import MCPAuthError mock_client_instance = mock_mcp_client.return_value.__enter__.return_value @@ -918,7 +918,7 @@ def test_list_mcp_tool_from_remote_server_connection_error( mock_decrypt.return_value = "https://example.com/mcp" # Mock MCPClient to raise connection error - with patch("services.tools.legacy_mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: + with patch("services.tools.mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: from core.mcp.error import MCPError mock_client_instance = mock_mcp_client.return_value.__enter__.return_value diff --git a/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py b/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py index 257b3e0b0e65ac..6282a4e253e1ee 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py @@ -6,6 +6,7 @@ from faker import Faker from sqlalchemy.orm import Session +from core.app.apps.draft_variable_saver import DraftVariableSaverFactory from core.plugin.plugin_service import PluginService from core.tools.__base.tool import Tool from core.tools.entities.api_entities import ToolApiEntity, ToolProviderApiEntity @@ -21,7 +22,7 @@ ToolProviderType, ) from models.tools import ApiToolProvider, WorkflowToolProvider -from services.tools.legacy_tools_transform_service import ToolTransformService +from services.tools.tools_transform_service import ToolTransformService class TestToolTransformService: @@ -30,7 +31,7 @@ class TestToolTransformService: @pytest.fixture def mock_external_service_dependencies(self): """Mock setup for external service dependencies.""" - with patch("services.tools.legacy_tools_transform_service.dify_config") as mock_dify_config: + with patch("services.tools.tools_transform_service.dify_config") as mock_dify_config: with patch("core.plugin.plugin_service.dify_config", new=mock_dify_config): # Setup default mock returns mock_dify_config.CONSOLE_API_URL = "https://console.example.com" @@ -465,7 +466,7 @@ def test_builtin_provider_to_user_provider_success( mock_db_provider.credentials = {"api_key": "encrypted_key"} # Mock encryption - with patch("services.tools.legacy_tools_transform_service.create_provider_encrypter") as mock_encrypter: + with patch("services.tools.tools_transform_service.create_provider_encrypter") as mock_encrypter: mock_encrypter_instance = Mock() mock_encrypter_instance.decrypt.return_value = {"api_key": "decrypted_key"} mock_encrypter_instance.mask_plugin_credentials.return_value = {"api_key": ""} @@ -689,7 +690,7 @@ def test_api_provider_to_controller_backward_compatibility( assert hasattr(result, "from_db") def test_workflow_provider_to_controller_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_queries ): """ Test successful conversion of workflow provider to controller. @@ -718,19 +719,22 @@ def test_workflow_provider_to_controller_success( db_session_with_containers.commit() # Mock the WorkflowToolProviderController.from_db method to avoid app dependency - with patch( - "services.tools.legacy_tools_transform_service.WorkflowToolProviderController.from_db" - ) as mock_from_db: + with patch("services.tools.tools_transform_service.WorkflowToolProviderController.from_db") as mock_from_db: mock_controller = Mock() mock_from_db.return_value = mock_controller # Act: Execute the method under test - result = ToolTransformService.workflow_provider_to_controller(provider) + result = ToolTransformService.workflow_provider_to_controller(provider, queries=workflow_queries) # Assert: Verify the expected outcomes assert result is not None assert result == mock_controller - mock_from_db.assert_called_once_with(provider) + mock_from_db.assert_called_once_with( + provider, + queries=workflow_queries, + draft_variable_saver=None, + workflow_runtime=None, + ) def _mock_tool(*, base_params, runtime_params): @@ -842,8 +846,8 @@ class TestWorkflowProviderToUserProvider: """Tests for ToolTransformService.workflow_provider_to_user_provider.""" @staticmethod - def _make_controller(provider_id="provider_123", **identity_overrides): - from core.tools.workflow_as_tool.provider import WorkflowToolProviderController + def _make_controller(provider_id="provider_123", *, workflow_queries, **identity_overrides): + from services.tools.workflow.provider import WorkflowToolProviderController defaults = { "author": "test_author", @@ -856,10 +860,15 @@ def _make_controller(provider_id="provider_123", **identity_overrides): defaults.update(identity_overrides) identity = ToolProviderIdentity(**defaults) entity = ToolProviderEntity(identity=identity) - return WorkflowToolProviderController(entity=entity, provider_id=provider_id) + return WorkflowToolProviderController( + draft_variable_saver=Mock(return_value=Mock(spec=DraftVariableSaverFactory)), + entity=entity, + provider_id=provider_id, + queries=workflow_queries, + ) - def test_with_workflow_app_id(self): - ctrl = self._make_controller() + def test_with_workflow_app_id(self, *, workflow_queries): + ctrl = self._make_controller(workflow_queries=workflow_queries) result = ToolTransformService.workflow_provider_to_user_provider( provider_controller=ctrl, @@ -874,8 +883,8 @@ def test_with_workflow_app_id(self): assert result.labels == ["l1", "l2"] assert result.is_team_authorization is True - def test_without_workflow_app_id(self): - ctrl = self._make_controller() + def test_without_workflow_app_id(self, *, workflow_queries): + ctrl = self._make_controller(workflow_queries=workflow_queries) result = ToolTransformService.workflow_provider_to_user_provider( provider_controller=ctrl, @@ -884,8 +893,8 @@ def test_without_workflow_app_id(self): assert result.workflow_app_id is None - def test_workflow_app_id_none_explicit(self): - ctrl = self._make_controller() + def test_workflow_app_id_none_explicit(self, *, workflow_queries): + ctrl = self._make_controller(workflow_queries=workflow_queries) result = ToolTransformService.workflow_provider_to_user_provider( provider_controller=ctrl, @@ -896,7 +905,7 @@ def test_workflow_app_id_none_explicit(self): assert result.workflow_app_id is None assert result.labels == [] - def test_preserves_other_fields(self): + def test_preserves_other_fields(self, *, workflow_queries): ctrl = self._make_controller( "provider_456", author="another_author", @@ -905,6 +914,7 @@ def test_preserves_other_fields(self): icon='{"type": "emoji", "content": "⚙️"}', icon_dark='{"type": "emoji", "content": "🔧"}', label=I18nObject(en_US="Another Tool", zh_Hans="Another Tool"), + workflow_queries=workflow_queries, ) result = ToolTransformService.workflow_provider_to_user_provider( diff --git a/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py b/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py index fb00f1c783d820..2ca43aa5a11803 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py @@ -4,14 +4,14 @@ import pytest from faker import Faker from pydantic import ValidationError +from sqlalchemy import select from sqlalchemy.orm import Session from core.tools.entities.tool_entities import WorkflowToolParameterConfiguration from core.tools.errors import WorkflowToolHumanInputNotSupportedError -from models.tools import WorkflowToolProvider +from models.tools import ToolLabelBinding, WorkflowToolProvider from models.workflow import Workflow as WorkflowModel from services.app_service import AppService, CreateAppParams -from services.tools.legacy_workflow_tools_manage_service import WorkflowToolManageService from tests.test_containers_integration_tests.helpers import accounts as account_fixtures from tests.test_containers_integration_tests.helpers import generate_valid_password @@ -27,13 +27,7 @@ def mock_external_service_dependencies(self): patch("services.app_service.EnterpriseService") as mock_enterprise_service, patch("services.app_service.ModelManager.for_tenant") as mock_model_manager, patch("services.account.login_adapters.SystemFeatureService") as mock_account_feature_service, - patch( - "services.tools.legacy_workflow_tools_manage_service.WorkflowToolProviderController" - ) as mock_workflow_tool_provider_controller, - patch("services.tools.legacy_workflow_tools_manage_service.ToolLabelManager") as mock_tool_label_manager, - patch( - "services.tools.legacy_workflow_tools_manage_service.ToolTransformService" - ) as mock_tool_transform_service, + patch("services.tools.workflow_tools_manage_service.ToolTransformService") as mock_tool_transform_service, ): # Setup default mock returns for app service mock_feature_service.is_webapp_auth_enabled.return_value = False @@ -48,12 +42,6 @@ def mock_external_service_dependencies(self): mock_model_instance.get_default_model_instance.return_value = None mock_model_instance.get_default_provider_model_name.return_value = ("openai", "gpt-3.5-turbo") - # Mock WorkflowToolProviderController - mock_workflow_tool_provider_controller.from_db.return_value = None - - # Mock ToolLabelManager - mock_tool_label_manager.update_tool_labels.return_value = None - # Mock ToolTransformService mock_tool_transform_service.workflow_provider_to_controller.return_value = None @@ -62,8 +50,6 @@ def mock_external_service_dependencies(self): "enterprise_service": mock_enterprise_service, "model_manager": mock_model_manager, "account_feature_service": mock_account_feature_service, - "workflow_tool_provider_controller": mock_workflow_tool_provider_controller, - "tool_label_manager": mock_tool_label_manager, "tool_transform_service": mock_tool_transform_service, } @@ -155,7 +141,7 @@ def _create_test_workflow_tool_parameters(self): ] def test_create_workflow_tool_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test successful workflow tool creation with valid parameters. @@ -181,10 +167,10 @@ def test_create_workflow_tool_success( tool_description = fake.text(max_nb_chars=200) tool_parameters = self._create_test_workflow_tool_parameters() tool_privacy_policy = fake.text(max_nb_chars=100) - tool_labels = ["automation", "workflow"] + tool_labels = ["productivity", "search"] # Execute the method under test - result = WorkflowToolManageService.create_workflow_tool( + result = workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -225,14 +211,13 @@ def test_create_workflow_tool_success( assert created_tool_provider.app_id == app.id # Verify external service calls - mock_external_service_dependencies["workflow_tool_provider_controller"].from_db.assert_called_once() - mock_external_service_dependencies["tool_label_manager"].update_tool_labels.assert_called_once() - mock_external_service_dependencies[ - "tool_transform_service" - ].workflow_provider_to_controller.assert_called_once() + assert set(db_session_with_containers.scalars(select(ToolLabelBinding.label_name))) == set(tool_labels) + assert ( + mock_external_service_dependencies["tool_transform_service"].workflow_provider_to_controller.call_count == 2 + ) def test_create_workflow_tool_duplicate_name_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation fails when name already exists. @@ -253,7 +238,7 @@ def test_create_workflow_tool_duplicate_name_error( first_tool_name = fake.word() first_tool_parameters = self._create_test_workflow_tool_parameters() - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -267,7 +252,7 @@ def test_create_workflow_tool_duplicate_name_error( # Attempt to create second workflow tool with same name second_tool_parameters = self._create_test_workflow_tool_parameters() with pytest.raises(ValueError) as exc_info: - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -294,7 +279,7 @@ def test_create_workflow_tool_duplicate_name_error( assert tool_count == 1 def test_create_workflow_tool_invalid_app_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation fails when app does not exist. @@ -317,7 +302,7 @@ def test_create_workflow_tool_invalid_app_error( # Attempt to create workflow tool with non-existent app tool_parameters = self._create_test_workflow_tool_parameters() with pytest.raises(ValueError) as exc_info: - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=non_existent_app_id, # Non-existent app ID @@ -329,7 +314,7 @@ def test_create_workflow_tool_invalid_app_error( ) # Verify error message - assert f"App {non_existent_app_id} not found" in str(exc_info.value) + assert "workflow not found or not published" in str(exc_info.value) # Verify no workflow tool was created @@ -344,7 +329,7 @@ def test_create_workflow_tool_invalid_app_error( assert tool_count == 0 def test_create_workflow_tool_invalid_parameters_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation fails when parameters are invalid. @@ -363,7 +348,7 @@ def test_create_workflow_tool_invalid_parameters_error( # Attempt to create workflow tool with invalid parameters with pytest.raises(ValidationError) as exc_info: # Setup invalid workflow tool parameters (missing required fields) - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -399,7 +384,7 @@ def test_create_workflow_tool_invalid_parameters_error( assert tool_count == 0 def test_create_workflow_tool_duplicate_app_id_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation fails when app_id already exists. @@ -419,7 +404,7 @@ def test_create_workflow_tool_duplicate_app_id_error( # Create first workflow tool first_tool_name = fake.word() first_tool_parameters = self._create_test_workflow_tool_parameters() - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -434,7 +419,7 @@ def test_create_workflow_tool_duplicate_app_id_error( second_tool_name = fake.word() second_tool_parameters = self._create_test_workflow_tool_parameters() with pytest.raises(ValueError) as exc_info: - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, # Same app_id @@ -461,7 +446,7 @@ def test_create_workflow_tool_duplicate_app_id_error( assert tool_count == 1 def test_create_workflow_tool_workflow_not_found_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation fails when app has no workflow. @@ -486,7 +471,7 @@ def test_create_workflow_tool_workflow_not_found_error( # Attempt to create workflow tool for app without workflow tool_parameters = self._create_test_workflow_tool_parameters() with pytest.raises(ValueError) as exc_info: - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -498,7 +483,7 @@ def test_create_workflow_tool_workflow_not_found_error( ) # Verify error message - assert f"Workflow not found for app {app.id}" in str(exc_info.value) + assert "workflow not found or not published" in str(exc_info.value) # Verify no workflow tool was created tool_count = ( @@ -512,7 +497,7 @@ def test_create_workflow_tool_workflow_not_found_error( assert tool_count == 0 def test_create_workflow_tool_human_input_node_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation fails when workflow contains human input nodes. @@ -545,7 +530,7 @@ def test_create_workflow_tool_human_input_node_error( tool_parameters = self._create_test_workflow_tool_parameters() with pytest.raises(WorkflowToolHumanInputNotSupportedError) as exc_info: - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -569,7 +554,7 @@ def test_create_workflow_tool_human_input_node_error( assert tool_count == 0 def test_update_workflow_tool_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test successful workflow tool update with valid parameters. @@ -591,7 +576,7 @@ def test_update_workflow_tool_success( # Create initial workflow tool initial_tool_name = fake.word() initial_tool_parameters = self._create_test_workflow_tool_parameters() - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -620,10 +605,10 @@ def test_update_workflow_tool_success( updated_tool_description = fake.text(max_nb_chars=200) updated_tool_parameters = self._create_test_workflow_tool_parameters() updated_tool_privacy_policy = fake.text(max_nb_chars=100) - updated_tool_labels = ["automation", "updated"] + updated_tool_labels = ["search"] # Execute the update method - result = WorkflowToolManageService.update_workflow_tool( + result = workflow_tools.update_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_tool_id=created_tool.id, @@ -652,12 +637,11 @@ def test_update_workflow_tool_success( assert created_tool.updated_at is not None # Verify external service calls - mock_external_service_dependencies["workflow_tool_provider_controller"].from_db.assert_called() - mock_external_service_dependencies["tool_label_manager"].update_tool_labels.assert_called() + assert list(db_session_with_containers.scalars(select(ToolLabelBinding.label_name))) == updated_tool_labels mock_external_service_dependencies["tool_transform_service"].workflow_provider_to_controller.assert_called() def test_update_workflow_tool_human_input_node_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool update fails when workflow contains human input nodes. @@ -677,7 +661,7 @@ def test_update_workflow_tool_human_input_node_error( # Create initial workflow tool initial_tool_name = fake.word() initial_tool_parameters = self._create_test_workflow_tool_parameters() - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -712,7 +696,7 @@ def test_update_workflow_tool_human_input_node_error( db_session_with_containers.commit() with pytest.raises(WorkflowToolHumanInputNotSupportedError) as exc_info: - WorkflowToolManageService.update_workflow_tool( + workflow_tools.update_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_tool_id=created_tool.id, @@ -729,7 +713,7 @@ def test_update_workflow_tool_human_input_node_error( assert created_tool.name == original_name def test_update_workflow_tool_not_found_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool update fails when tool does not exist. @@ -752,7 +736,7 @@ def test_update_workflow_tool_not_found_error( # Attempt to update non-existent workflow tool tool_parameters = self._create_test_workflow_tool_parameters() with pytest.raises(ValueError) as exc_info: - WorkflowToolManageService.update_workflow_tool( + workflow_tools.update_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_tool_id=non_existent_tool_id, # Non-existent tool ID @@ -779,7 +763,7 @@ def test_update_workflow_tool_not_found_error( assert tool_count == 0 def test_update_workflow_tool_same_name_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool update succeeds when keeping the same name. @@ -799,7 +783,7 @@ def test_update_workflow_tool_same_name_success( # Create first workflow tool first_tool_name = fake.word() first_tool_parameters = self._create_test_workflow_tool_parameters() - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -822,7 +806,7 @@ def test_update_workflow_tool_same_name_success( ) # Attempt to update tool with same name (should not fail) - result = WorkflowToolManageService.update_workflow_tool( + result = workflow_tools.update_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_tool_id=created_tool.id, @@ -842,7 +826,7 @@ def test_update_workflow_tool_same_name_success( assert created_tool.updated_at is not None def test_create_workflow_tool_with_file_parameter_default( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation with FILE parameter having a file object as default. @@ -900,8 +884,8 @@ def test_create_workflow_tool_with_file_parameter_default( ] # Execute the method under test - # Note: from_db is mocked, so this test primarily validates the parameter configuration - result = WorkflowToolManageService.create_workflow_tool( + # Note: provider construction is mocked, so this test primarily validates the parameter configuration + result = workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -916,7 +900,7 @@ def test_create_workflow_tool_with_file_parameter_default( assert result == {"result": "success"} def test_create_workflow_tool_with_files_parameter_default( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation with FILES (Array[File]) parameter having file objects as default. @@ -978,8 +962,8 @@ def test_create_workflow_tool_with_files_parameter_default( ] # Execute the method under test - # Note: from_db is mocked, so this test primarily validates the parameter configuration - result = WorkflowToolManageService.create_workflow_tool( + # Note: provider construction is mocked, so this test primarily validates the parameter configuration + result = workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -994,13 +978,13 @@ def test_create_workflow_tool_with_files_parameter_default( assert result == {"result": "success"} def test_create_workflow_tool_db_commit_before_validation( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test that database commit happens before validation, causing DB pollution on validation failure. This test verifies the second bug: - - WorkflowToolProvider is committed to database BEFORE from_db validation + - WorkflowToolProvider is committed to database BEFORE provider construction validation - If validation fails, the record remains in the database - Subsequent attempts fail with "Tool already exists" error @@ -1016,14 +1000,14 @@ def test_create_workflow_tool_db_commit_before_validation( tool_name = fake.word() - # Mock from_db to raise validation error - mock_external_service_dependencies["workflow_tool_provider_controller"].from_db.side_effect = ValueError( - "Validation failed: default parameter type mismatch" - ) + # Mock provider construction to raise validation error + mock_external_service_dependencies[ + "tool_transform_service" + ].workflow_provider_to_controller.side_effect = ValueError("Validation failed: default parameter type mismatch") # Attempt to create workflow tool (will fail at validation stage) with pytest.raises(ValueError) as exc_info: - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -1055,7 +1039,7 @@ def test_create_workflow_tool_db_commit_before_validation( # assert tool_count == 0 # Expected after fix def test_delete_workflow_tool_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """Test successful deletion of a workflow tool.""" fake = Faker() @@ -1064,7 +1048,7 @@ def test_delete_workflow_tool_success( ) tool_name = fake.unique.word() - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -1082,7 +1066,7 @@ def test_delete_workflow_tool_success( ) assert tool is not None - result = WorkflowToolManageService.delete_workflow_tool(account.id, account.current_tenant.id, tool.id) + result = workflow_tools.delete_workflow_tool(account.id, account.current_tenant.id, tool.id) assert result == {"result": "success"} deleted = ( @@ -1091,7 +1075,7 @@ def test_delete_workflow_tool_success( assert deleted is None def test_list_tenant_workflow_tools_empty( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """Test listing workflow tools when none exist returns empty list.""" fake = Faker() @@ -1099,12 +1083,12 @@ def test_list_tenant_workflow_tools_empty( db_session_with_containers, mock_external_service_dependencies ) - result = WorkflowToolManageService.list_tenant_workflow_tools(account.id, account.current_tenant.id) + result = workflow_tools.list_tenant_workflow_tools(account.id, account.current_tenant.id) assert result == [] def test_get_workflow_tool_by_tool_id_not_found( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """Test that get_workflow_tool_by_tool_id raises ValueError when tool not found.""" fake = Faker() @@ -1113,10 +1097,10 @@ def test_get_workflow_tool_by_tool_id_not_found( ) with pytest.raises(ValueError, match="Tool not found"): - WorkflowToolManageService.get_workflow_tool_by_tool_id(account.id, account.current_tenant.id, fake.uuid4()) + workflow_tools.get_workflow_tool_by_tool_id(account.id, account.current_tenant.id, fake.uuid4()) def test_get_workflow_tool_by_app_id_not_found( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """Test that get_workflow_tool_by_app_id raises ValueError when tool not found.""" fake = Faker() @@ -1125,10 +1109,10 @@ def test_get_workflow_tool_by_app_id_not_found( ) with pytest.raises(ValueError, match="Tool not found"): - WorkflowToolManageService.get_workflow_tool_by_app_id(account.id, account.current_tenant.id, fake.uuid4()) + workflow_tools.get_workflow_tool_by_app_id(account.id, account.current_tenant.id, fake.uuid4()) def test_list_single_workflow_tools_not_found( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """Test that list_single_workflow_tools raises ValueError when tool not found.""" fake = Faker() @@ -1137,18 +1121,18 @@ def test_list_single_workflow_tools_not_found( ) with pytest.raises(ValueError, match="not found"): - WorkflowToolManageService.list_single_workflow_tools(account.id, account.current_tenant.id, fake.uuid4()) + workflow_tools.list_single_workflow_tools(account.id, account.current_tenant.id, fake.uuid4()) def test_create_workflow_tool_with_labels( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): - """Test that labels are forwarded to ToolLabelManager when provided.""" + """Labels are persisted atomically with the Workflow Tool.""" fake = Faker() app, account, workflow = self._create_test_app_and_account( db_session_with_containers, mock_external_service_dependencies ) - result = WorkflowToolManageService.create_workflow_tool( + result = workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -1157,8 +1141,11 @@ def test_create_workflow_tool_with_labels( icon={"type": "emoji", "emoji": "🔧"}, description=fake.text(max_nb_chars=200), parameters=self._create_test_workflow_tool_parameters(), - labels=["label-1", "label-2"], + labels=["productivity", "search"], ) assert result == {"result": "success"} - mock_external_service_dependencies["tool_label_manager"].update_tool_labels.assert_called_once() + assert set(db_session_with_containers.scalars(select(ToolLabelBinding.label_name))) == { + "productivity", + "search", + } diff --git a/api/tests/test_containers_integration_tests/services/workflow/test_workflow_converter.py b/api/tests/test_containers_integration_tests/services/workflow/test_workflow_converter.py index dfddf26ddf6dbb..623e371385e1e6 100644 --- a/api/tests/test_containers_integration_tests/services/workflow/test_workflow_converter.py +++ b/api/tests/test_containers_integration_tests/services/workflow/test_workflow_converter.py @@ -1,756 +1,130 @@ -from __future__ import annotations +"""Conversion persists an App aggregate and its draft atomically on PostgreSQL.""" import json -from unittest.mock import MagicMock, patch +from uuid import uuid4 import pytest -from faker import Faker -from sqlalchemy.orm import Session +from sqlalchemy import event, select +from sqlalchemy.orm import Session, sessionmaker -from core.app.app_config.entities import ( - AdvancedChatMessageEntity, - AdvancedChatPromptTemplateEntity, - AdvancedCompletionPromptTemplateEntity, - DatasetEntity, - DatasetRetrieveConfigEntity, - ExternalDataVariableEntity, - ModelConfigEntity, - PromptTemplateEntity, -) -from core.prompt.utils.prompt_template_parser import PromptTemplateParser -from graphon.model_runtime.entities.llm_entities import LLMMode -from graphon.model_runtime.entities.message_entities import PromptMessageRole -from graphon.variables.input_entities import VariableEntity, VariableEntityType -from models import Account, Tenant -from models.api_based_extension import APIBasedExtension, APIBasedExtensionPoint -from models.model import App, AppMode, AppModelConfig +from machinery.context import RequestContext +from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole +from models.model import App, AppMode, AppModelConfig, InstalledApp, Site from models.workflow import Workflow -from services.workflow.workflow_converter import WorkflowConverter +from repositories.app.console_repository import ConsoleAppRepository +from services.workflow.conversion_service import WorkflowConversionService -class TestWorkflowConverter: - """Integration tests for WorkflowConverter using testcontainers.""" - - @pytest.fixture - def mock_external_service_dependencies(self): - """Mock setup for external service dependencies.""" - with ( - patch("services.workflow.workflow_converter.encrypter") as mock_encrypter, - patch("services.workflow.workflow_converter.SimplePromptTransform") as mock_prompt_transform, - patch("services.workflow.workflow_converter.AgentChatAppConfigManager") as mock_agent_chat_config_manager, - patch("services.workflow.workflow_converter.ChatAppConfigManager") as mock_chat_config_manager, - patch("services.workflow.workflow_converter.CompletionAppConfigManager") as mock_completion_config_manager, - ): - # Setup default mock returns - mock_encrypter.decrypt_token.return_value = "decrypted_api_key" - mock_prompt_transform.return_value.get_prompt_template.return_value = { - "prompt_template": PromptTemplateParser(template="You are a helpful assistant {{text_input}}"), - "prompt_rules": {"human_prefix": "Human", "assistant_prefix": "Assistant"}, - } - mock_agent_chat_config_manager.get_app_config.return_value = self._create_mock_app_config() - mock_chat_config_manager.get_app_config.return_value = self._create_mock_app_config() - mock_completion_config_manager.get_app_config.return_value = self._create_mock_app_config() - - yield { - "encrypter": mock_encrypter, - "prompt_transform": mock_prompt_transform, - "agent_chat_config_manager": mock_agent_chat_config_manager, - "chat_config_manager": mock_chat_config_manager, - "completion_config_manager": mock_completion_config_manager, - } - - def _create_mock_app_config(self): - """Helper method to create a mock app config.""" - mock_config = type("obj", (object,), {})() - mock_config.variables = [ - VariableEntity( - variable="text_input", - label="Text Input", - type=VariableEntityType.TEXT_INPUT, - ) - ] - mock_config.model = ModelConfigEntity( - provider="openai", - model="gpt-4", - mode=LLMMode.CHAT, - parameters={}, - stop=[], - ) - mock_config.prompt_template = PromptTemplateEntity( - prompt_type=PromptTemplateEntity.PromptType.SIMPLE, - simple_prompt_template="You are a helpful assistant {{text_input}}", - ) - mock_config.dataset = None - mock_config.external_data_variables = [] - mock_config.additional_features = type("obj", (object,), {"file_upload": None})() - mock_config.app_model_config_dict = {} - return mock_config - - def _create_test_account_and_tenant(self, db_session_with_containers: Session, mock_external_service_dependencies): - """ - Helper method to create a test account and tenant for testing. - - Args: - db_session_with_containers: Database session from testcontainers infrastructure - mock_external_service_dependencies: Mock dependencies - - Returns: - tuple: (account, tenant) - Created account and tenant instances - """ - fake = Faker() - - # Create account - account = Account( - email=fake.email(), - name=fake.name(), - interface_language="en-US", - status="active", - ) - - db_session_with_containers.add(account) - db_session_with_containers.commit() - - # Create tenant for the account - tenant = Tenant( - name=fake.company(), - status="normal", - ) - db_session_with_containers.add(tenant) - db_session_with_containers.commit() - - # Create tenant-account join - from models.account import TenantAccountJoin, TenantAccountRole - - join = TenantAccountJoin( - tenant_id=tenant.id, - account_id=account.id, - role=TenantAccountRole.OWNER, - current=True, - ) - db_session_with_containers.add(join) - db_session_with_containers.commit() - - # Set current tenant for account - account.set_current_tenant_with_session(tenant, session=db_session_with_containers) - - return account, tenant - - def _create_test_app( - self, db_session_with_containers: Session, mock_external_service_dependencies, tenant, account - ): - """ - Helper method to create a test app for testing. - - Args: - db_session_with_containers: Database session from testcontainers infrastructure - mock_external_service_dependencies: Mock dependencies - tenant: Tenant instance - account: Account instance - - Returns: - App: Created app instance - """ - fake = Faker() - - # Create app - app = App( - tenant_id=tenant.id, - name=fake.company(), - mode=AppMode.CHAT, - icon_type="emoji", - icon="🤖", - icon_background="#FF6B6B", - enable_site=True, - enable_api=True, - api_rpm=100, - api_rph=10, - is_demo=False, - is_public=False, - created_by=account.id, - updated_by=account.id, - ) - - db_session_with_containers.add(app) - db_session_with_containers.commit() - - # Create app model config - app_model_config = AppModelConfig( - app_id=app.id, - provider="openai", - model="gpt-4", - configs={}, - created_by=account.id, - updated_by=account.id, - ) - db_session_with_containers.add(app_model_config) - db_session_with_containers.commit() - - # Link app model config to app - app.app_model_config_id = app_model_config.id - db_session_with_containers.commit() - - return app - - def test_convert_to_workflow_success(self, db_session_with_containers: Session, mock_external_service_dependencies): - """ - Test successful conversion of app to workflow. - - This test verifies: - - Proper app to workflow conversion - - Correct database state after conversion - - Proper relationship establishment - - Workflow creation with correct configuration - """ - # Arrange: Create test data - fake = Faker() - account, tenant = self._create_test_account_and_tenant( - db_session_with_containers, mock_external_service_dependencies - ) - app = self._create_test_app(db_session_with_containers, mock_external_service_dependencies, tenant, account) - - # Act: Execute the conversion - workflow_converter = WorkflowConverter() - new_app = workflow_converter.convert_to_workflow( - app_model=app, - account=account, - name="Test Workflow App", - icon_type="emoji", - icon="🚀", - icon_background="#4CAF50", - session=db_session_with_containers, - ) - - # Assert: Verify the expected outcomes - assert new_app is not None - assert new_app.name == "Test Workflow App" - assert new_app.mode == AppMode.ADVANCED_CHAT - assert new_app.icon_type == "emoji" - assert new_app.icon == "🚀" - assert new_app.icon_background == "#4CAF50" - assert new_app.tenant_id == app.tenant_id - assert new_app.created_by == account.id - - # Verify database state - - db_session_with_containers.refresh(new_app) - assert new_app.id is not None - - # Verify workflow was created - workflow = db_session_with_containers.query(Workflow).where(Workflow.app_id == new_app.id).first() - assert workflow is not None - assert workflow.tenant_id == app.tenant_id - assert workflow.type == "chat" - - def test_convert_to_workflow_without_app_model_config_error( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test error handling when app model config is missing. - - This test verifies: - - Proper error handling for missing app model config - - Correct exception type and message - - Database state remains unchanged - """ - # Arrange: Create test data without app model config - fake = Faker() - account, tenant = self._create_test_account_and_tenant( - db_session_with_containers, mock_external_service_dependencies - ) - - app = App( - tenant_id=tenant.id, - name=fake.company(), - mode=AppMode.CHAT, - icon_type="emoji", - icon="🤖", - icon_background="#FF6B6B", - enable_site=True, - enable_api=True, - api_rpm=100, - api_rph=10, - is_demo=False, - is_public=False, - created_by=account.id, - updated_by=account.id, - ) - - db_session_with_containers.add(app) - db_session_with_containers.commit() - - # Act & Assert: Verify proper error handling - workflow_converter = WorkflowConverter() - - # Check initial state - initial_workflow_count = db_session_with_containers.query(Workflow).count() - - with pytest.raises(ValueError, match="App model config is required"): - workflow_converter.convert_to_workflow( - app_model=app, - account=account, - name="Test Workflow App", - icon_type="emoji", - icon="🚀", - icon_background="#4CAF50", - session=db_session_with_containers, - ) - - # Verify database state remains unchanged - # The workflow creation happens in convert_app_model_config_to_workflow - # which is called before the app_model_config check, so we need to clean up - db_session_with_containers.rollback() - final_workflow_count = db_session_with_containers.query(Workflow).count() - assert final_workflow_count == initial_workflow_count - - def test_convert_app_model_config_to_workflow_success( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test successful conversion of app model config to workflow. - - This test verifies: - - Proper app model config to workflow conversion - - Correct workflow graph structure - - Proper node creation and configuration - - Database state management - """ - # Arrange: Create test data - fake = Faker() - account, tenant = self._create_test_account_and_tenant( - db_session_with_containers, mock_external_service_dependencies - ) - app = self._create_test_app(db_session_with_containers, mock_external_service_dependencies, tenant, account) - - # Act: Execute the conversion - workflow_converter = WorkflowConverter() - workflow = workflow_converter.convert_app_model_config_to_workflow( - app_model=app, - app_model_config=app.app_model_config_with_session(session=db_session_with_containers), - account_id=account.id, - session=db_session_with_containers, - ) - - # Assert: Verify the expected outcomes +@pytest.fixture +def conversion_source(db_session_with_containers: Session): + class ConversionSession(Session): + pass + + sessions = sessionmaker(db_session_with_containers.get_bind(), class_=ConversionSession, expire_on_commit=False) + account_id, tenant_id, app_id, config_id = (str(uuid4()) for _ in range(4)) + with sessions.begin() as session: + session.add_all( + [ + (account := Account(name="Converter", email=f"{account_id}@example.com", interface_language="en-US")), + (tenant := Tenant(name="Conversion workspace")), + ] + ) + account.id = account_id + tenant.id = tenant_id + session.flush() + session.add_all( + [ + TenantAccountJoin( + tenant_id=tenant_id, account_id=account_id, role=TenantAccountRole.OWNER, current=True + ), + App( + id=app_id, + tenant_id=tenant_id, + name="Source", + mode=AppMode.CHAT, + app_model_config_id=config_id, + enable_api=True, + enable_site=True, + created_by=account_id, + updated_by=account_id, + ), + ( + config := AppModelConfig( + app_id=app_id, + model=json.dumps( + {"provider": "openai", "name": "gpt-4", "mode": "chat", "completion_params": {}} + ), + pre_prompt="Hello {{name}}", + user_input_form=json.dumps([{"text-input": {"variable": "name", "label": "Name"}}]), + ) + ), + ] + ) + config.id = config_id + return sessions, RequestContext("conversion", None, account_id, tenant_id), app_id + + +@pytest.mark.parametrize("mode", [AppMode.CHAT, AppMode.COMPLETION]) +def test_conversion_creates_complete_target_before_notification(conversion_source, mode: AppMode): + sessions, context, app_id = conversion_source + with sessions.begin() as session: + app = session.get(App, app_id) + assert app is not None + app.mode = mode + observed = [] + + def created(*, event, account_id, backing_agent_id): + assert (account_id, backing_agent_id) == (context.account_id, None) + with sessions() as session: + assert session.scalar(select(Workflow).where(Workflow.app_id == event.id)) is not None + assert session.scalar(select(Site).where(Site.app_id == event.id)) is not None + assert session.scalar(select(InstalledApp).where(InstalledApp.app_id == event.id)) is not None + observed.append(event.id) + + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), decrypt_token=lambda *_: "key", notify_created=created + ) + new_id = service.convert(context, app_id, {"name": "Converted", "icon": "🚀"}) + assert observed == [new_id] + with sessions() as session: + app = session.get(App, new_id) + assert app is not None + assert app.name == "Converted" + assert app.icon == "🚀" + assert app.mode == (AppMode.ADVANCED_CHAT if mode == AppMode.CHAT else AppMode.WORKFLOW) + assert app.created_by == context.account_id + assert session.scalar(select(Workflow.id).where(Workflow.app_id == app_id)) is None + workflow = session.scalar(select(Workflow).where(Workflow.app_id == new_id)) assert workflow is not None - assert workflow.tenant_id == app.tenant_id - assert workflow.app_id == app.id - assert workflow.type == "chat" assert workflow.version == Workflow.VERSION_DRAFT - assert workflow.created_by == account.id - - # Verify workflow graph structure - graph = json.loads(workflow.graph) - assert "nodes" in graph - assert "edges" in graph - assert len(graph["nodes"]) > 0 - assert len(graph["edges"]) > 0 - - # Verify start node exists - start_node = next((node for node in graph["nodes"] if node["data"]["type"] == "start"), None) - assert start_node is not None - assert start_node["id"] == "start" - - # Verify LLM node exists - llm_node = next((node for node in graph["nodes"] if node["data"]["type"] == "llm"), None) - assert llm_node is not None - assert llm_node["id"] == "llm" - - # Verify answer node exists for chat mode - answer_node = next((node for node in graph["nodes"] if node["data"]["type"] == "answer"), None) - assert answer_node is not None - assert answer_node["id"] == "answer" - - # Verify database state - - db_session_with_containers.refresh(workflow) - assert workflow.id is not None - - # Verify features were set - features = json.loads(workflow._features) if workflow._features else {} - assert isinstance(features, dict) - - def test_convert_to_start_node_success( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test successful conversion to start node. - - This test verifies: - - Proper start node creation with variables - - Correct node structure and data - - Variable encoding and formatting - """ - # Arrange: Create test variables - variables = [ - VariableEntity( - variable="text_input", - label="Text Input", - type=VariableEntityType.TEXT_INPUT, - ), - VariableEntity( - variable="number_input", - label="Number Input", - type=VariableEntityType.NUMBER, - ), + assert [node["id"] for node in json.loads(workflow.graph)["nodes"]] == [ + "start", + "llm", + "answer" if mode == AppMode.CHAT else "end", ] - # Act: Execute the conversion - workflow_converter = WorkflowConverter() - start_node = workflow_converter._convert_to_start_node(variables=variables) - - # Assert: Verify the expected outcomes - assert start_node is not None - assert start_node["id"] == "start" - assert start_node["data"]["title"] == "START" - assert start_node["data"]["type"] == "start" - assert len(start_node["data"]["variables"]) == 2 - # Verify variable encoding - first_variable = start_node["data"]["variables"][0] - assert first_variable["variable"] == "text_input" - assert first_variable["label"] == "Text Input" - assert first_variable["type"] == "text-input" - - second_variable = start_node["data"]["variables"][1] - assert second_variable["variable"] == "number_input" - assert second_variable["label"] == "Number Input" - assert second_variable["type"] == "number" - - def test_convert_to_http_request_node_success( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test successful conversion to HTTP request node. - - This test verifies: - - Proper HTTP request node creation - - Correct API configuration and authorization - - Code node creation for response parsing - - External data variable mapping - """ - # Arrange: Create test data - fake = Faker() - account, tenant = self._create_test_account_and_tenant( - db_session_with_containers, mock_external_service_dependencies - ) - - app = self._create_test_app(db_session_with_containers, mock_external_service_dependencies, tenant, account) - - # Create API based extension - api_based_extension = APIBasedExtension( - tenant_id=tenant.id, - name="Test API Extension", - api_key="encrypted_api_key", - api_endpoint="https://api.example.com/test", - ) - - db_session_with_containers.add(api_based_extension) - db_session_with_containers.commit() - - # Mock encrypter - mock_external_service_dependencies["encrypter"].decrypt_token.return_value = "decrypted_api_key" - - variables = [ - VariableEntity( - variable="user_input", - label="User Input", - type=VariableEntityType.TEXT_INPUT, - ) - ] - - external_data_variables = [ - ExternalDataVariableEntity( - variable="external_data", type="api", config={"api_based_extension_id": api_based_extension.id} - ) - ] - - # Act: Execute the conversion - workflow_converter = WorkflowConverter() - nodes, external_data_variable_node_mapping = workflow_converter._convert_to_http_request_node( - app_model=app, - variables=variables, - external_data_variables=external_data_variables, - session=db_session_with_containers, - ) - - # Assert: Verify the expected outcomes - assert len(nodes) == 2 # HTTP request node + code node - assert len(external_data_variable_node_mapping) == 1 - - # Verify HTTP request node - http_request_node = nodes[0] - assert http_request_node["data"]["type"] == "http-request" - assert http_request_node["data"]["method"] == "post" - assert http_request_node["data"]["url"] == api_based_extension.api_endpoint - assert http_request_node["data"]["authorization"]["type"] == "api-key" - assert http_request_node["data"]["authorization"]["config"]["type"] == "bearer" - assert http_request_node["data"]["authorization"]["config"]["api_key"] == "decrypted_api_key" - - # Verify code node - code_node = nodes[1] - assert code_node["data"]["type"] == "code" - assert code_node["data"]["code_language"] == "python3" - assert "response_json" in code_node["data"]["variables"][0]["variable"] - - # Verify mapping - assert external_data_variable_node_mapping["external_data"] == code_node["id"] - - def test_convert_to_knowledge_retrieval_node_success( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test successful conversion to knowledge retrieval node. - - This test verifies: - - Proper knowledge retrieval node creation - - Correct dataset configuration - - Model configuration integration - - Query variable selector setup - """ - # Arrange: Create test data - fake = Faker() - account, tenant = self._create_test_account_and_tenant( - db_session_with_containers, mock_external_service_dependencies - ) - - # Create dataset config - dataset_config = DatasetEntity( - dataset_ids=["dataset_1", "dataset_2"], - retrieve_config=DatasetRetrieveConfigEntity( - retrieve_strategy=DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE, - top_k=10, - score_threshold=0.8, - reranking_model={"reranking_provider_name": "cohere", "reranking_model_name": "rerank-v2"}, - reranking_enabled=True, - ), - ) - - model_config = ModelConfigEntity( - provider="openai", - model="gpt-4", - mode=LLMMode.CHAT, - parameters={"temperature": 0.7}, - stop=[], - ) - - # Act: Execute the conversion for advanced chat mode - workflow_converter = WorkflowConverter() - node = workflow_converter._convert_to_knowledge_retrieval_node( - new_app_mode=AppMode.ADVANCED_CHAT, - dataset_config=dataset_config, - model_config=model_config, - ) - - # Assert: Verify the expected outcomes - assert node is not None - assert node["data"]["type"] == "knowledge-retrieval" - assert node["data"]["title"] == "KNOWLEDGE RETRIEVAL" - assert node["data"]["dataset_ids"] == ["dataset_1", "dataset_2"] - assert node["data"]["retrieval_mode"] == "multiple" - assert node["data"]["query_variable_selector"] == ["sys", "query"] - - # Verify multiple retrieval config - multiple_config = node["data"]["multiple_retrieval_config"] - assert multiple_config["top_k"] == 10 - assert multiple_config["score_threshold"] == 0.8 - assert multiple_config["reranking_model"]["reranking_provider_name"] == "cohere" - assert multiple_config["reranking_model"]["reranking_model_name"] == "rerank-v2" - - # Verify single retrieval config is None for multiple strategy - assert node["data"]["single_retrieval_config"] is None - - -@pytest.fixture -def default_variables(): - return [ - VariableEntity(variable="text_input", label="text-input", type=VariableEntityType.TEXT_INPUT), - VariableEntity(variable="paragraph", label="paragraph", type=VariableEntityType.PARAGRAPH), - VariableEntity(variable="select", label="select", type=VariableEntityType.SELECT), - ] - - -class TestConvertToHttpRequestNodeVariants: - """Tests for chatbot vs workflow differences in HTTP request node conversion.""" - - @staticmethod - def _setup(app_mode, default_variables, db_session_with_containers: Session): - app_model = App( - tenant_id="tenant_id", - mode=app_mode, - name="test", - icon_type="emoji", - icon="🤖", - icon_background="#FFFFFF", - ) - - ext = APIBasedExtension(tenant_id="tenant_id", name="api-1", api_key="enc", api_endpoint="https://dify.ai") - ext.id = "ext_id" - - converter = WorkflowConverter() - converter._get_api_based_extension = MagicMock(return_value=ext) - - from core.helper import encrypter - - encrypter.decrypt_token = MagicMock(return_value="api_key") - - ext_vars = [ - ExternalDataVariableEntity( - variable="external_variable", type="api", config={"api_based_extension_id": "ext_id"} - ) - ] - nodes, _ = converter._convert_to_http_request_node( - app_model=app_model, - variables=default_variables, - external_data_variables=ext_vars, - session=db_session_with_containers, - ) - return nodes - - def test_chatbot_query_uses_sys_query(self, default_variables, db_session_with_containers: Session): - nodes = self._setup(AppMode.CHAT, default_variables, db_session_with_containers) - - body = json.loads(nodes[0]["data"]["body"]["data"]) - assert body["params"]["query"] == "{{#sys.query#}}" - assert body["point"] == APIBasedExtensionPoint.APP_EXTERNAL_DATA_TOOL_QUERY - assert nodes[1]["data"]["type"] == "code" - - def test_workflow_query_is_empty(self, default_variables, db_session_with_containers: Session): - nodes = self._setup(AppMode.WORKFLOW, default_variables, db_session_with_containers) - - body = json.loads(nodes[0]["data"]["body"]["data"]) - assert body["params"]["query"] == "" - - -class TestConvertToKnowledgeRetrievalNodeVariants: - """Tests for chatbot vs workflow differences in knowledge retrieval node.""" - - @staticmethod - def _dataset_config(query_variable=None): - return DatasetEntity( - dataset_ids=["ds1", "ds2"], - retrieve_config=DatasetRetrieveConfigEntity( - query_variable=query_variable, - retrieve_strategy=DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE, - top_k=5, - score_threshold=0.8, - reranking_model={"reranking_provider_name": "cohere", "reranking_model_name": "rerank-english-v2.0"}, - reranking_enabled=True, - ), - ) - - @staticmethod - def _model_config(): - return ModelConfigEntity(provider="openai", model="gpt-4", mode="chat", parameters={}, stop=[]) - - def test_chatbot_uses_sys_query(self): - node = WorkflowConverter()._convert_to_knowledge_retrieval_node( - new_app_mode=AppMode.ADVANCED_CHAT, - dataset_config=self._dataset_config(), - model_config=self._model_config(), - ) - assert node["data"]["query_variable_selector"] == ["sys", "query"] - - def test_workflow_uses_start_variable(self): - node = WorkflowConverter()._convert_to_knowledge_retrieval_node( - new_app_mode=AppMode.WORKFLOW, - dataset_config=self._dataset_config(query_variable="query"), - model_config=self._model_config(), - ) - assert node["data"]["query_variable_selector"] == ["start", "query"] - - -class TestConvertToLlmNode: - """Tests for LLM node conversion across model modes and prompt types.""" - - @staticmethod - def _model_config(model, mode): - return ModelConfigEntity( - provider="openai", - model=model, - mode=mode.value, - parameters={}, - stop=[], - ) - - @staticmethod - def _graph(default_variables): - start = WorkflowConverter()._convert_to_start_node(default_variables) - return {"nodes": [start], "edges": []} - - def test_simple_chat_model(self, default_variables): - prompt = PromptTemplateEntity( - prompt_type=PromptTemplateEntity.PromptType.SIMPLE, - simple_prompt_template="You are helpful {{text_input}}, {{paragraph}}, {{select}}.", - ) - node = WorkflowConverter()._convert_to_llm_node( - original_app_mode=AppMode.CHAT, - new_app_mode=AppMode.ADVANCED_CHAT, - model_config=self._model_config("gpt-4", LLMMode.CHAT), - graph=self._graph(default_variables), - prompt_template=prompt, - ) - assert node["data"]["type"] == "llm" - assert node["data"]["model"]["mode"] == LLMMode.CHAT.value - assert node["data"]["context"]["enabled"] is False - expected = "You are helpful {{#start.text_input#}}, {{#start.paragraph#}}, {{#start.select#}}.\n" - assert node["data"]["prompt_template"][0]["text"] == expected - - def test_simple_completion_model(self, default_variables): - prompt = PromptTemplateEntity( - prompt_type=PromptTemplateEntity.PromptType.SIMPLE, - simple_prompt_template="You are helpful {{text_input}}, {{paragraph}}, {{select}}.", - ) - node = WorkflowConverter()._convert_to_llm_node( - original_app_mode=AppMode.CHAT, - new_app_mode=AppMode.ADVANCED_CHAT, - model_config=self._model_config("gpt-3.5-turbo-instruct", LLMMode.COMPLETION), - graph=self._graph(default_variables), - prompt_template=prompt, - ) - assert node["data"]["model"]["mode"] == LLMMode.COMPLETION.value - expected = "You are helpful {{#start.text_input#}}, {{#start.paragraph#}}, {{#start.select#}}.\n" - assert node["data"]["prompt_template"]["text"] == expected - - def test_advanced_chat_model(self, default_variables): - prompt = PromptTemplateEntity( - prompt_type=PromptTemplateEntity.PromptType.ADVANCED, - advanced_chat_prompt_template=AdvancedChatPromptTemplateEntity( - messages=[ - AdvancedChatMessageEntity( - text="You are helpful named {{name}}.\n\nContext:\n{{#context#}}", - role=PromptMessageRole.SYSTEM, - ), - AdvancedChatMessageEntity(text="Hi.", role=PromptMessageRole.USER), - AdvancedChatMessageEntity(text="Hello!", role=PromptMessageRole.ASSISTANT), - ] - ), - ) - node = WorkflowConverter()._convert_to_llm_node( - original_app_mode=AppMode.CHAT, - new_app_mode=AppMode.ADVANCED_CHAT, - model_config=self._model_config("gpt-4", LLMMode.CHAT), - graph=self._graph(default_variables), - prompt_template=prompt, - ) - assert isinstance(node["data"]["prompt_template"], list) - assert len(node["data"]["prompt_template"]) == 3 - - def test_advanced_completion_model(self, default_variables): - prompt = PromptTemplateEntity( - prompt_type=PromptTemplateEntity.PromptType.ADVANCED, - advanced_completion_prompt_template=AdvancedCompletionPromptTemplateEntity( - prompt="You are helpful named {{name}}.\n\nContext:\n{{#context#}}\n\nHuman: hi\nAssistant: ", - role_prefix=AdvancedCompletionPromptTemplateEntity.RolePrefixEntity( - user="Human", assistant="Assistant" - ), - ), - ) - node = WorkflowConverter()._convert_to_llm_node( - original_app_mode=AppMode.CHAT, - new_app_mode=AppMode.ADVANCED_CHAT, - model_config=self._model_config("gpt-3.5-turbo-instruct", LLMMode.COMPLETION), - graph=self._graph(default_variables), - prompt_template=prompt, - ) - assert isinstance(node["data"]["prompt_template"], dict) - assert "text" in node["data"]["prompt_template"] +def test_workflow_write_failure_rolls_back_target_and_required_records(conversion_source): + sessions, context, app_id = conversion_source + + def reject_workflow(session, *_): + if any(isinstance(row, Workflow) for row in session.new): + raise RuntimeError("workflow write failed") + + event.listen(sessions.class_, "before_flush", reject_workflow) + notifications = [] + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), + decrypt_token=lambda *_: "key", + notify_created=lambda **kwargs: notifications.append(kwargs), + ) + with pytest.raises(RuntimeError, match="workflow write failed"): + service.convert(context, app_id, {}) + assert notifications == [] + with sessions() as session: + assert list(session.scalars(select(App.id).where(App.tenant_id == context.active_workspace_id))) == [app_id] + assert session.scalar(select(Workflow.id).where(Workflow.tenant_id == context.active_workspace_id)) is None + assert ( + session.scalar(select(InstalledApp.id).where(InstalledApp.tenant_id == context.active_workspace_id)) is None + ) + assert session.scalar(select(Site.id).where(Site.created_by == context.account_id)) is None diff --git a/api/tests/test_containers_integration_tests/services/workflow/test_workflow_deletion.py b/api/tests/test_containers_integration_tests/services/workflow/test_workflow_deletion.py index 3c70dea134f325..c314a15a62e732 100644 --- a/api/tests/test_containers_integration_tests/services/workflow/test_workflow_deletion.py +++ b/api/tests/test_containers_integration_tests/services/workflow/test_workflow_deletion.py @@ -1,18 +1,18 @@ -"""Testcontainers integration tests for WorkflowService.delete_workflow.""" +"""Testcontainers integration tests for WorkflowDefinitionStore.delete_workflow.""" import json from uuid import uuid4 import pytest -from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.orm import Session -from extensions.ext_database import db from models.account import Account, Tenant, TenantAccountJoin from models.model import App from models.tools import WorkflowToolProvider from models.workflow import Workflow +from repositories.workflow.definition_repository import WorkflowDefinitionStore +from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError from services.workflow_ref_service import WorkflowRef -from services.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError, WorkflowService class TestWorkflowDeletion: @@ -110,13 +110,13 @@ def test_delete_workflow_success(self, db_session_with_containers: Session): db_session_with_containers.commit() workflow_id = workflow.id - service = WorkflowService(sessionmaker(bind=db.engine)) - result = service.delete_workflow( + result = WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=tenant.id, owner_id=app.id, workflow_id=workflow_id), ) assert result == [] + db_session_with_containers.flush() db_session_with_containers.expire_all() assert db_session_with_containers.get(Workflow, workflow_id) is None @@ -128,9 +128,8 @@ def test_delete_draft_workflow_raises_error(self, db_session_with_containers: Se ) db_session_with_containers.commit() - service = WorkflowService(sessionmaker(bind=db.engine)) with pytest.raises(DraftWorkflowDeletionError): - service.delete_workflow( + WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=tenant.id, owner_id=app.id, workflow_id=workflow.id), ) @@ -145,9 +144,8 @@ def test_delete_workflow_in_use_by_app_raises_error(self, db_session_with_contai app.workflow_id = workflow.id db_session_with_containers.commit() - service = WorkflowService(sessionmaker(bind=db.engine)) with pytest.raises(WorkflowInUseError, match="currently in use by app"): - service.delete_workflow( + WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=tenant.id, owner_id=app.id, workflow_id=workflow.id), ) @@ -161,9 +159,8 @@ def test_delete_workflow_published_as_tool_raises_error(self, db_session_with_co self._create_tool_provider(db_session_with_containers, tenant=tenant, app=app, account=account, version="1.0") db_session_with_containers.commit() - service = WorkflowService(sessionmaker(bind=db.engine)) with pytest.raises(WorkflowInUseError, match="published as a tool"): - service.delete_workflow( + WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=tenant.id, owner_id=app.id, workflow_id=workflow.id), ) diff --git a/api/tests/test_containers_integration_tests/tasks/test_mail_human_input_delivery_task.py b/api/tests/test_containers_integration_tests/tasks/test_mail_human_input_delivery_task.py index f79383906ea4fe..9e23f0407de392 100644 --- a/api/tests/test_containers_integration_tests/tasks/test_mail_human_input_delivery_task.py +++ b/api/tests/test_containers_integration_tests/tasks/test_mail_human_input_delivery_task.py @@ -10,13 +10,13 @@ from core.app.app_config.entities import WorkflowUIBasedAppConfig from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity from core.app.layers.pause_state_persist_layer import WorkflowResumptionContext -from core.repositories.human_input_repository import FormCreateParams, HumanInputFormRepositoryImpl from extensions.ext_storage import storage from graphon.enums import WorkflowExecutionStatus from graphon.runtime import GraphRuntimeState, VariablePool from models.account import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom from models.human_input import HumanInputDelivery, HumanInputForm, HumanInputFormRecipient +from models.human_input_contracts import FormCreateParams from models.human_input_delivery import ( EmailDeliveryConfig, EmailDeliveryMethod, @@ -27,6 +27,7 @@ from models.human_input_entities import HumanInputNodeData from models.model import AppMode from models.workflow import WorkflowPause, WorkflowRun, WorkflowType +from repositories.human_input.form_repository import HumanInputFormRepositoryImpl from tasks.mail_human_input_delivery_task import dispatch_human_input_email_task diff --git a/api/tests/test_containers_integration_tests/tasks/test_rag_pipeline_run_tasks.py b/api/tests/test_containers_integration_tests/tasks/test_rag_pipeline_run_tasks.py index 5eea985fdcbe95..0ac8ed0425d13d 100644 --- a/api/tests/test_containers_integration_tests/tasks/test_rag_pipeline_run_tasks.py +++ b/api/tests/test_containers_integration_tests/tasks/test_rag_pipeline_run_tasks.py @@ -36,7 +36,9 @@ class TestRagPipelineRunTasks: @pytest.fixture def mock_pipeline_generator(self): """Mock PipelineGenerator._generate method.""" - with patch("core.app.apps.pipeline.pipeline_generator.PipelineGenerator._generate") as mock_generate: + with patch( + "services.workflow.execution.adapters.pipeline.pipeline_generator.PipelineGenerator._generate" + ) as mock_generate: # Mock the _generate method to return a simple response mock_generate.return_value = {"answer": "Test response", "metadata": {"test": "data"}} yield mock_generate diff --git a/api/tests/test_containers_integration_tests/trigger/test_trigger_e2e.py b/api/tests/test_containers_integration_tests/trigger/test_trigger_e2e.py index 7bc578ed6a521c..beea1b4b61c2b0 100644 --- a/api/tests/test_containers_integration_tests/trigger/test_trigger_e2e.py +++ b/api/tests/test_containers_integration_tests/trigger/test_trigger_e2e.py @@ -11,7 +11,7 @@ from flask import Flask, Response from flask.testing import FlaskClient from sqlalchemy import select -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from configs import dify_config from core.plugin.entities.request import TriggerInvokeEventResponse @@ -39,6 +39,7 @@ WorkflowWebhookTrigger, ) from models.workflow import Workflow +from repositories.credentials.query_repository import CredentialQueryRepository from schedule import workflow_schedule_task from schedule.workflow_schedule_task import poll_workflow_schedules from services.system_feature_service import SystemFeatureService @@ -121,7 +122,13 @@ def test_publish_blocks_start_and_trigger_coexistence( ) with pytest.raises(ValueError, match="Start node and trigger nodes cannot coexist"): - workflow_service.publish_workflow(session=db_session_with_containers, app_model=app_model, account=account) + workflow_service.validate_publication( + app_model, + draft_workflow, + credentials=CredentialQueryRepository( + session_factory=sessionmaker(bind=db_session_with_containers.get_bind()) + ), + ) def test_trigger_url_uses_config_base(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/api/tests/unit_tests/commands/test_data_migration_commands.py b/api/tests/unit_tests/commands/test_data_migration_commands.py index ad53964d135a3f..8924ef4e5e58c9 100644 --- a/api/tests/unit_tests/commands/test_data_migration_commands.py +++ b/api/tests/unit_tests/commands/test_data_migration_commands.py @@ -6,6 +6,7 @@ import json from pathlib import Path +from unittest.mock import Mock from click.testing import CliRunner from sqlalchemy.engine import Engine @@ -18,6 +19,7 @@ export_migration_data_template, import_migration_data, ) +from extensions import ext_application_services from services.data_migration.entities import ( ConflictStrategy, ExportResult, @@ -26,6 +28,7 @@ MigrationPackage, ReportContext, ) +from services.workflow.console_service import ConsoleWorkflowService def test_export_command_requires_input_and_output(): @@ -89,7 +92,9 @@ def test_export_template_command_requires_overwrite_for_existing_output(tmp_path assert "already exists" in result.output -def test_export_command_uses_cli_owned_session(monkeypatch, tmp_path: Path, sqlite_engine: Engine): +def test_export_command_uses_cli_owned_session( + monkeypatch, tmp_path: Path, sqlite_engine: Engine, workflow_application +): captured: dict[str, object] = {} input_file = tmp_path / "export-config.json" output_file = tmp_path / "migration-package.json" @@ -97,6 +102,10 @@ def test_export_command_uses_cli_owned_session(monkeypatch, tmp_path: Path, sqli package = MigrationPackage.from_mapping({"metadata": {"version": "1", "source_scope": "single"}}) class FakeMigrationExportService: + def __init__(self, *, workflow_tools, tool_providers): + assert tool_providers is workflow_application.tools.tool_providers + assert workflow_tools is workflow_application.tools.workflows + def export(self, selection, *, session): captured["session"] = session captured["selection"] = selection @@ -126,7 +135,9 @@ def save_package(self, package_to_save, path, *, overwrite): assert captured["overwrite"] is False -def test_import_command_uses_cli_owned_session(monkeypatch, tmp_path: Path, sqlite_engine: Engine): +def test_import_command_uses_cli_owned_session( + monkeypatch, tmp_path: Path, sqlite_engine: Engine, workflow_application +): captured: dict[str, object] = {} input_file = tmp_path / "migration-package.json" input_file.write_text("{}") @@ -141,7 +152,20 @@ def test_import_command_uses_cli_owned_session(monkeypatch, tmp_path: Path, sqli } ) + publisher = Mock(spec=ConsoleWorkflowService) + services = Mock(spec=ext_application_services.ApplicationServices) + services.console_workflows = publisher + services.tools = workflow_application.tools + monkeypatch.setattr(data_migration, "application_services", lambda: services) + class FakeMigrationImportService: + def __init__(self, *, workflows, app_dsl, workflow_tools): + assert workflow_tools is services.tools.workflows + from extensions.application_services.workflow import build_app_dsl_service + + assert workflows is publisher + assert app_dsl is build_app_dsl_service + def import_package(self, request, *, session): captured["session"] = session captured["request"] = request diff --git a/api/tests/unit_tests/conftest.py b/api/tests/unit_tests/conftest.py index 0618eee34880e1..41514d9395c1d0 100644 --- a/api/tests/unit_tests/conftest.py +++ b/api/tests/unit_tests/conftest.py @@ -17,11 +17,109 @@ if TYPE_CHECKING: from extensions.ext_application_services import ApplicationServices + from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.unit_tests.account_domain import AccountDomain # Getting the absolute path of the current file's directory ABS_PATH = os.path.dirname(os.path.abspath(__file__)) + +@pytest.fixture +def workflow_suggestions(sqlite_session_factory: sessionmaker[Session]): + from extensions.application_services.workflow import build_workflow_suggestions + + return build_workflow_suggestions(sqlite_session_factory) + + +@pytest.fixture +def workflow_queries(sqlite_session_factory: sessionmaker[Session]): + from repositories.tools.workflow_repository import WorkflowToolRepository + + return WorkflowToolRepository(sqlite_session_factory) + + +@pytest.fixture +def tool_providers(sqlite_session_factory: sessionmaker[Session]): + from repositories.tools.provider_repository import ToolProviderRepository + + return ToolProviderRepository(sqlite_session_factory) + + +@pytest.fixture +def workflow_tools(sqlite_session_factory: sessionmaker[Session]): + from extensions.application_services.tools import build_tool_services + + return build_tool_services(sqlite_session_factory).workflows + + +@pytest.fixture +def app_records(sqlite_session_factory: sessionmaker[Session]): + from repositories.app.generation_repository import AppGenerationRepository + + return AppGenerationRepository(sqlite_session_factory) + + +@pytest.fixture +def annotation_replies(sqlite_session_factory: sessionmaker[Session]): + from extensions.application_services.annotation import build_annotation_replies + + return build_annotation_replies(sqlite_session_factory) + + +@pytest.fixture +def workflow_contexts(sqlite_session_factory: sessionmaker[Session]): + from repositories.workflow.runtime_context_repository import WorkflowRuntimeContextRepository + + return WorkflowRuntimeContextRepository(sqlite_session_factory) + + +@pytest.fixture +def workflow_runtime(sqlite_session_factory: sessionmaker[Session]): + from extensions.application_services.workflow import build_workflow_execution_dependencies + + return build_workflow_execution_dependencies(sqlite_session_factory) + + +@pytest.fixture +def workflow_variables(sqlite_session_factory: sessionmaker[Session]) -> WorkflowExecutionVariables: + from extensions.application_services.workflow_variables import build_workflow_variable_service + + return build_workflow_variable_service(database_client=sqlite_session_factory) + + +@pytest.fixture +def console_workflow_variables(sqlite_session_factory: sessionmaker[Session]): + from extensions.application_services.workflow_variables import ( + build_console_workflow_variables, + build_workflow_variable_service, + ) + + return build_console_workflow_variables( + database_client=sqlite_session_factory, + variables=build_workflow_variable_service(database_client=sqlite_session_factory), + ) + + +@pytest.fixture +def workflow_application( + app: Flask, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch +) -> ApplicationServices: + from unittest.mock import Mock + + from enums import DeploymentEdition + from extensions.ext_application_services import build_application_services + from extensions.ext_redis import RedisClientWrapper + + services = build_application_services( + database_client=sqlite_session_factory, + deployment_edition=DeploymentEdition.COMMUNITY, + initialization_password="", + redis=Mock(spec=RedisClientWrapper), + ) + monkeypatch.setitem(app.extensions, "application_services", services) + return services + + # Getting the absolute path of the project's root directory PROJECT_DIR = os.path.abspath(os.path.join(ABS_PATH, os.pardir, os.pardir)) @@ -316,7 +414,7 @@ def account_application_services( @pytest.fixture -def app_services(sqlite_session_factory: sessionmaker[Session]) -> AppServices: +def app_services(sqlite_session_factory: sessionmaker[Session], tool_providers, workflow_queries) -> AppServices: from unittest.mock import Mock from extensions.application_services.app import build_app_services @@ -328,6 +426,8 @@ def app_services(sqlite_session_factory: sessionmaker[Session]) -> AppServices: database_client=sqlite_session_factory, oauth=_build_oauth_server_service(database_client=sqlite_session_factory, redis=redis_client), recommended_packages=RecommendedAppPackageService(sources=Mock(), exporter=Mock()), + tool_providers=tool_providers, + workflow_queries=workflow_queries, ) @@ -339,61 +439,6 @@ def application_tags(sqlite_session_factory: sessionmaker[Session]) -> TagApplic return TagApplicationService(tags=TagRepository(sqlite_session_factory)) -@pytest.fixture -def workflow_queries(sqlite_session_factory: sessionmaker[Session]): - from repositories.tools.workflow_repository import WorkflowToolRepository - - return WorkflowToolRepository(sqlite_session_factory) - - -@pytest.fixture -def tool_providers(sqlite_session_factory: sessionmaker[Session]): - from repositories.tools.provider_repository import ToolProviderRepository - - return ToolProviderRepository(sqlite_session_factory) - - -@pytest.fixture -def workflow_tools(sqlite_session_factory: sessionmaker[Session]): - from extensions.application_services.tools import build_tool_services - - return build_tool_services(sqlite_session_factory).workflows - - -@pytest.fixture -def workflow_variables(sqlite_session_factory: sessionmaker[Session]): - from extensions.application_services.workflow_variables import build_workflow_variable_service - - return build_workflow_variable_service(database_client=sqlite_session_factory) - - -@pytest.fixture -def console_workflow_variables(sqlite_session_factory: sessionmaker[Session]): - from extensions.application_services.workflow_variables import ( - build_console_workflow_variables, - build_workflow_variable_service, - ) - - return build_console_workflow_variables( - database_client=sqlite_session_factory, - variables=build_workflow_variable_service(database_client=sqlite_session_factory), - ) - - -@pytest.fixture -def app_records(sqlite_session_factory: sessionmaker[Session]): - from repositories.app.generation_repository import AppGenerationRepository - - return AppGenerationRepository(sqlite_session_factory) - - -@pytest.fixture -def annotation_replies(sqlite_session_factory: sessionmaker[Session]): - from extensions.application_services.annotation import build_annotation_replies - - return build_annotation_replies(sqlite_session_factory) - - @pytest.fixture def human_forms(sqlite_session_factory): from functools import partial @@ -401,44 +446,3 @@ def human_forms(sqlite_session_factory): from repositories.human_input.form_repository import HumanInputFormRepositoryImpl return partial(HumanInputFormRepositoryImpl, sessions=sqlite_session_factory) - - -@pytest.fixture -def workflow_contexts(sqlite_session_factory: sessionmaker[Session]): - from repositories.workflow.runtime_context_repository import WorkflowRuntimeContextRepository - - return WorkflowRuntimeContextRepository(sqlite_session_factory) - - -@pytest.fixture -def workflow_runtime(sqlite_session_factory: sessionmaker[Session]): - from extensions.application_services.workflow import build_workflow_execution_dependencies - - return build_workflow_execution_dependencies(sqlite_session_factory) - - -@pytest.fixture -def workflow_suggestions(sqlite_session_factory: sessionmaker[Session]): - from extensions.application_services.workflow import build_workflow_suggestions - - return build_workflow_suggestions(sqlite_session_factory) - - -@pytest.fixture -def workflow_application( - app: Flask, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch -) -> ApplicationServices: - from unittest.mock import Mock - - from enums import DeploymentEdition - from extensions.ext_application_services import build_application_services - from extensions.ext_redis import RedisClientWrapper - - services = build_application_services( - database_client=sqlite_session_factory, - deployment_edition=DeploymentEdition.COMMUNITY, - initialization_password="", - redis=Mock(spec=RedisClientWrapper), - ) - monkeypatch.setitem(app.extensions, "application_services", services) - return services diff --git a/api/tests/unit_tests/controllers/conftest.py b/api/tests/unit_tests/controllers/conftest.py index 5974afc04fd6dd..7b90a95ec4db05 100644 --- a/api/tests/unit_tests/controllers/conftest.py +++ b/api/tests/unit_tests/controllers/conftest.py @@ -2,15 +2,14 @@ import sys from dataclasses import dataclass -from types import SimpleNamespace from unittest.mock import create_autospec import pytest from flask import Flask from sqlalchemy.orm import Session, sessionmaker -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from extensions.application_services.app import AppServices +from extensions.application_services.tools import ToolServices, build_tool_services from extensions.ext_application_services import ( _batch_get_enterprise_webapp_access_modes, _batch_get_enterprise_webapp_user_permissions, @@ -23,15 +22,28 @@ from services.credentials.query import CredentialQuery from services.data_source.provider_service import DatasourceProviderService from services.enterprise.enterprise_service import EnterpriseService +from services.rag_pipeline.execution_service import RagPipelineExecutionService from services.tag_application_service import TagApplicationService from services.webapp_access_adapters import EnterpriseWebAppAccessPolicyGateway from services.webapp_access_query_service import WebAppAccessQueryService +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator +from services.workflow.variable_contracts import WorkflowExecutionVariables @pytest.fixture(autouse=True) -def datasource_application_dependencies(monkeypatch: pytest.MonkeyPatch) -> None: - registry = SimpleNamespace( - data_sources=SimpleNamespace(providers=create_autospec(DatasourceProviderService, instance=True, spec_set=True)) +def datasource_application_dependencies( + monkeypatch: pytest.MonkeyPatch, workflow_variables: WorkflowExecutionVariables, sqlite_session_factory +) -> None: + registry = DatasourceApplicationStub( + tools=build_tool_services(sqlite_session_factory), + workflow_variables=workflow_variables, + knowledge=PipelineKnowledgeStub( + pipeline_generator=create_autospec(PipelineGenerator, instance=True, spec_set=True), + pipeline_execution=create_autospec(RagPipelineExecutionService, instance=True, spec_set=True), + ), + data_sources=PipelineDataSourceStub( + providers=create_autospec(DatasourceProviderService, instance=True, spec_set=True) + ), ) for name in ( "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow", @@ -46,6 +58,7 @@ def datasource_application_dependencies(monkeypatch: pytest.MonkeyPatch) -> None @dataclass(frozen=True) class PipelineKnowledgeStub: pipeline_generator: PipelineGenerator + pipeline_execution: RagPipelineExecutionService @dataclass(frozen=True) @@ -53,21 +66,38 @@ class PipelineDataSourceStub: providers: DatasourceProviderService +@dataclass(frozen=True) +class DatasourceApplicationStub: + tools: ToolServices + data_sources: PipelineDataSourceStub + workflow_variables: WorkflowExecutionVariables + knowledge: PipelineKnowledgeStub + + @dataclass(frozen=True) class PipelineApplicationStub: + tools: ToolServices + workflow_variables: WorkflowExecutionVariables knowledge: PipelineKnowledgeStub credential_queries: CredentialQuery data_sources: PipelineDataSourceStub @pytest.fixture -def pipeline_application(monkeypatch: pytest.MonkeyPatch) -> PipelineGenerator: +def pipeline_application( + monkeypatch: pytest.MonkeyPatch, workflow_variables: WorkflowExecutionVariables, sqlite_session_factory +) -> PipelineGenerator: from controllers.console.datasets.rag_pipeline import rag_pipeline_workflow as console_workflow from controllers.service_api.dataset.rag_pipeline import rag_pipeline_workflow as service_api_workflow generator = create_autospec(PipelineGenerator, instance=True, spec_set=True) registry = PipelineApplicationStub( - knowledge=PipelineKnowledgeStub(pipeline_generator=generator), + tools=build_tool_services(sqlite_session_factory), + workflow_variables=workflow_variables, + knowledge=PipelineKnowledgeStub( + pipeline_generator=generator, + pipeline_execution=create_autospec(RagPipelineExecutionService, instance=True, spec_set=True), + ), credential_queries=create_autospec(CredentialQuery, instance=True, spec_set=True), data_sources=PipelineDataSourceStub( providers=create_autospec(DatasourceProviderService, instance=True, spec_set=True) diff --git a/api/tests/unit_tests/controllers/console/agent/test_agent_controllers.py b/api/tests/unit_tests/controllers/console/agent/test_agent_controllers.py index 5a378cd1fe2997..e618ebb33548a8 100644 --- a/api/tests/unit_tests/controllers/console/agent/test_agent_controllers.py +++ b/api/tests/unit_tests/controllers/console/agent/test_agent_controllers.py @@ -7,8 +7,9 @@ import pytest from flask import Flask from sqlalchemy.orm import Session -from werkzeug.exceptions import Forbidden, InternalServerError, NotFound +from werkzeug.exceptions import NotFound as HttpNotFound +from controllers.common.errors import AccessDeniedError, InternalServerError, NotFoundError from controllers.console import console_ns from controllers.console.agent import composer as composer_controller from controllers.console.agent import roster as roster_controller @@ -61,6 +62,7 @@ ) from core.app.entities.app_invoke_entities import InvokeFrom from enums import CloudPlan, DeploymentEdition +from extensions.ext_application_services import ApplicationServices from models.account import Account, TenantAccountRole from models.agent import Agent, AgentConfigDraftType, AgentScope, AgentSource, AgentStatus from models.enums import ApiTokenType, ConversationFromSource, CustomizeTokenStrategy, TagType @@ -317,6 +319,7 @@ def account_id() -> str: return "account-1" +@pytest.mark.usefixtures("workflow_application") def test_agent_app_list_and_create_use_agent_route( app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str, sqlite_session: Session ) -> None: @@ -592,6 +595,7 @@ def create_app(self, tenant_id: str, params: object, account: object, *, session assert create_params.agent_role == "" +@pytest.mark.usefixtures("workflow_application") def test_agent_app_detail_update_delete_resolve_app_from_agent_id( app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str, sqlite_session: Session ) -> None: @@ -1001,6 +1005,7 @@ def update_app_api_status(self, app_obj: object, enable_api: bool, *, session: o resolve_app.assert_called_once_with(unbound_session, tenant_id="tenant-1", agent_id=agent_id) +@pytest.mark.usefixtures("workflow_application") def test_agent_app_update_allows_empty_role( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session ) -> None: @@ -1191,7 +1196,7 @@ def test_agent_version_restore_requires_cloud_paid_plan( api = AgentRosterVersionRestoreApi() if not allowed: - with pytest.raises(Forbidden, match="This feature requires a paid plan.") as exc_info: + with pytest.raises(AccessDeniedError, match="This feature requires a paid plan.") as exc_info: unwrap(api.post)(api, session, "tenant-1", _account(), agent_id, version_id) assert exc_info.value.code == 403 restore.assert_not_called() @@ -1402,6 +1407,7 @@ def get_statistics_summary(self, *, app, agent_id, params): def test_workflow_composer_get_put_validate_candidates_impact_and_save( app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str ) -> None: + monkeypatch.setattr(composer_controller, "application_services", lambda: Mock(workflow_variables=Mock())) app_model = _app_detail_obj(id="app-1") payload = { "variant": ComposerVariant.WORKFLOW.value, @@ -1584,7 +1590,7 @@ def test_workflow_impact_returns_empty_without_version(app: Flask) -> None: def test_agent_composer_routes_resolve_app_from_agent_id( - app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str + app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str, *, workflow_application ) -> None: agent_id = "00000000-0000-0000-0000-000000000001" captured: dict[str, object] = {} @@ -1811,7 +1817,11 @@ def create_finalization_message(**kwargs: object) -> dict[str, object]: def test_build_chat_finalization_helper_forces_debug_build_and_push_prompt( - app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str, unbound_session: Session + workflow_application: ApplicationServices, + app: Flask, + monkeypatch: pytest.MonkeyPatch, + account_id: str, + unbound_session: Session, ) -> None: app_model = _app_detail_obj(id="app-1", tenant_id="tenant-1", mode=AppMode.AGENT) captured: dict[str, object] = {} @@ -1851,6 +1861,7 @@ def generate(**kwargs: object) -> object: generate_call = cast(dict[str, object], captured["generate"]) assert generate_call["app_model"] is app_model assert generate_call["streaming"] is True + assert generate_call["variables"] is workflow_application.workflow_variables args = cast(dict[str, object], generate_call["args"]) assert args["draft_type"] == "debug_build" assert args["response_mode"] == "streaming" @@ -1922,6 +1933,7 @@ def test_drain_streaming_generate_response_raises_when_stream_ends_early() -> No ], ) def test_agent_chat_helper_resolves_scoped_conversation_and_forces_streaming( + workflow_application: ApplicationServices, app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str, @@ -1962,6 +1974,7 @@ def resolve_debug_conversation(**kwargs: object) -> str: assert captured["app_model"] is app_model assert captured["user"] is current_user assert captured["streaming"] is True + assert captured["variables"] is workflow_application.workflow_variables args = cast(dict[str, object], captured["args"]) assert args["response_mode"] == "streaming" assert args["conversation_id"] == "00000000-0000-0000-0000-000000000001" @@ -1973,7 +1986,11 @@ def resolve_debug_conversation(**kwargs: object) -> str: def test_agent_chat_helper_ignores_private_exit_intent_payload_key( - app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str, unbound_session: Session + workflow_application: ApplicationServices, + app: Flask, + monkeypatch: pytest.MonkeyPatch, + account_id: str, + unbound_session: Session, ) -> None: app_model = _app_detail_obj(id="app-1", tenant_id="tenant-1", mode=AppMode.AGENT) current_user = _account(account_id=account_id) @@ -2011,6 +2028,7 @@ def generate(**kwargs: object) -> dict[str, object]: assert result == {"response": {"answer": "ok"}} assert captured["streaming"] is True + assert captured["variables"] is workflow_application.workflow_variables args = cast(dict[str, object], captured["args"]) assert args["response_mode"] == "streaming" assert args["conversation_id"] == "debug-conversation-1" @@ -2050,7 +2068,7 @@ def test_agent_chat_helper_rejects_foreign_debug_conversation_before_generation( **payload_extra, } ): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): completion_controller._create_chat_message( current_tenant_id="tenant-1", current_user=_account(account_id=account_id), @@ -2165,7 +2183,7 @@ def get_app_backing_agent(self, **kwargs: object) -> object: @pytest.mark.parametrize( ("error", "expected"), [ - (completion_controller.services.errors.conversation.ConversationNotExistsError(), NotFound), + (completion_controller.services.errors.conversation.ConversationNotExistsError(), NotFoundError), ( completion_controller.services.errors.conversation.ConversationCompletedError(), completion_controller.ConversationCompletedError, @@ -2189,6 +2207,7 @@ def get_app_backing_agent(self, **kwargs: object) -> object: ], ) def test_agent_chat_helper_maps_generation_errors( + workflow_application: ApplicationServices, app: Flask, monkeypatch: pytest.MonkeyPatch, error: Exception, @@ -2196,12 +2215,15 @@ def test_agent_chat_helper_maps_generation_errors( unbound_session: Session, ) -> None: app_model = _app_detail_obj(id="app-1", mode=AppMode.CHAT) - monkeypatch.setattr(completion_controller.AppGenerateService, "generate", lambda **_: (_ for _ in ()).throw(error)) + generate = Mock(side_effect=error) + monkeypatch.setattr(completion_controller.AppGenerateService, "generate", generate) with app.test_request_context(json={"inputs": {}, "query": "hello"}): with pytest.raises(expected): completion_controller._create_chat_message( current_user=_account(), app_model=app_model, session=unbound_session ) + generate.assert_called_once() + assert generate.call_args.kwargs["variables"] is workflow_application.workflow_variables def test_agent_chat_message_routes_resolve_app_from_agent_id( @@ -2384,7 +2406,7 @@ def test_list_agent_chat_messages_rejects_foreign_conversation( lambda **kwargs: (_ for _ in ()).throw(message_controller.ConversationNotExistsError()), ) with app.test_request_context(f"/console/api/agent/agent-1/chat-messages?conversation_id={conversation_id}"): - with pytest.raises(NotFound): + with pytest.raises(HttpNotFound): message_controller._list_chat_messages( args=message_controller.ChatMessagesQuery(conversation_id=conversation_id), session=unbound_session, diff --git a/api/tests/unit_tests/controllers/console/app/test_app_apis.py b/api/tests/unit_tests/controllers/console/app/test_app_apis.py index bb9ed018407d24..6b3591c9bd4929 100644 --- a/api/tests/unit_tests/controllers/console/app/test_app_apis.py +++ b/api/tests/unit_tests/controllers/console/app/test_app_apis.py @@ -13,8 +13,10 @@ from pydantic import ValidationError from sqlalchemy import Engine from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, NotFound +from werkzeug.exceptions import BadRequest as HttpBadRequest +from werkzeug.exceptions import NotFound as HttpNotFound +from controllers.common.errors import NotFoundError from controllers.console.app import ( annotation as annotation_module, ) @@ -39,9 +41,6 @@ from controllers.console.app import ( workflow_app_log as workflow_app_log_module, ) -from controllers.console.app import ( - workflow_draft_variable as workflow_draft_variable_module, -) from controllers.console.app import ( workflow_statistic as workflow_statistic_module, ) @@ -64,7 +63,6 @@ from controllers.console.app.workflow_app_log import WorkflowAppLogQuery from controllers.console.app.workflow_draft_variable import ( EnvironmentVariableUpdatePayload, - WorkflowDraftVariableListQuery, WorkflowDraftVariableUpdatePayload, ) from controllers.console.app.workflow_statistic import WorkflowStatisticQuery @@ -195,7 +193,7 @@ def test_completion_api_conversation_not_exists( with ( Session(sqlite_engine) as session, app.test_request_context("/", json={"inputs": {}, "model_config": {}, "query": "hi"}), - pytest.raises(NotFound), + pytest.raises(NotFoundError), ): method( api, @@ -440,7 +438,7 @@ def test_app_site_reset_maps_missing_site(self) -> None: with ( patch.object(site_module, "application_services", return_value=services), - pytest.raises(NotFound), + pytest.raises(HttpNotFound), ): method(api, context, app_id=uuid.UUID(APP_ID)) @@ -510,35 +508,6 @@ def test_workflow_variable_creation(self): payload = WorkflowDraftVariableUpdatePayload(name="var1", value="test") assert payload.name == "var1" - def test_workflow_variable_collection_get(self, database_app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - api = workflow_draft_variable_module.WorkflowVariableCollectionApi() - method = unwrap(api.get) - - class DummyDraftService: - def __init__(self, session: Session): - self.session = session - - def list_variables_without_values(self, **_kwargs): - assert self.session.get_bind() is db.engine - return {"items": [], "total": 0} - - class DummyWorkflowService: - def is_workflow_exist(self, *args, **kwargs): - return True - - monkeypatch.setattr(workflow_draft_variable_module, "WorkflowDraftVariableService", DummyDraftService) - monkeypatch.setattr(workflow_draft_variable_module, "WorkflowService", DummyWorkflowService) - - with database_app.test_request_context("/?page=1&limit=20"): - result = method( - api, - WorkflowDraftVariableListQuery(page=1, limit=20), - _make_account(), - app_model=_make_app("app-1"), - ) - - assert result == {"items": [], "total": 0} - def test_environment_variable_update_payload_preserves_full_replace_default(self) -> None: payload = EnvironmentVariableUpdatePayload(environment_variables=[]) @@ -564,48 +533,6 @@ def test_environment_variable_patch_payload_rejects_ambiguous_mutations(self, pa with pytest.raises(ValidationError): EnvironmentVariableUpdatePayload.model_validate(payload) - def test_environment_variable_collection_post_routes_patch_to_service( - self, database_app: Flask, monkeypatch: pytest.MonkeyPatch - ) -> None: - api = workflow_draft_variable_module.EnvironmentVariableCollectionApi() - method = unwrap(api.post) - captured: dict = {} - - class DummyWorkflowService: - def patch_draft_workflow_environment_variables(self, **kwargs) -> None: - captured.update(kwargs) - - def update_draft_workflow_environment_variables(self, **_kwargs) -> None: - raise AssertionError("patch request must not use full replacement") - - monkeypatch.setattr(workflow_draft_variable_module, "WorkflowService", DummyWorkflowService) - - with database_app.test_request_context( - "/", - json={ - "environment_variables": [{"id": "env-a", "name": "a", "value_type": "string", "value": "new-a"}], - "patch": True, - "deleted_environment_variable_ids": ["env-b"], - }, - ): - result = method( - api, - EnvironmentVariableUpdatePayload( - environment_variables=[{"id": "env-a", "name": "a", "value_type": "string", "value": "new-a"}], - patch=True, - deleted_environment_variable_ids=["env-b"], - ), - _make_account(), - app_model=_make_app(), - ) - - assert result == {"result": "success"} - assert [(variable.id, variable.value) for variable in captured["environment_variables"]] == [("env-a", "new-a")] - assert captured["deleted_environment_variable_ids"] == ["env-b"] - assert captured["app_model"].id == APP_ID - assert captured["account"].id == USER_ID - assert captured["session"].get_bind() is db.engine - class TestWorkflowStatisticEndpoints: def test_workflow_statistic_time_range(self): @@ -763,7 +690,7 @@ def test_workflow_statistic_invalid_time_range(self, monkeypatch: pytest.MonkeyP MagicMock(side_effect=ValueError("invalid range")), ) - with pytest.raises(BadRequest, match="invalid range"): + with pytest.raises(HttpBadRequest, match="invalid range"): workflow_statistic_module._resolve_statistic_time_range(WorkflowStatisticQuery()) @staticmethod @@ -849,3 +776,6 @@ def test_multiple_payload_types(self): ] assert len(payloads) == 3 assert all(p is not None for p in payloads) + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/controllers/console/app/test_app_response_models.py b/api/tests/unit_tests/controllers/console/app/test_app_response_models.py index ca1425f3947fe0..ae50f62ff473fe 100644 --- a/api/tests/unit_tests/controllers/console/app/test_app_response_models.py +++ b/api/tests/unit_tests/controllers/console/app/test_app_response_models.py @@ -351,7 +351,7 @@ def test_app_detail_with_site_includes_nested_serialization( def test_app_response_view_uses_the_caller_session_for_query_backed_fields( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session + monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, *, tool_providers, workflow_queries ): app_obj = _persist_response_graph(sqlite_session) decoy_tenant_id = "00000000-0000-0000-0000-000000000002" @@ -377,7 +377,9 @@ def load_annotation_reply(session, app_id): monkeypatch.setattr("services.app_service.load_annotation_reply_config", load_annotation_reply) - view = AppResponseView(app_obj, session=sqlite_session) + view = AppResponseView( + app_obj, session=sqlite_session, tool_providers=tool_providers, workflow_queries=workflow_queries + ) site = view.site workflow = view.workflow model_config = view.app_model_config diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow.py b/api/tests/unit_tests/controllers/console/app/test_workflow.py index 7b0075a2e0d471..388057fd89b769 100644 --- a/api/tests/unit_tests/controllers/console/app/test_workflow.py +++ b/api/tests/unit_tests/controllers/console/app/test_workflow.py @@ -1,1133 +1,245 @@ -from __future__ import annotations +"""Transport contracts; workflow behavior is tested at the use-case and adapter boundaries.""" import inspect -import json -from contextlib import contextmanager, nullcontext -from datetime import datetime -from types import SimpleNamespace +from dataclasses import dataclass +from datetime import UTC, datetime from unittest.mock import Mock +from uuid import UUID import pytest -from flask import Flask -from pydantic import ValidationError -from sqlalchemy.orm import Session -from werkzeug.exceptions import HTTPException, NotFound - -from controllers.common.errors import InvalidArgumentError -from controllers.console.app import workflow as workflow_module -from controllers.console.app.error import DraftWorkflowNotExist, DraftWorkflowNotSync -from core.workflow.llm_environment_variable import LLMEnvironmentVariable -from graphon.file import File, FileTransferMethod, FileType -from graphon.variables import SecretVariable, StringVariable -from graphon.variables.variables import RAGPipelineVariable -from models.account import Account -from models.model import App, AppMode -from models.workflow import Workflow, WorkflowType -from tests.unit_tests.config_override import apply_config_overrides - - -@pytest.fixture(autouse=True) -def _identity_workflow_encryption(monkeypatch: pytest.MonkeyPatch) -> None: - """Keep variable serialization focused on ORM behavior, not the external key provider.""" - - monkeypatch.setattr(workflow_module.encrypter, "encrypt_token", lambda *, tenant_id, token: token) - monkeypatch.setattr(workflow_module.encrypter, "decrypt_token", lambda *, tenant_id, token: token) - - -def _account() -> Account: - account = Account(name="Alice", email="alice@example.com") - account.id = "user-1" - return account - - -def _app(*, app_id: str = "app", tenant_id: str = "t1") -> App: - return App( - id=app_id, - tenant_id=tenant_id, - name="Workflow App", - description="", - mode=AppMode.WORKFLOW, - enable_site=True, - enable_api=True, - max_active_requests=0, - ) - +from flask import Flask, Response +from flask_restx import Resource +from pydantic import BaseModel, JsonValue, ValidationError +from werkzeug.exceptions import HTTPException + +from controllers.common.errors import InvalidArgumentError, InvalidRequestError, NotFoundError +from controllers.console.app import workflow as controller +from controllers.console.app.error import DraftWorkflowNotSync +from graphon.variables.exc import VariableError +from libs.external_api import ExternalApi +from machinery.context import RequestContext +from services.agent.workflow_contracts import WorkflowAgentBindingStore +from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError +from services.workflow.console_service import ConsoleWorkflowService +from services.workflow.contracts import WorkflowChange, WorkflowOwner, WorkflowPublication, WorkflowTriggerError + +CONTEXT = RequestContext("request-1", "trace-1", "account-1", "tenant-1") +APP_ID = UUID("00000000-0000-0000-0000-000000000001") + + +@dataclass +class Services: + console_workflows: ConsoleWorkflowService[WorkflowAgentBindingStore] + + +@pytest.fixture(name="workflows") +def workflow_use_cases(monkeypatch: pytest.MonkeyPatch) -> Mock: + use_cases = Mock(spec=ConsoleWorkflowService) + monkeypatch.setattr(controller, "application_services", lambda: Services(use_cases)) + return use_cases + + +def invoke(resource: type[Resource], method: str = "post", *args: object, **kwargs: object) -> object: + api = resource() + return inspect.unwrap(getattr(api, method))(api, *args, request_context=CONTEXT, app_id=APP_ID, **kwargs) + + +@pytest.mark.parametrize("content_type", ["application/json", "text/plain"]) +def test_sync_parses_transport_and_preserves_patch(app: Flask, workflows: Mock, content_type: str) -> None: + workflows.sync.return_value = WorkflowChange("new-hash", datetime(2024, 1, 1, tzinfo=UTC)) + body: dict[str, JsonValue] = { + "graph": {"nodes": []}, + "features": {}, + "hash": "old-hash", + "_is_collaborative": True, + "environment_variable_patch": { + "environment_variables": [{"id": "env-1", "name": "KEY", "value_type": "secret", "value": "masked"}], + "deleted_environment_variable_ids": ["env-2"], + }, + } + import json -def _make_workflow(**overrides) -> Workflow: - graph = overrides.pop("graph_dict", {"nodes": [], "edges": []}) - features = overrides.pop("features_dict", {"file_upload": {"enabled": False}}) - environment_variables = overrides.pop( - "environment_variables", - [ - SecretVariable( - id="env-1", - name="API_KEY", - value="plain-token", - selector=["env", "API_KEY"], - description="API key", - ) - ], - ) - conversation_variables = overrides.pop( - "conversation_variables", - [ - StringVariable( - id="conv-1", - name="topic", - value="hello", - selector=["conversation", "topic"], - description="Topic", - ) - ], - ) - rag_pipeline_variables = overrides.pop( - "rag_pipeline_variables", - [ - RAGPipelineVariable.model_validate( - { - "variable": "query", - "type": "text-input", - "label": "Query", - "belong_to_node_id": "shared", - "max_length": 0, - "required": False, - "unit": "", - "default_value": "", - "options": [], - "placeholder": "", - "tooltips": "", - "allowed_file_types": ["custom"], - "allowed_file_extensions": [".pdf"], - "allowed_file_upload_methods": ["local_file"], - } - ) - ], - ) - workflow = Workflow( - id="workflow-1", - tenant_id="t1", - app_id="app", - type=WorkflowType.WORKFLOW, - version="1", - graph=json.dumps(graph), - features=json.dumps(features), - marked_name="Release 1", - marked_comment="Initial release", - created_by="user-1", - created_at=datetime(2024, 1, 1, 12, 0, 0), - updated_by=None, - updated_at=datetime(2024, 1, 1, 12, 1, 0), - environment_variables=[], - conversation_variables=[], - rag_pipeline_variables=[], - ) - if environment_variables and isinstance(environment_variables[0], dict): - workflow._environment_variables = json.dumps( - {str(index): value for index, value in enumerate(environment_variables)} - ) - else: - workflow.environment_variables = environment_variables - workflow.conversation_variables = conversation_variables - workflow.rag_pipeline_variables = rag_pipeline_variables - for key, value in overrides.items(): - setattr(workflow, key, value) - return workflow + with app.test_request_context(method="POST", data=json.dumps(body), content_type=content_type): + response = invoke(controller.DraftWorkflowApi) + assert response == {"result": "success", "hash": "new-hash", "updated_at": 1704067200} + context, app_id, command = workflows.sync.call_args.args + assert (context, app_id) == (CONTEXT, str(APP_ID)) + assert command.is_collaborative is True + assert command.unique_hash == "old-hash" + patch = body["environment_variable_patch"] + assert isinstance(patch, dict) + assert command.environment_upserts == patch["environment_variables"] + assert command.environment_deletions == ["env-2"] @pytest.mark.parametrize( - "advisory", ["clean", "warning", "checker-error", "formatter-error", "empty", "non-object", "invalid-json"] + ("content_type", "data", "status"), + [("application/xml", "x", 415), ("application/json", "[]", 400), ("text/plain", "invalid", 400)], ) -def test_publish_workflow_returns_success( - app: Flask, - monkeypatch: pytest.MonkeyPatch, - advisory: str, -) -> None: - current_user = SimpleNamespace(id="account-1") - app_model = SimpleNamespace(id="app-1", tenant_id="tenant-1") - graph = { - "nodes": [ - {"id": "start", "data": {"type": "start"}}, - {"id": "branch", "data": {"type": "if-else"}}, - {"id": "producer", "data": {"type": "code", "title": "Producer"}}, - { - "id": "consumer", - "data": {"type": "answer", "title": "Consumer", "answer": "{{#producer.text#}}"}, - }, - ], - "edges": [ - {"source": "start", "target": "branch"}, - {"source": "branch", "target": "producer", "sourceHandle": "true"}, - {"source": "branch", "target": "consumer", "sourceHandle": "false"}, - ], - } - workflow = SimpleNamespace( - id="published-workflow", - created_at=datetime(2026, 8, 17, 12, 0, 0), - graph={"clean": "{}", "empty": None, "non-object": "[]", "invalid-json": "{"}.get(advisory, json.dumps(graph)), - ) - if advisory == "checker-error": - monkeypatch.setattr(workflow_module, "validate_variable_references", Mock(side_effect=RuntimeError("checker"))) - elif advisory == "formatter-error": - monkeypatch.setattr( - workflow_module, "format_variable_reference_errors", Mock(side_effect=RuntimeError("format")) - ) - session = Mock() - session.get.return_value = app_model - monkeypatch.setattr( - workflow_module, - "WorkflowService", - Mock(return_value=SimpleNamespace(publish_workflow=Mock(return_value=workflow))), - ) - monkeypatch.setattr( - workflow_module, - "sessionmaker", - lambda _engine: SimpleNamespace(begin=lambda: nullcontext(session)), - ) - monkeypatch.setattr(workflow_module, "db", SimpleNamespace(engine=object())) - with app.test_request_context("/apps/app-1/workflows/publish", method="POST", json={}): - response = inspect.unwrap(workflow_module.PublishedWorkflowApi.post)( - workflow_module.PublishedWorkflowApi(), - workflow_module.PublishWorkflowPayload.model_validate({}), - current_user, - app_model, - ) - - assert response["result"] == "success" - assert app_model.workflow_id == workflow.id - assert isinstance(response["created_at"], int) - if advisory == "warning": - assert "Consumer" in response["warning"] - assert "Producer" in response["warning"] - assert "skipped branch" in response["warning"] - else: - assert "warning" not in response - - -@pytest.mark.parametrize("transaction_fails", [False, True], ids=["commit-succeeds", "commit-fails"]) -def test_delete_workflow_retires_candidates_only_after_transaction_exit( - app: Flask, - monkeypatch: pytest.MonkeyPatch, - transaction_fails: bool, +def test_sync_invalid_input_does_not_dispatch( + app: Flask, workflows: Mock, content_type: str, data: str, status: int ) -> None: - current_user = SimpleNamespace(id="account-1") - app_model = SimpleNamespace(id="app-1", tenant_id="tenant-1") - session = Mock() - events: list[str] = [] - error = RuntimeError("commit failed") - workflow_service = SimpleNamespace( - delete_workflow=Mock(side_effect=lambda **_kwargs: events.append("delete") or ["inline-agent"]) - ) - - @contextmanager - def transaction(): - events.append("transaction-enter") - yield session - events.append("transaction-exit") - if transaction_fails: - raise error - - monkeypatch.setattr(workflow_module, "WorkflowService", Mock(return_value=workflow_service)) - monkeypatch.setattr( - workflow_module, - "sessionmaker", - lambda _engine: SimpleNamespace(begin=transaction), - ) - monkeypatch.setattr(workflow_module, "db", SimpleNamespace(engine=object())) - retire_unowned = Mock(side_effect=lambda **_kwargs: events.append("retire")) - monkeypatch.setattr(workflow_module.WorkflowAgentRetirementService, "retire_unowned", retire_unowned) - - with app.test_request_context("/apps/app-1/workflows/workflow-1", method="DELETE"): - if transaction_fails: - with pytest.raises(RuntimeError) as exc_info: - inspect.unwrap(workflow_module.WorkflowByIdApi.delete)( - workflow_module.WorkflowByIdApi(), - current_user, - app_model, - "workflow-1", - ) - assert exc_info.value is error + with app.test_request_context(method="POST", data=data, content_type=content_type): + if status == 415: + with pytest.raises(HTTPException) as error: + invoke(controller.DraftWorkflowApi) + assert error.value.code == status else: - response = inspect.unwrap(workflow_module.WorkflowByIdApi.delete)( - workflow_module.WorkflowByIdApi(), - current_user, - app_model, - "workflow-1", - ) - assert response == (None, 204) - - assert events == ["transaction-enter", "delete", "transaction-exit"] + ([] if transaction_fails else ["retire"]) - if transaction_fails: - retire_unowned.assert_not_called() - else: - retire_unowned.assert_called_once_with( - tenant_id=app_model.tenant_id, - agent_ids=["inline-agent"], - account_id=current_user.id, - ) - - -def test_parse_file_no_config(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(workflow_module.FileUploadConfigManager, "convert", lambda *_args, **_kwargs: None) - workflow = _make_workflow(features_dict={}) - - assert workflow_module._parse_file(workflow, files=[{"id": "f"}]) == [] - - -def test_parse_file_with_config(monkeypatch: pytest.MonkeyPatch) -> None: - config = object() - file_list = [ - File( - tenant_id="t1", - file_type=FileType.IMAGE, - transfer_method=FileTransferMethod.REMOTE_URL, - remote_url="http://u", - ) - ] - build_mock = Mock(return_value=file_list) - monkeypatch.setattr(workflow_module.FileUploadConfigManager, "convert", lambda *_args, **_kwargs: config) - monkeypatch.setattr(workflow_module.file_factory, "build_from_mappings", build_mock) - - workflow = _make_workflow(features_dict={}) - result = workflow_module._parse_file(workflow, files=[{"id": "f"}]) - - assert result == file_list - build_mock.assert_called_once() - - -def test_sync_draft_workflow_invalid_content_type(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context("/apps/app/workflows/draft", method="POST", data="x", content_type="text/html"): - with pytest.raises(HTTPException) as exc: - handler(api, _account(), app_model=_app()) - - assert exc.value.code == 415 - - -def test_sync_draft_workflow_invalid_json(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - data="[]", - content_type="application/json", - ): - response, status = handler(api, _account(), app_model=_app()) - - assert status == 400 - assert response["message"] == "Invalid JSON data" - - -def test_sync_draft_workflow_success(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - workflow = _make_workflow(updated_at=None, created_at=datetime(2024, 1, 1)) - - monkeypatch.setattr( - workflow_module.variable_factory, "build_environment_variable_from_mapping", lambda *_args: "env" - ) - monkeypatch.setattr( - workflow_module.variable_factory, "build_conversation_variable_from_mapping", lambda *_args: "conv" - ) - - sync_draft_workflow = Mock(return_value=workflow) - service = SimpleNamespace(sync_draft_workflow=sync_draft_workflow) - monkeypatch.setattr(workflow_module, "WorkflowService", lambda: service) - - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - json={"graph": {}, "features": {}, "hash": "h"}, - ): - response = handler(api, _account(), app_model=_app()) - - assert response["result"] == "success" - assert sync_draft_workflow.call_args.kwargs["environment_variables"] == [] - assert sync_draft_workflow.call_args.kwargs["preserve_environment_variables"] is True - - -def test_sync_draft_workflow_passes_environment_patch(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - workflow = _make_workflow(updated_at=None, created_at=datetime(2024, 1, 1)) - patched_variable = StringVariable( - id="env-model", - name="shared_model", - value="model", - selector=["env", "shared_model"], - ) - build_environment_variable = Mock(return_value=patched_variable) - sync_draft_workflow = Mock(return_value=workflow) - monkeypatch.setattr( - workflow_module.variable_factory, - "build_environment_variable_from_mapping", - build_environment_variable, - ) - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(sync_draft_workflow=sync_draft_workflow), - ) - - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - json={ - "graph": {}, - "features": {}, - "hash": "current-hash", - "environment_variable_patch": { - "environment_variables": [ - { - "id": "env-model", - "name": "shared_model", - "value": "model", - "value_type": "string", - } - ], - "deleted_environment_variable_ids": ["env-old"], - }, - }, - ): - response = handler(api, _account(), app_model=_app()) - - assert response["result"] == "success" - assert sync_draft_workflow.call_args.kwargs["preserve_environment_variables"] is True - assert sync_draft_workflow.call_args.kwargs["environment_variable_upserts"] == [patched_variable] - assert sync_draft_workflow.call_args.kwargs["deleted_environment_variable_ids"] == ["env-old"] - build_environment_variable.assert_called_once() - - -def test_sync_draft_workflow_rejects_overlapping_environment_patch_ids() -> None: - with pytest.raises(ValidationError, match="cannot be upserted and deleted"): - workflow_module.SyncDraftWorkflowPayload.model_validate( - { - "graph": {}, - "features": {}, - "environment_variable_patch": { - "environment_variables": [{"id": "env-model"}], - "deleted_environment_variable_ids": ["env-model"], - }, - } - ) - - -def test_sync_draft_workflow_rejects_legacy_environment_variables() -> None: - with pytest.raises(ValidationError, match="Extra inputs are not permitted"): - workflow_module.SyncDraftWorkflowPayload.model_validate( - { - "graph": {}, - "features": {}, - "environment_variables": [], - } - ) - - -def test_sync_draft_workflow_hash_mismatch(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - - def _raise(*_args, **_kwargs): - raise workflow_module.WorkflowHashNotEqualError() - - service = SimpleNamespace(sync_draft_workflow=_raise) - monkeypatch.setattr(workflow_module, "WorkflowService", lambda: service) - - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - json={"graph": {}, "features": {}, "hash": "h"}, - ): - with pytest.raises(DraftWorkflowNotSync): - handler(api, _account(), app_model=_app()) - - -def test_sync_draft_workflow_variable_validation_error(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - def _raise(*_args, **_kwargs): - raise workflow_module.VariableError("description too long") - - monkeypatch.setattr(workflow_module.variable_factory, "build_conversation_variable_from_mapping", _raise) - monkeypatch.setattr( - workflow_module, "WorkflowService", lambda: SimpleNamespace(sync_draft_workflow=lambda **_kwargs: None) - ) - - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - json={"graph": {}, "features": {}, "hash": "h", "conversation_variables": [{"name": "topic"}]}, - ): - with pytest.raises(InvalidArgumentError) as exc: - handler(api, _account(), app_model=_app()) - - assert exc.value.description == "description too long" - - -def test_restore_published_workflow_to_draft_success(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - workflow = _make_workflow(updated_at=None, created_at=datetime(2024, 1, 1)) - - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(restore_published_workflow_to_draft=lambda **_kwargs: workflow), - ) - - api = workflow_module.DraftWorkflowRestoreApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/published-workflow/restore", - method="POST", - ): - response = handler( - api, - _account(), - app_model=_app(tenant_id="tenant-1"), - workflow_id="published-workflow", - ) - - assert response["result"] == "success" - assert response["hash"] == workflow.unique_hash - - -def test_restore_published_workflow_to_draft_not_found(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace( - restore_published_workflow_to_draft=lambda **_kwargs: (_ for _ in ()).throw( - workflow_module.WorkflowNotFoundError("Workflow not found") - ) - ), - ) - - api = workflow_module.DraftWorkflowRestoreApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/published-workflow/restore", - method="POST", - ): - with pytest.raises(NotFound): - handler( - api, - _account(), - app_model=_app(tenant_id="tenant-1"), - workflow_id="published-workflow", - ) - - -def test_restore_published_workflow_to_draft_returns_400_for_draft_source( - app: Flask, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace( - restore_published_workflow_to_draft=lambda **_kwargs: (_ for _ in ()).throw( - workflow_module.IsDraftWorkflowError( - "Cannot use draft workflow version. Workflow ID: draft-workflow. " - "Please use a published workflow version or leave workflow_id empty." - ) - ) - ), - ) - - api = workflow_module.DraftWorkflowRestoreApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft-workflow/restore", - method="POST", - ): - with pytest.raises(HTTPException) as exc: - handler( - api, - _account(), - app_model=_app(tenant_id="tenant-1"), - workflow_id="draft-workflow", - ) - - assert exc.value.code == 400 - assert exc.value.description == workflow_module.RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE + assert invoke(controller.DraftWorkflowApi) == ({"message": "Invalid JSON data"}, status) + workflows.sync.assert_not_called() -def test_restore_published_workflow_to_draft_returns_400_for_invalid_structure( - app: Flask, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace( - restore_published_workflow_to_draft=lambda **_kwargs: (_ for _ in ()).throw( - ValueError("invalid workflow graph") - ) - ), - ) - - api = workflow_module.DraftWorkflowRestoreApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/published-workflow/restore", - method="POST", - ): - with pytest.raises(HTTPException) as exc: - handler( - api, - _account(), - app_model=_app(tenant_id="tenant-1"), - workflow_id="published-workflow", - ) - - assert exc.value.code == 400 - assert exc.value.description == "invalid workflow graph" - - -def test_get_published_workflows_uses_the_request_session( - app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session -) -> None: - api = workflow_module.PublishedAllWorkflowApi() - handler = inspect.unwrap(api.get) - workflow = _make_workflow() - sqlite_session.add_all([_account(), workflow]) - sqlite_session.commit() - - def get_all_published_workflow(*, session: Session, **_kwargs): - assert session is sqlite_session - return [workflow], False +@pytest.mark.parametrize( + ("failure", "error_type"), + [(WorkflowHashNotEqualError(), DraftWorkflowNotSync), (VariableError("bad variable"), InvalidArgumentError)], +) +def test_sync_domain_errors(app: Flask, workflows: Mock, failure: Exception, error_type: type[Exception]) -> None: + workflows.sync.side_effect = failure + with app.test_request_context(method="POST", json={"graph": {}, "features": {}}), pytest.raises(error_type): + invoke(controller.DraftWorkflowApi) - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(get_all_published_workflow=get_all_published_workflow), - ) - with app.test_request_context( - "/apps/app/workflows", - method="GET", - query_string={"page": 1, "limit": 10, "user_id": "", "named_only": "false"}, - ): - query = workflow_module.WorkflowListQuery.model_validate( - {"page": "1", "limit": "10", "user_id": "", "named_only": "false"} +@pytest.mark.parametrize( + "patch", + [ + {"environment_variables": [{"id": "same"}, {"id": "same"}]}, + {"environment_variables": [{"name": "missing-id"}]}, + {"environment_variables": [{"id": "same"}], "deleted_environment_variable_ids": ["same"]}, + {"deleted_environment_variable_ids": [""]}, + ], +) +def test_sync_rejects_ambiguous_variable_patch(patch: dict[str, object]) -> None: + with pytest.raises(ValidationError): + controller.SyncDraftWorkflowPayload.model_validate( + {"graph": {}, "features": {}, "environment_variable_patch": patch} ) - response = handler(api, query, sqlite_session, _account(), app_model=_app()) - - assert response["items"][0]["id"] == "workflow-1" - assert response["page"] == 1 - assert response["limit"] == 10 - assert response["has_more"] is False - - -def test_draft_workflow_get_serializes_response_model(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session) -> None: - workflow = _make_workflow() - sqlite_session.add_all([_account(), workflow]) - sqlite_session.commit() - monkeypatch.setattr( - workflow_module, "WorkflowService", lambda: SimpleNamespace(get_draft_workflow=lambda **_kwargs: workflow) - ) - - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.get) - response = handler(api, sqlite_session, app_model=_app()) - - assert response["id"] == "workflow-1" - assert response["graph"] == {"nodes": [], "edges": []} - assert response["features"] == {"file_upload": {"enabled": False}} - assert response["hash"] == workflow.unique_hash - assert response["created_by"] == {"id": "user-1", "name": "Alice", "email": "alice@example.com"} - assert response["updated_by"] is None - assert response["created_at"] == int(datetime(2024, 1, 1, 12, 0, 0).timestamp()) - assert response["updated_at"] == int(datetime(2024, 1, 1, 12, 1, 0).timestamp()) - assert response["environment_variables"] == [ - { - "id": "env-1", - "name": "API_KEY", - "value": workflow_module.encrypter.full_mask_token(), - "value_type": "secret", - "description": "API key", - } - ] - assert response["conversation_variables"] == [ - { - "id": "conv-1", - "name": "topic", - "value": "hello", - "value_type": "string", - "description": "Topic", - } - ] - assert response["rag_pipeline_variables"] == [ - { - "label": "Query", - "variable": "query", - "type": "text-input", - "belong_to_node_id": "shared", - "max_length": 0, - "required": False, - "unit": "", - "default_value": "", - "options": [], - "placeholder": "", - "tooltips": "", - "allowed_file_types": ["custom"], - "allowed_file_extensions": [".pdf"], - "allowed_file_upload_methods": ["local_file"], - } - ] - - -def test_published_workflow_get_uses_session_aware_response_source( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session -) -> None: - sqlite_session.add(_account()) - sqlite_session.commit() - workflow = _make_workflow() - monkeypatch.setattr(workflow_module.db, "session", lambda: sqlite_session) - monkeypatch.setattr( - workflow_module, "WorkflowService", lambda: SimpleNamespace(get_published_workflow=lambda **_kwargs: workflow) - ) - - api = workflow_module.PublishedWorkflowApi() - handler = inspect.unwrap(api.get) - - response = handler(api, app_model=SimpleNamespace(id="app")) - - assert response["id"] == "workflow-1" - assert response["created_by"] == {"id": "user-1", "name": "Alice", "email": "alice@example.com"} - assert response["updated_by"] is None - assert response["tool_published"] is False - - -def test_pipeline_variable_response_accepts_legacy_file_field_names() -> None: - response = workflow_module.PipelineVariableResponse.model_validate( - { - "label": "Query", - "variable": "query", - "type": "single-file", - "belong_to_node_id": "shared", - "max_length": 0, - "required": False, - "unit": "", - "default_value": "", - "options": [], - "placeholder": "", - "tooltips": "", - "allowed_file_types": [], - "allow_file_extension": [".txt"], - "allow_file_upload_methods": ["remote_url"], - } - ).model_dump(mode="json") - - assert response["allowed_file_extensions"] == [".txt"] - assert response["allowed_file_upload_methods"] == ["remote_url"] - - -def test_pipeline_variable_response_accepts_explicit_null_optional_fields() -> None: - pipeline_variable = RAGPipelineVariable.model_validate( - { - "label": "Query", - "variable": "query", - "type": "text-input", - "belong_to_node_id": "shared", - "max_length": None, - "unit": None, - "default_value": None, - "options": None, - "placeholder": None, - "tooltips": None, - "allowed_file_types": None, - "allowed_file_extensions": None, - "allowed_file_upload_methods": None, - } - ).model_dump(mode="json") - - response = workflow_module.PipelineVariableResponse.model_validate(pipeline_variable).model_dump(mode="json") - - assert response["max_length"] is None - assert response["allowed_file_types"] is None - assert response["allowed_file_extensions"] is None - assert response["allowed_file_upload_methods"] is None - - -def test_workflow_response_masks_secret_environment_variables(sqlite_session: Session) -> None: - workflow = _make_workflow( - environment_variables=[ - SecretVariable(id="env-secret", name="API_KEY", value="plain-token", selector=["env", "API_KEY"]), - StringVariable(id="env-string", name="REGION", value="us-east-1", selector=["env", "REGION"]), - ] - ) - sqlite_session.add_all([_account(), workflow]) - sqlite_session.commit() - - response = workflow_module.WorkflowResponse.model_validate( - workflow_module.WorkflowResponseSource(workflow, session=sqlite_session), - from_attributes=True, - ).model_dump(mode="json") - - assert response["environment_variables"] == [ - { - "id": "env-secret", - "name": "API_KEY", - "value": workflow_module.encrypter.full_mask_token(), - "value_type": "secret", - "description": "", - }, - { - "id": "env-string", - "name": "REGION", - "value": "us-east-1", - "value_type": "string", - "description": "", - }, - ] - - -def test_workflow_response_preserves_llm_environment_variable_type(sqlite_session: Session) -> None: - workflow = _make_workflow( - environment_variables=[ - LLMEnvironmentVariable( - id="env-llm", - name="for_summarize", - value={"provider": "provider", "name": "model", "mode": "chat"}, - selector=["env", "for_summarize"], - ) - ] - ) - sqlite_session.add_all([_account(), workflow]) - sqlite_session.commit() - - response = workflow_module.WorkflowResponse.model_validate( - workflow_module.WorkflowResponseSource(workflow, session=sqlite_session), - from_attributes=True, - ).model_dump(mode="json") - - assert response["environment_variables"] == [ - { - "id": "env-llm", - "name": "for_summarize", - "value": {"provider": "provider", "name": "model", "mode": "chat"}, - "value_type": "llm", - "description": "", - } - ] - - -def test_workflow_response_rejects_invalid_environment_variable_dict(sqlite_session: Session) -> None: - workflow = _make_workflow(environment_variables=[{"value_type": "not-a-segment-type"}]) - sqlite_session.add_all([_account(), workflow]) - sqlite_session.commit() +def test_sync_rejects_removed_full_environment_payload() -> None: with pytest.raises(ValidationError): - workflow_module.WorkflowResponse.model_validate( - workflow_module.WorkflowResponseSource(workflow, session=sqlite_session), - from_attributes=True, - ) + controller.SyncDraftWorkflowPayload.model_validate({"graph": {}, "features": {}, "environment_variables": []}) -def test_draft_workflow_get_not_found(monkeypatch: pytest.MonkeyPatch, unbound_session: Session) -> None: - monkeypatch.setattr( - workflow_module, "WorkflowService", lambda: SimpleNamespace(get_draft_workflow=lambda **_k: None) - ) +@pytest.mark.parametrize("warning", [None, "branch may be skipped"]) +def test_publish_serializes_advisory(workflows: Mock, warning: str | None) -> None: + workflows.publish.return_value = (WorkflowPublication(datetime(2024, 1, 1, tzinfo=UTC), "{}"), warning) + response = invoke(controller.PublishedWorkflowApi, "post", controller.PublishWorkflowPayload()) + assert isinstance(response, dict) + assert response["created_at"] == 1704067200 + assert response.get("warning") == warning + assert ("warning" in response) is (warning is not None) + workflows.publish.assert_called_once_with(CONTEXT, str(APP_ID), marked_name="", marked_comment="") - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.get) - with pytest.raises(DraftWorkflowNotExist): - handler(api, unbound_session, app_model=_app()) +@pytest.mark.parametrize( + ("failure", "error_type"), + [ + (IsDraftWorkflowError(), InvalidRequestError), + (WorkflowNotFoundError("missing"), NotFoundError), + (ValueError("invalid graph"), InvalidRequestError), + ], +) +def test_restore_error_contract(workflows: Mock, failure: Exception, error_type: type[Exception]) -> None: + workflows.restore.side_effect = failure + with pytest.raises(error_type): + invoke(controller.DraftWorkflowRestoreApi, workflow_id="version") -def test_draft_workflow_get_projects_agent_node_job_to_graph( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session -) -> None: - workflow = _make_workflow( - graph_dict={ - "nodes": [ - { - "id": "agent-node", - "data": { - "type": "agent", - "version": "2", - "agent_node_kind": "dify_agent", - }, - } - ], - "edges": [], - } - ) - sqlite_session.add_all([_account(), workflow]) - sqlite_session.commit() - projected_graph = { - "nodes": [ - { - "id": "agent-node", - "data": { - "type": "agent", - "version": "2", - "agent_node_kind": "dify_agent", - "agent_task": "Summarize it.", - "agent_declared_outputs": [{"name": "summary", "type": "string"}], - }, - } - ], - "edges": [], +@pytest.mark.parametrize( + ("failure", "status", "code"), + [(WorkflowNotFoundError("missing"), 404, "not_found"), (ValueError("invalid graph"), 400, "bad_request")], +) +def test_restore_domain_error_http_response(workflows: Mock, failure: Exception, status: int, code: str) -> None: + from controllers.console import api as console_api + + workflows.restore.side_effect = failure + app = Flask(__name__) + app.config["RESTX_ERROR_404_HELP"] = False + api = ExternalApi(app) + api.error_handlers = console_api.error_handlers.copy() + + class Restore(Resource): + def post(self): + return invoke(controller.DraftWorkflowRestoreApi, workflow_id="version") + + api.add_resource(Restore, "/restore") + response = app.test_client().post("/restore") + assert response.status_code == status + assert response.json == {"code": code, "status": status, "message": str(failure)} + + +def test_restore_serializes_change(workflows: Mock) -> None: + workflows.restore.return_value = WorkflowChange("hash", datetime(2024, 1, 1, tzinfo=UTC)) + assert invoke(controller.DraftWorkflowRestoreApi, workflow_id="version") == { + "result": "success", + "hash": "hash", + "updated_at": 1704067200, } - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(get_draft_workflow=lambda **_k: workflow), - ) - - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - monkeypatch.setattr( - WorkflowAgentPublishService, - "project_draft_bindings_to_graph", - lambda **_k: projected_graph, - ) - - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.get) - - response = handler(api, sqlite_session, app_model=_app()) - - assert response["graph"] == projected_graph - -def test_advanced_chat_run_conversation_not_exists( - app: Flask, monkeypatch: pytest.MonkeyPatch, unbound_session: Session +@pytest.mark.parametrize( + ("resource", "payload", "kwargs"), + [ + (controller.DraftWorkflowTriggerRunApi, controller.DraftWorkflowTriggerRunPayload(node_id="node"), {}), + (controller.DraftWorkflowTriggerRunAllApi, controller.DraftWorkflowTriggerRunAllPayload(node_ids=["node"]), {}), + (controller.DraftWorkflowTriggerNodeApi, None, {"node_id": "node"}), + ], +) +def test_trigger_waiting_and_error_contract( + workflows: Mock, resource: type[Resource], payload: BaseModel | None, kwargs: dict[str, str] ) -> None: - monkeypatch.setattr( - workflow_module.AppGenerateService, - "generate", - lambda *_args, **_kwargs: (_ for _ in ()).throw( - workflow_module.services.errors.conversation.ConversationNotExistsError() - ), - ) - - api = workflow_module.AdvancedChatDraftWorkflowRunApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/advanced-chat/workflows/draft/run", - method="POST", - json={"inputs": {}}, - ): - payload = workflow_module.AdvancedChatWorkflowRunPayload.model_validate({"inputs": {}}) - with pytest.raises(NotFound): - handler(api, payload, unbound_session, _account(), app_model=_app()) + args = (payload,) if payload is not None else () + workflows.trigger.return_value = None + assert invoke(resource, "post", *args, **kwargs) == {"status": "waiting", "retry_in": 2000} + workflows.trigger.side_effect = WorkflowTriggerError("plugin unavailable") + assert invoke(resource, "post", *args, **kwargs) == ({"status": "error", "error": "plugin unavailable"}, 400) @pytest.mark.parametrize( - ("resource", "payload_model", "payload"), + ("resource", "payload"), [ - ( - workflow_module.DraftWorkflowTriggerRunApi, - workflow_module.DraftWorkflowTriggerRunPayload, - {"node_id": "node-1"}, - ), - ( - workflow_module.DraftWorkflowTriggerRunAllApi, - workflow_module.DraftWorkflowTriggerRunAllPayload, - {"node_ids": ["node-1"]}, - ), + (controller.DraftWorkflowRunApi, controller.DraftWorkflowRunPayload(inputs={})), + (controller.AdvancedChatDraftWorkflowRunApi, controller.AdvancedChatWorkflowRunPayload(inputs={})), ], ) -def test_trigger_run_loads_draft_with_request_session( - app: Flask, - monkeypatch: pytest.MonkeyPatch, - unbound_session: Session, - resource: type, - payload_model: type, - payload: dict[str, object], +def test_debug_run_passes_external_trace_and_stream( + app: Flask, workflows: Mock, monkeypatch: pytest.MonkeyPatch, resource: type[Resource], payload: BaseModel ) -> None: - get_draft_workflow = Mock(return_value=None) - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(get_draft_workflow=get_draft_workflow), - ) - session = unbound_session - app_model = _app(app_id="app-1") - handler = inspect.unwrap(resource.post) + workflows.generate.return_value = {"task_id": "task"} + monkeypatch.setattr(controller, "get_external_trace_id", lambda request: "external-trace") + with app.test_request_context(method="POST", json={}): + response = invoke(resource, "post", payload) + assert isinstance(response, Response) + assert response.get_json() == {"task_id": "task"} + assert workflows.generate.call_args.args[2]["external_trace_id"] == "external-trace" - with app.test_request_context("/", method="POST", json=payload): - with pytest.raises(ValueError, match="Workflow not found"): - handler( - resource(), - payload_model.model_validate(payload), - session, - _account(), - app_model, - ) - get_draft_workflow.assert_called_once_with(app_model, session=session) +def test_convert_delegates_permissions_result(workflows: Mock) -> None: + result = {"new_app_id": "new-app", "permission_keys": ["app.acl.edit"]} + workflows.convert.return_value = result + assert invoke(controller.ConvertToWorkflowApi, "post", controller.ConvertToWorkflowPayload(name="Copy")) == result + workflows.convert.assert_called_once_with(CONTEXT, str(APP_ID), {"name": "Copy"}) -def test_workflow_online_users_filters_inaccessible_workflow(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - app_id_1 = "11111111-1111-1111-1111-111111111111" - app_id_2 = "22222222-2222-2222-2222-222222222222" - signed_avatar_url = "https://files.example.com/signed/avatar-1" - sign_avatar = Mock(return_value=signed_avatar_url) - get_tenant_app_maintainers = Mock(return_value={app_id_1: "owner-1", app_id_2: "owner-2"}) - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(get_tenant_app_maintainers=get_tenant_app_maintainers), +def test_update_empty_payload_does_not_dispatch(workflows: Mock) -> None: + assert invoke(controller.WorkflowByIdApi, "patch", controller.WorkflowUpdatePayload(), workflow_id="version") == ( + {"message": "No valid fields to update"}, + 400, ) - access_filter = SimpleNamespace(is_app_accessible=lambda app_id, _maintainer, _account_id: app_id == app_id_1) - resolve_access = Mock(return_value=access_filter) - monkeypatch.setattr(workflow_module, "resolve_app_access_filter", resolve_access) - apply_config_overrides(monkeypatch, RBAC_ENABLED=True) - monkeypatch.setattr(workflow_module.file_helpers, "get_signed_file_url", sign_avatar) - short_session = Mock() - monkeypatch.setattr(workflow_module.session_factory, "create_session", lambda: nullcontext(short_session)) - - redis_pipeline = Mock() - redis_pipeline.execute.return_value = [ - { - b"sid-1": json.dumps( - { - "user_id": "u-1", - "username": "Alice", - "avatar": "avatar-file-id", - "sid": "sid-1", - } - ), - b"sid-malformed": json.dumps({"avatar": "avatar-file-id", "sid": "sid-malformed"}), - b"sid-invalid-avatar": json.dumps( - { - "user_id": "u-2", - "username": "Bob", - "avatar": {"file_id": "avatar-file-id"}, - } - ), - b"sid-invalid-user-id": json.dumps( - { - "user_id": 42, - "username": "Carol", - "avatar": "avatar-file-id", - } - ), - b"sid-invalid-username": json.dumps( - { - "user_id": "u-4", - "username": ["Dave"], - "avatar": "avatar-file-id", - } - ), - } - ] - redis_pipeline_factory = Mock(return_value=redis_pipeline) - monkeypatch.setattr(workflow_module.redis_client, "pipeline", redis_pipeline_factory) + workflows.update.assert_not_called() - api = workflow_module.WorkflowOnlineUsersApi() - handler = inspect.unwrap(api.post) - with app.test_request_context( - "/apps/workflows/online-users", - method="POST", - json={"app_ids": [app_id_1, app_id_2]}, - ): - args = workflow_module.WorkflowOnlineUsersPayload.model_validate({"app_ids": [app_id_1, app_id_2]}) - response = handler(api, args, "tenant-1", SimpleNamespace(id="account-1")) - - assert response == { - "data": [ - { - "app_id": app_id_1, - "users": [ - { - "user_id": "u-1", - "username": "Alice", - "avatar": signed_avatar_url, - }, - { - "user_id": "u-2", - "username": "Bob", - "avatar": None, - }, - ], - } - ] - } - redis_pipeline_factory.assert_called_once_with(transaction=False) - redis_pipeline.hgetall.assert_called_once_with(f"{workflow_module.WORKFLOW_ONLINE_USERS_PREFIX}{app_id_1}") - redis_pipeline.execute.assert_called_once_with() - sign_avatar.assert_called_once_with("avatar-file-id") - get_tenant_app_maintainers.assert_called_once() - resolve_access.assert_called_once() - assert get_tenant_app_maintainers.call_args.args == ([app_id_1, app_id_2], "tenant-1") - assert resolve_access.call_args.args == ("tenant-1", "account-1") - assert get_tenant_app_maintainers.call_args.kwargs["session"] is resolve_access.call_args.kwargs["session"] +def test_delete_has_no_response_body(workflows: Mock) -> None: + assert invoke(controller.WorkflowByIdApi, "delete", workflow_id="version") == (None, 204) + workflows.delete.assert_called_once_with(CONTEXT, WorkflowOwner(str(APP_ID)), "version") -def test_workflow_online_users_batches_redis_reads(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - app_ids = [f"wf-{index}" for index in range(workflow_module.WORKFLOW_ONLINE_USERS_REDIS_BATCH_SIZE + 1)] - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(get_tenant_app_maintainers=lambda app_ids, tenant_id, session: dict.fromkeys(app_ids)), - ) - apply_config_overrides(monkeypatch, RBAC_ENABLED=False) - monkeypatch.setattr(workflow_module.session_factory, "create_session", lambda: nullcontext(Mock())) - - first_pipeline = Mock() - first_pipeline.execute.return_value = [{} for _ in range(workflow_module.WORKFLOW_ONLINE_USERS_REDIS_BATCH_SIZE)] - second_pipeline = Mock() - second_pipeline.execute.return_value = [{}] - redis_pipeline_factory = Mock(side_effect=[first_pipeline, second_pipeline]) - monkeypatch.setattr(workflow_module.redis_client, "pipeline", redis_pipeline_factory) - - api = workflow_module.WorkflowOnlineUsersApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/workflows/online-users", - method="POST", - json={"app_ids": app_ids}, - ): - args = workflow_module.WorkflowOnlineUsersPayload.model_validate({"app_ids": app_ids}) - response = handler(api, args, "tenant-1", SimpleNamespace(id="account-1")) - - assert len(response["data"]) == len(app_ids) - assert redis_pipeline_factory.call_count == 2 - assert first_pipeline.hgetall.call_count == workflow_module.WORKFLOW_ONLINE_USERS_REDIS_BATCH_SIZE - assert second_pipeline.hgetall.call_count == 1 - - -def test_workflow_online_users_rejects_excessive_workflow_ids(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - get_tenant_app_maintainers = Mock(return_value={}) - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(get_tenant_app_maintainers=get_tenant_app_maintainers), - ) - - excessive_ids = [f"wf-{index}" for index in range(workflow_module.MAX_WORKFLOW_ONLINE_USERS_REQUEST_IDS + 1)] - - api = workflow_module.WorkflowOnlineUsersApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/workflows/online-users", - method="POST", - json={"app_ids": excessive_ids}, - ): - args = workflow_module.WorkflowOnlineUsersPayload.model_validate({"app_ids": excessive_ids}) - with pytest.raises(HTTPException) as exc: - handler(api, args, "tenant-1", SimpleNamespace(id="account-1")) - - assert exc.value.code == 400 - assert exc.value.description is not None - assert "Maximum" in exc.value.description - get_tenant_app_maintainers.assert_not_called() +def test_online_users_normalization_and_response(workflows: Mock) -> None: + workflows.online_users.return_value = [ + {"app_id": "a", "users": [{"user_id": "u", "username": "Name", "avatar": None}]} + ] + payload = controller.WorkflowOnlineUsersPayload(app_ids=[" a ", "", "a"]) + api = controller.WorkflowOnlineUsersApi() + result = inspect.unwrap(api.post)(api, payload, CONTEXT) + assert result == {"data": workflows.online_users.return_value} + workflows.online_users.assert_called_once_with(CONTEXT, ["a"]) diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow_admission.py b/api/tests/unit_tests/controllers/console/app/test_workflow_admission.py new file mode 100644 index 00000000000000..886e433a8d9f4d --- /dev/null +++ b/api/tests/unit_tests/controllers/console/app/test_workflow_admission.py @@ -0,0 +1,150 @@ +from controllers.common.rbac import RBACCheck + +"""Keep the existing workflow role, RBAC and app-mode admission policy.""" + +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from inspect import unwrap +from unittest.mock import Mock +from uuid import UUID + +import pytest +from flask import Flask +from werkzeug.exceptions import Forbidden + +from controllers.common.rbac import PlainApp +from controllers.console import flask_admission +from controllers.console.app import workflow, workflow_admission +from controllers.console.app.error import AppNotFoundError, DraftWorkflowNotSync +from controllers.console.wraps import RBACPermission +from libs.login import AccountWithTenant +from machinery.context import RequestContext +from models.account import TenantAccountRole +from services.agent.workflow_contracts import WorkflowAgentBindingStore +from services.app.console_service import ConsoleAppService +from services.entities.app_entities import AppReference +from services.errors.app import WorkflowHashNotEqualError +from services.workflow.console_service import ConsoleWorkflowService +from tests.unit_tests.config_override import apply_config_overrides +from tests.unit_tests.model_factories import make_account + +APP_ID = UUID("11111111-1111-1111-1111-111111111111") + + +@dataclass +class Apps: + console: ConsoleAppService + + +@dataclass +class Services: + apps: Apps + console_workflows: ConsoleWorkflowService[WorkflowAgentBindingStore] + + +def admission(method: Callable[..., object]) -> Callable[..., object]: + return unwrap(method, stop=lambda view: "inject_request_context" in view.__code__.co_qualname) + + +@pytest.mark.parametrize( + ("method", "permission"), + [ + (workflow.DraftWorkflowApi.get, RBACPermission.APP_VIEW_LAYOUT), + (workflow.DraftWorkflowApi.post, RBACPermission.APP_VIEW_LAYOUT), + (workflow.PublishedWorkflowApi.post, RBACPermission.APP_RELEASE_AND_VERSION), + (workflow.DraftWorkflowRestoreApi.post, RBACPermission.APP_RELEASE_AND_VERSION), + (workflow.WorkflowByIdApi.patch, RBACPermission.APP_EDIT), + (workflow.WorkflowByIdApi.delete, RBACPermission.APP_EDIT), + (workflow.DraftWorkflowRunApi.post, RBACPermission.APP_TEST_AND_RUN), + (workflow.DraftWorkflowTriggerRunApi.post, RBACPermission.APP_TEST_AND_RUN), + (workflow.WorkflowDraftHumanInputFormPreviewApi.post, RBACPermission.APP_VIEW_LAYOUT), + (workflow.WorkflowDraftHumanInputFormRunApi.post, RBACPermission.APP_TEST_AND_RUN), + (workflow.DraftWorkflowNodeLastRunApi.get, RBACPermission.APP_VIEW_LAYOUT), + ], +) +def test_rbac_checks_keep_admitted_tenant_and_app( + app: Flask, monkeypatch: pytest.MonkeyPatch, method: Callable[..., object], permission: RBACPermission +) -> None: + account = make_account(role=TenantAccountRole.NORMAL) + apply_config_overrides(monkeypatch, RBAC_ENABLED=True) + monkeypatch.setattr(flask_admission, "current_account_with_tenant", lambda: AccountWithTenant(account, "tenant")) + calls = [] + + def deny(*, tenant_id: str, account_id: str, checks: Sequence[RBACCheck], path_args: dict[str, object]) -> None: + calls.append(True) + assert (tenant_id, account_id) == ("tenant", account.id) + assert checks[0].scene == permission + assert isinstance(checks[0].locator, PlainApp) + assert path_args["app_id"] == APP_ID + raise Forbidden() + + monkeypatch.setattr(flask_admission, "enforce_rbac_checks", deny) + with app.test_request_context(), pytest.raises(Forbidden): + admission(method)(None, app_id=APP_ID) + assert calls == [True] + + +@pytest.mark.parametrize("role", [TenantAccountRole.NORMAL, TenantAccountRole.DATASET_OPERATOR]) +@pytest.mark.parametrize( + "method", + [ + workflow.DraftWorkflowApi.get, + workflow.DraftWorkflowRunApi.post, + workflow.PublishedWorkflowApi.post, + workflow.WorkflowByIdApi.delete, + ], +) +def test_readonly_role_is_rejected_before_use_case( + app: Flask, monkeypatch: pytest.MonkeyPatch, role: TenantAccountRole, method: Callable[..., object] +) -> None: + apply_config_overrides(monkeypatch, RBAC_ENABLED=False) + monkeypatch.setattr( + flask_admission, "current_account_with_tenant", lambda: AccountWithTenant(make_account(role=role), "tenant") + ) + with app.test_request_context(), pytest.raises(Forbidden): + admission(method)(None, app_id=APP_ID) + + +@pytest.mark.parametrize("mode", ["workflow", "advanced-chat", "chat"]) +def test_context_is_stable_and_app_mode_is_checked(app: Flask, monkeypatch: pytest.MonkeyPatch, mode: str) -> None: + apply_config_overrides(monkeypatch, RBAC_ENABLED=False) + account = make_account(role=TenantAccountRole.EDITOR) + monkeypatch.setattr(flask_admission, "current_account_with_tenant", lambda: AccountWithTenant(account, "tenant")) + monkeypatch.setattr(flask_admission, "get_request_id", lambda: "request") + monkeypatch.setattr(flask_admission, "get_trace_id", lambda: None) + monkeypatch.setattr(flask_admission, "enforce_rbac_checks", lambda **_kwargs: None) + apps, use_cases = Mock(spec=ConsoleAppService), Mock(spec=ConsoleWorkflowService) + apps.get_reference.return_value = AppReference(str(APP_ID), "App", mode, None) + use_cases.published.return_value = None + services = Services(Apps(apps), use_cases) + monkeypatch.setattr(workflow_admission, "application_services", lambda: services) + monkeypatch.setattr(workflow, "application_services", lambda: services) + with app.test_request_context(headers={"X-Trace-Id": "trace"}): + if mode == "chat": + with pytest.raises(AppNotFoundError): + admission(workflow.PublishedWorkflowApi.get)(None, app_id=APP_ID) + use_cases.published.assert_not_called() + else: + assert admission(workflow.PublishedWorkflowApi.get)(None, app_id=APP_ID) is None + context, app_id = use_cases.published.call_args.args + assert context == RequestContext("request", "trace", account.id, "tenant") + assert app_id == str(APP_ID) + + +def test_revision_conflict_is_translated_to_existing_409(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: + apply_config_overrides(monkeypatch, RBAC_ENABLED=False) + monkeypatch.setattr( + flask_admission, + "current_account_with_tenant", + lambda: AccountWithTenant(make_account(role=TenantAccountRole.EDITOR), "tenant"), + ) + monkeypatch.setattr(flask_admission, "enforce_rbac_checks", lambda **_kwargs: None) + apps, use_cases = Mock(spec=ConsoleAppService), Mock(spec=ConsoleWorkflowService) + apps.get_reference.return_value = AppReference(str(APP_ID), "App", "workflow", None) + use_cases.publish.side_effect = WorkflowHashNotEqualError() + services = Services(Apps(apps), use_cases) + monkeypatch.setattr(workflow_admission, "application_services", lambda: services) + monkeypatch.setattr(workflow, "application_services", lambda: services) + with app.test_request_context(json={}), pytest.raises(DraftWorkflowNotSync) as error: + admission(workflow.PublishedWorkflowApi.post)(None, app_id=APP_ID) + assert error.value.code == 409 diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow_convert_api.py b/api/tests/unit_tests/controllers/console/app/test_workflow_convert_api.py deleted file mode 100644 index f9a9bc80d0669e..00000000000000 --- a/api/tests/unit_tests/controllers/console/app/test_workflow_convert_api.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Unit tests for convert-to-workflow endpoint.""" - -from __future__ import annotations - -from inspect import unwrap - -import pytest -from flask import Flask - -from controllers.console.app import workflow as workflow_module -from controllers.console.app.workflow import ConvertToWorkflowApi -from models import Account, App -from tests.unit_tests.model_factories import make_app - - -def _app(app_id: str) -> App: - return make_app(app_id=app_id, name=f"App {app_id}") - - -class TestConvertToWorkflowApi: - @pytest.fixture - def api(self): - return workflow_module.ConvertToWorkflowApi() - - def test_convert_to_workflow_attaches_permission_keys_when_rbac_enabled( - self, api: ConvertToWorkflowApi, app: Flask, monkeypatch: pytest.MonkeyPatch - ) -> None: - method = unwrap(api.post) - new_app = _app("new-app-1") - - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: type("WorkflowServiceStub", (), {"convert_to_workflow": lambda self, **_kwargs: new_app})(), - ) - monkeypatch.setattr( - workflow_module, - "get_app_permission_keys", - lambda tenant_id, account_id, app_id: ["app.acl.view_layout", "app.acl.edit"], - ) - - with app.test_request_context( - "/console/api/apps/app-1/convert-to-workflow", - method="POST", - json={}, - ): - current_user = Account(name="Current user", email="user@example.com") - current_user.id = "u1" - response = method( - api, - workflow_module.ConvertToWorkflowPayload.model_validate({}), - current_tenant_id="tenant-1", - current_user=current_user, - app_model=_app("app-1"), - ) - - assert response["new_app_id"] == "new-app-1" - assert response["permission_keys"] == ["app.acl.view_layout", "app.acl.edit"] diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py b/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py index f223228048dc40..76137cb7e90837 100644 --- a/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py +++ b/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py @@ -1,249 +1,80 @@ -from __future__ import annotations +"""Human input transport delegates parsed payloads to application services.""" -from dataclasses import dataclass -from unittest.mock import ANY, MagicMock +from unittest.mock import Mock import pytest from flask import Flask -from werkzeug.exceptions import UnprocessableEntity - -from controllers.console import wraps as console_wraps -from controllers.console.app import workflow as workflow_module -from controllers.console.app import wraps as app_wraps -from enums import DeploymentEdition -from libs import login as login_lib -from models import App, Tenant -from models.account import Account, AccountStatus, TenantAccountRole -from models.model import AppMode, IconType -from tests.unit_tests.config_override import apply_config_overrides - - -def _make_account() -> Account: - account = Account(name="tester", email="tester@example.com") - account.status = AccountStatus.ACTIVE - account.role = TenantAccountRole.OWNER - account.id = "account-123" # type: ignore[assignment] - tenant = Tenant(name="Test tenant") - tenant.id = "tenant-123" - account._current_tenant = tenant - account._get_current_object = lambda: account # type: ignore[attr-defined] - return account - - -def _make_app(mode: AppMode) -> App: - return App( - id="app-123", - tenant_id="tenant-123", - name="Human input app", - description="", - mode=mode, - icon_type=IconType.EMOJI, - icon="robot", - icon_background="#FFFFFF", - enable_site=True, - enable_api=True, - max_active_requests=None, - ) - - -def _patch_console_guards(monkeypatch: pytest.MonkeyPatch, account: Account, app_model: App) -> None: - # Skip setup and auth guardrails - apply_config_overrides( - monkeypatch, - DEPLOYMENT_EDITION=DeploymentEdition.CLOUD, - LOGIN_DISABLED=True, - INIT_PASSWORD="", - ) - monkeypatch.setattr(login_lib, "current_user", account) - monkeypatch.setattr(login_lib, "current_account_with_tenant", lambda: (account, account.current_tenant_id)) - monkeypatch.setattr(login_lib, "check_csrf_token", lambda *_, **__: None) - monkeypatch.setattr(console_wraps, "current_account_with_tenant", lambda: (account, account.current_tenant_id)) - monkeypatch.setattr(app_wraps, "current_account_with_tenant", lambda: (account, account.current_tenant_id)) - - # Avoid hitting the database when resolving the app model - monkeypatch.setattr(app_wraps, "_load_app_model_from_scoped_session", lambda _app_id: app_model) +from flask_restx import Resource +from pydantic import ValidationError + +from controllers.console.app import workflow as controller +from extensions.ext_application_services import ApplicationServices +from services.human_input.debug_service import HumanInputDebugService +from tests.unit_tests.controllers.console.app.test_workflow import ( + APP_ID, + CONTEXT, + invoke, +) -@dataclass -class PreviewCase: - resource_cls: type - path: str - mode: AppMode +@pytest.fixture(name="workflows") +def debug_use_cases(monkeypatch: pytest.MonkeyPatch) -> Mock: + service = Mock(spec=HumanInputDebugService) + dependencies = Mock(spec=ApplicationServices) + dependencies.human_input_debug = service + monkeypatch.setattr(controller, "application_services", lambda: dependencies) + return service @pytest.mark.parametrize( - "case", - [ - PreviewCase( - resource_cls=workflow_module.AdvancedChatDraftHumanInputFormPreviewApi, - path="/console/api/apps/app-123/advanced-chat/workflows/draft/human-input/nodes/node-42/form/preview", - mode=AppMode.ADVANCED_CHAT, - ), - PreviewCase( - resource_cls=workflow_module.WorkflowDraftHumanInputFormPreviewApi, - path="/console/api/apps/app-123/workflows/draft/human-input/nodes/node-42/form/preview", - mode=AppMode.WORKFLOW, - ), - ], + "resource", [controller.AdvancedChatDraftHumanInputFormPreviewApi, controller.WorkflowDraftHumanInputFormPreviewApi] ) -def test_human_input_preview_delegates_to_service( - app: Flask, monkeypatch: pytest.MonkeyPatch, case: PreviewCase -) -> None: - account = _make_account() - app_model = _make_app(case.mode) - _patch_console_guards(monkeypatch, account, app_model) - - preview_payload = { - "form_id": "node-42", - "form_content": "
example
", - "inputs": [{"name": "topic"}], - "actions": [{"id": "continue"}], - } - service_instance = MagicMock() - service_instance.get_human_input_form_preview.return_value = preview_payload - monkeypatch.setattr(workflow_module, "WorkflowService", MagicMock(return_value=service_instance)) - - with app.test_request_context(case.path, method="POST", json={"inputs": {"topic": "tech"}}): - response = case.resource_cls().post(app_id=app_model.id, node_id="node-42") - - assert response == preview_payload - service_instance.get_human_input_form_preview.assert_called_once_with( - app_model=app_model, - account=account, - node_id="node-42", - inputs={"topic": "tech"}, - session=ANY, - ) - - -@dataclass -class SubmitCase: - resource_cls: type - path: str - mode: AppMode +def test_preview_forwards_inputs(app: Flask, workflows: Mock, resource: type[Resource]) -> None: + payload = {"form_id": "human", "form_content": "Hello"} + workflows.preview_form.return_value = payload + args = controller.HumanInputFormPreviewPayload(inputs={"topic": "tech"}) + with app.test_request_context(method="POST"): + assert invoke(resource, "post", args, node_id="human") == payload + workflows.preview_form.assert_called_once_with(CONTEXT, str(APP_ID), "human", {"topic": "tech"}) @pytest.mark.parametrize( - "case", - [ - SubmitCase( - resource_cls=workflow_module.AdvancedChatDraftHumanInputFormRunApi, - path="/console/api/apps/app-123/advanced-chat/workflows/draft/human-input/nodes/node-99/form/run", - mode=AppMode.ADVANCED_CHAT, - ), - SubmitCase( - resource_cls=workflow_module.WorkflowDraftHumanInputFormRunApi, - path="/console/api/apps/app-123/workflows/draft/human-input/nodes/node-99/form/run", - mode=AppMode.WORKFLOW, - ), - ], + "resource", [controller.AdvancedChatDraftHumanInputFormRunApi, controller.WorkflowDraftHumanInputFormRunApi] ) -def test_human_input_submit_forwards_payload(app: Flask, monkeypatch: pytest.MonkeyPatch, case: SubmitCase) -> None: - account = _make_account() - app_model = _make_app(case.mode) - _patch_console_guards(monkeypatch, account, app_model) - - result_payload = {"node_id": "node-99", "outputs": {"__rendered_content": "

done

"}, "action": "approve"} - service_instance = MagicMock() - service_instance.submit_human_input_form_preview.return_value = result_payload - monkeypatch.setattr(workflow_module, "WorkflowService", MagicMock(return_value=service_instance)) - - with app.test_request_context( - case.path, - method="POST", - json={"form_inputs": {"answer": "42"}, "inputs": {"#node-1.result#": "LLM output"}, "action": "approve"}, - ): - response = case.resource_cls().post(app_id=app_model.id, node_id="node-99") - - assert response == result_payload - service_instance.submit_human_input_form_preview.assert_called_once_with( - app_model=app_model, - account=account, - node_id="node-99", +def test_submission_forwards_form_and_upstream_inputs(app: Flask, workflows: Mock, resource: type[Resource]) -> None: + outputs = {"answer": "42", "__action_id": "approve"} + workflows.submit_form.return_value = outputs + args = controller.HumanInputFormSubmitPayload( + form_inputs={"answer": "42"}, inputs={"#upstream.output#": "text"}, action="approve" + ) + with app.test_request_context(method="POST"): + assert invoke(resource, "post", args, node_id="human") == outputs + workflows.submit_form.assert_called_once_with( + CONTEXT, + str(APP_ID), + "human", form_inputs={"answer": "42"}, - inputs={"#node-1.result#": "LLM output"}, + inputs={"#upstream.output#": "text"}, action="approve", - session=ANY, ) -@dataclass -class DeliveryTestCase: - resource_cls: type - path: str - mode: AppMode - - -@pytest.mark.parametrize( - "case", - [ - DeliveryTestCase( - resource_cls=workflow_module.WorkflowDraftHumanInputDeliveryTestApi, - path="/console/api/apps/app-123/workflows/draft/human-input/nodes/node-7/delivery-test", - mode=AppMode.ADVANCED_CHAT, - ), - DeliveryTestCase( - resource_cls=workflow_module.WorkflowDraftHumanInputDeliveryTestApi, - path="/console/api/apps/app-123/workflows/draft/human-input/nodes/node-7/delivery-test", - mode=AppMode.WORKFLOW, - ), - ], -) -def test_human_input_delivery_test_calls_service( - app: Flask, monkeypatch: pytest.MonkeyPatch, case: DeliveryTestCase -) -> None: - account = _make_account() - app_model = _make_app(case.mode) - _patch_console_guards(monkeypatch, account, app_model) - - service_instance = MagicMock() - monkeypatch.setattr(workflow_module, "WorkflowService", MagicMock(return_value=service_instance)) - - with app.test_request_context( - case.path, - method="POST", - json={"delivery_method_id": "delivery-123"}, - ): - response = case.resource_cls().post(app_id=app_model.id, node_id="node-7") - - assert response == {} - service_instance.test_human_input_delivery.assert_called_once_with( - app_model=app_model, - account=account, - node_id="node-7", - delivery_method_id="delivery-123", - inputs={}, - session=ANY, +def test_delivery_forwards_default_inputs(app: Flask, workflows: Mock) -> None: + args = controller.HumanInputDeliveryTestPayload(delivery_method_id="email") + with app.test_request_context(method="POST"): + assert invoke(controller.WorkflowDraftHumanInputDeliveryTestApi, "post", args, node_id="human") == {} + workflows.test_delivery.assert_called_once_with( + CONTEXT, str(APP_ID), "human", inputs={}, delivery_method_id="email" ) -def test_human_input_delivery_test_maps_validation_error(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - account = _make_account() - app_model = _make_app(AppMode.ADVANCED_CHAT) - _patch_console_guards(monkeypatch, account, app_model) - - service_instance = MagicMock() - service_instance.test_human_input_delivery.side_effect = ValueError("bad delivery method") - monkeypatch.setattr(workflow_module, "WorkflowService", MagicMock(return_value=service_instance)) - - with app.test_request_context( - "/console/api/apps/app-123/workflows/draft/human-input/nodes/node-1/delivery-test", - method="POST", - json={"delivery_method_id": "bad"}, - ): - with pytest.raises(ValueError): - workflow_module.WorkflowDraftHumanInputDeliveryTestApi().post(app_id=app_model.id, node_id="node-1") - +def test_delivery_preserves_domain_error(app: Flask, workflows: Mock) -> None: + workflows.test_delivery.side_effect = ValueError("bad delivery method") + args = controller.HumanInputDeliveryTestPayload(delivery_method_id="bad") + with app.test_request_context(method="POST"), pytest.raises(ValueError, match="bad delivery method"): + invoke(controller.WorkflowDraftHumanInputDeliveryTestApi, "post", args, node_id="human") -def test_human_input_preview_rejects_non_mapping(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - account = _make_account() - app_model = _make_app(AppMode.ADVANCED_CHAT) - _patch_console_guards(monkeypatch, account, app_model) - with app.test_request_context( - "/console/api/apps/app-123/advanced-chat/workflows/draft/human-input/nodes/node-1/form/preview", - method="POST", - json={"inputs": ["not-a-dict"]}, - ): - with pytest.raises(UnprocessableEntity): - workflow_module.AdvancedChatDraftHumanInputFormPreviewApi().post(app_id=app_model.id, node_id="node-1") +def test_preview_rejects_non_mapping_inputs() -> None: + with pytest.raises(ValidationError): + controller.HumanInputFormPreviewPayload.model_validate({"inputs": ["not-a-dict"]}) diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow_run_api.py b/api/tests/unit_tests/controllers/console/app/test_workflow_run_api.py index 7457dae7d80c18..4db412d94074d1 100644 --- a/api/tests/unit_tests/controllers/console/app/test_workflow_run_api.py +++ b/api/tests/unit_tests/controllers/console/app/test_workflow_run_api.py @@ -332,13 +332,13 @@ def test_workflow_run_detail_maps_missing_run_to_not_found( def test_workflow_run_node_executions_return_frontend_trace_contract( - app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session + app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, *, tool_providers ) -> None: _account(sqlite_session) execution = _workflow_run_node_execution(sqlite_session) workflow_runs = Mock() workflow_runs.get_workflow_run_node_executions.return_value = [ - node_execution_response_source(execution, session=sqlite_session) + node_execution_response_source(execution, session=sqlite_session, tool_providers=tool_providers) ] _mock_application_services(monkeypatch, workflow_runs) monkeypatch.setattr(db, "session", lambda: sqlite_session) diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py index daae29fd75ce2e..e58ef2b898f03d 100644 --- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py +++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py @@ -1,10 +1,11 @@ from inspect import unwrap -from unittest.mock import ANY, MagicMock, patch +from unittest.mock import ANY, patch import pytest from flask import Flask from controllers.console import console_ns +from controllers.console.datasets.rag_pipeline import datasource_content_preview as preview_module from controllers.console.datasets.rag_pipeline.datasource_content_preview import ( DataSourceContentPreviewApi, Parser, @@ -39,21 +40,19 @@ def test_post_success(self, app: Flask): preview_result = {"content": "preview data"} - service_instance = MagicMock() - service_instance.run_datasource_node_preview.return_value = preview_result - with ( app.test_request_context("/", json=payload), patch.object(type(console_ns), "payload", payload), - patch( - "controllers.console.datasets.rag_pipeline.datasource_content_preview.RagPipelineService", - return_value=service_instance, - ), + patch.object( + preview_module.application_services().knowledge.pipeline_execution, + "run_datasource_node_preview", + return_value=preview_result, + ) as preview, ): req_data = Parser.model_validate(payload) response, status = method(api, req_data, account, pipeline, node_id) - service_instance.run_datasource_node_preview.assert_called_once_with( + preview.assert_called_once_with( pipeline=pipeline, node_id=node_id, user_inputs=payload["inputs"], @@ -98,20 +97,18 @@ def test_post_without_credential_id(self, app: Flask): pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline") account = make_account() - service_instance = MagicMock() - service_instance.run_datasource_node_preview.return_value = {"ok": True} - with ( app.test_request_context("/", json=payload), patch.object(type(console_ns), "payload", payload), - patch( - "controllers.console.datasets.rag_pipeline.datasource_content_preview.RagPipelineService", - return_value=service_instance, - ), + patch.object( + preview_module.application_services().knowledge.pipeline_execution, + "run_datasource_node_preview", + return_value={"ok": True}, + ) as preview, ): req_data = Parser.model_validate(payload) response, status = method(api, req_data, account, pipeline, "node-1") - service_instance.run_datasource_node_preview.assert_called_once() + preview.assert_called_once() assert status == 200 assert response == {"ok": True} diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow.py index 8794c7d7b6af4a..27590605c34b4e 100644 --- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow.py +++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow.py @@ -13,8 +13,8 @@ from flask import Flask from sqlalchemy import Engine from sqlalchemy.orm import Session -from werkzeug.exceptions import Forbidden, NotFound +from controllers.common.errors import AccessDeniedError, NotFoundError from controllers.console.datasets.rag_pipeline import rag_pipeline_workflow as module from controllers.console.datasets.rag_pipeline.rag_pipeline_workflow import ( DraftWorkflowRunPayload, @@ -25,7 +25,6 @@ WorkflowUpdatePayload, ) from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from models.account import Account, TenantAccountRole from models.dataset import Dataset, Pipeline from models.engine import db @@ -35,6 +34,7 @@ from services.errors.llm import InvokeRateLimitError from services.errors.rag_pipeline import RagPipelineResourceNotFoundError from services.rag_pipeline.rag_pipeline import RagPipelineService +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator from tests.unit_tests.config_override import config_overrides_context from tests.unit_tests.model_factories import make_account, make_dataset, make_tenant @@ -262,7 +262,7 @@ def test_rag_pipeline_transform_rejects_read_only_member(sqlite_engine: Engine) with ( config_overrides_context(RBAC_ENABLED=False), - pytest.raises(Forbidden), + pytest.raises(AccessDeniedError), ): handler(api, session, DEFAULT_WORKFLOW_TENANT_ID, account, UUID(DEFAULT_DATASET_ID)) @@ -278,7 +278,7 @@ def test_rag_pipeline_transform_rejects_dataset_from_another_tenant_before_servi with ( patch.object(module.RagPipelineTransformService, "transform_dataset") as transform_dataset, - pytest.raises(NotFound), + pytest.raises(NotFoundError), ): handler(api, session, DEFAULT_WORKFLOW_TENANT_ID, _account(), UUID(DEFAULT_DATASET_ID)) @@ -297,7 +297,7 @@ def test_rag_pipeline_transform_enforces_legacy_dataset_permission_before_servic with ( config_overrides_context(RBAC_ENABLED=False), patch.object(module.RagPipelineTransformService, "transform_dataset") as transform_dataset, - pytest.raises(Forbidden), + pytest.raises(AccessDeniedError), ): handler(api, session, DEFAULT_WORKFLOW_TENANT_ID, _account(), UUID(DEFAULT_DATASET_ID)) @@ -341,7 +341,7 @@ def test_rag_pipeline_transform_maps_missing_pipeline_to_not_found(sqlite_engine "transform_dataset", side_effect=RagPipelineResourceNotFoundError("Pipeline not found"), ), - pytest.raises(NotFound, match="Pipeline not found"), + pytest.raises(NotFoundError, match="Pipeline not found"), ): handler(api, session, DEFAULT_WORKFLOW_TENANT_ID, _account(), UUID(DEFAULT_DATASET_ID)) @@ -385,7 +385,7 @@ def test_rag_pipeline_transform_skips_legacy_acl_when_rbac_is_enabled(sqlite_eng ), ], ) -def test_rag_pipeline_run_uses_sqlite_session( +def test_rag_pipeline_run_uses_injected_generator( app: Flask, sqlite_engine: Engine, pipeline_application: PipelineGenerator, @@ -412,7 +412,8 @@ def test_rag_pipeline_run_uses_sqlite_session( assert response == {"ok": True} load_pipeline.assert_called_once_with(session, pipeline.id) - assert generate.call_args.kwargs["session"] is session + assert generate.call_args.kwargs["pipeline"] is pipeline + assert generate.call_args.kwargs["streaming"] is (api_type is module.DraftRagPipelineRunApi) assert generate.call_args.kwargs["generator"] is pipeline_application assert session.get_bind() is sqlite_engine @@ -448,5 +449,5 @@ def test_rag_pipeline_run_translates_rate_limit( ): handler(api, req_data, session, _account(), pipeline.id) - assert generate.call_args.kwargs["session"] is session + assert generate.call_args.kwargs["pipeline"] is pipeline assert generate.call_args.kwargs["generator"] is pipeline_application diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow_apis.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow_apis.py index 9226d33433ba4b..20cc81c257137f 100644 --- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow_apis.py +++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow_apis.py @@ -3,23 +3,23 @@ from __future__ import annotations import json -from collections.abc import Generator, Iterator -from contextlib import contextmanager +from collections.abc import Iterator from dataclasses import dataclass from datetime import datetime from inspect import unwrap from typing import TypedDict, Unpack -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, Mock, patch from uuid import uuid4 import pytest from flask import Flask from sqlalchemy import Engine from sqlalchemy.orm import Session, scoped_session, sessionmaker -from werkzeug.exceptions import BadRequest, Forbidden, HTTPException, NotFound +from werkzeug.exceptions import HTTPException import models.workflow as workflow_models import services +from controllers.common.errors import AccessDeniedError, InvalidRequestError, NotFoundError from controllers.console.app.error import DraftWorkflowNotExist, DraftWorkflowNotSync from controllers.console.datasets.rag_pipeline import rag_pipeline_workflow as workflow_controller from controllers.console.datasets.rag_pipeline.rag_pipeline_workflow import ( @@ -45,14 +45,20 @@ WorkflowListQuery, WorkflowUpdatePayload, ) +from extensions.ext_application_services import ApplicationServices from fields.workflow_run_fields import node_execution_response_source from graphon.enums import WorkflowNodeExecutionStatus from libs.datetime_utils import naive_utc_now +from machinery.context import RequestContext from models.account import Account, TenantAccountRole from models.dataset import Pipeline from models.enums import CreatorUserRole from models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom +from repositories.workflow.definition_repository import workflow_record, workflow_snapshot from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError +from services.errors.workflow_service import WorkflowInUseError +from services.workflow.contracts import WorkflowOwner, WorkflowSnapshot +from services.workflow.draft_service import WorkflowDraftService pytestmark = pytest.mark.usefixtures("pipeline_application") @@ -266,7 +272,7 @@ def test_get_draft_success(self, app: Flask, workflow_author: Account, sqlite_se workflow = make_workflow(created_by=workflow_author.id) service = MagicMock() - service.get_draft_workflow.return_value = workflow + service.get_draft_workflow_record.return_value = workflow_record(workflow, sqlite_session) with ( app.test_request_context("/"), @@ -294,7 +300,7 @@ def test_get_draft_not_exist(self, app: Flask, sqlite_session: Session) -> None: pipeline = make_pipeline() service = MagicMock() - service.get_draft_workflow.return_value = None + service.get_draft_workflow_record.return_value = None with ( app.test_request_context("/"), @@ -306,25 +312,24 @@ def test_get_draft_not_exist(self, app: Flask, sqlite_session: Session) -> None: with pytest.raises(DraftWorkflowNotExist): method(api, sqlite_session, pipeline) - def test_sync_hash_not_match(self, app: Flask) -> None: + def test_sync_hash_not_match(self, app: Flask, workflow_application: ApplicationServices) -> None: api = DraftRagPipelineApi() method = unwrap(api.post) pipeline = make_pipeline() user = make_account() - service = MagicMock() - service.sync_draft_workflow.side_effect = WorkflowHashNotEqualError() + service = Mock(spec=WorkflowDraftService) + service.sync.side_effect = WorkflowHashNotEqualError() with ( app.test_request_context("/", json={"graph": empty_mapping(), "features": empty_mapping()}), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, - ), + patch.object(workflow_controller, "application_services", return_value=workflow_application), + patch.object(workflow_application.workflow_drafts, "restore", service.restore), + patch.object(workflow_application.workflow_drafts, "sync", service.sync), ): with pytest.raises(DraftWorkflowNotSync): - method(api, user, pipeline) + method(api, RequestContext("test", None, user.id, pipeline.tenant_id), pipeline.id) def test_sync_invalid_text_plain(self, app: Flask) -> None: api = DraftRagPipelineApi() @@ -336,10 +341,12 @@ def test_sync_invalid_text_plain(self, app: Flask) -> None: with ( app.test_request_context("/", data="bad-json", headers={"Content-Type": "text/plain"}), ): - response, status = method(api, user, pipeline) + response, status = method(api, RequestContext("test", None, user.id, pipeline.tenant_id), pipeline.id) assert status == 400 - def test_restore_published_workflow_to_draft_success(self, app: Flask) -> None: + def test_restore_published_workflow_to_draft_success( + self, app: Flask, workflow_application: ApplicationServices + ) -> None: api = RagPipelineDraftWorkflowRestoreApi() method = unwrap(api.post) @@ -350,62 +357,65 @@ def test_restore_published_workflow_to_draft_success(self, app: Flask) -> None: created_at=datetime(2024, 1, 1), ) - service = MagicMock() - service.restore_published_workflow_to_draft.return_value = workflow + service = Mock(spec=WorkflowDraftService) + service.restore.return_value = workflow_snapshot(workflow) with ( app.test_request_context("/", method="POST"), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, - ), + patch.object(workflow_controller, "application_services", return_value=workflow_application), + patch.object(workflow_application.workflow_drafts, "restore", service.restore), + patch.object(workflow_application.workflow_drafts, "sync", service.sync), ): - result = method(api, user, pipeline, "published-workflow") + result = method( + api, RequestContext("test", None, user.id, pipeline.tenant_id), pipeline.id, "published-workflow" + ) assert result["result"] == "success" assert result["hash"] == workflow.unique_hash - def test_restore_published_workflow_to_draft_not_found(self, app: Flask) -> None: + def test_restore_published_workflow_to_draft_not_found( + self, app: Flask, workflow_application: ApplicationServices + ) -> None: api = RagPipelineDraftWorkflowRestoreApi() method = unwrap(api.post) pipeline = make_pipeline() user = make_account(id="account-1") - service = MagicMock() - service.restore_published_workflow_to_draft.side_effect = WorkflowNotFoundError("Workflow not found") + service = Mock(spec=WorkflowDraftService) + service.restore.side_effect = WorkflowNotFoundError("Workflow not found") with ( app.test_request_context("/", method="POST"), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, - ), + patch.object(workflow_controller, "application_services", return_value=workflow_application), + patch.object(workflow_application.workflow_drafts, "restore", service.restore), + patch.object(workflow_application.workflow_drafts, "sync", service.sync), ): - with pytest.raises(NotFound): - method(api, user, pipeline, "published-workflow") + with pytest.raises(NotFoundError): + method( + api, RequestContext("test", None, user.id, pipeline.tenant_id), pipeline.id, "published-workflow" + ) - def test_restore_published_workflow_to_draft_returns_400_for_draft_source(self, app: Flask) -> None: + def test_restore_published_workflow_to_draft_returns_400_for_draft_source( + self, app: Flask, workflow_application: ApplicationServices + ) -> None: api = RagPipelineDraftWorkflowRestoreApi() method = unwrap(api.post) pipeline = make_pipeline() user = make_account(id="account-1") - service = MagicMock() - service.restore_published_workflow_to_draft.side_effect = IsDraftWorkflowError( - "source workflow must be published" - ) + service = Mock(spec=WorkflowDraftService) + service.restore.side_effect = IsDraftWorkflowError("source workflow must be published") with ( app.test_request_context("/", method="POST"), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, - ), + patch.object(workflow_controller, "application_services", return_value=workflow_application), + patch.object(workflow_application.workflow_drafts, "restore", service.restore), + patch.object(workflow_application.workflow_drafts, "sync", service.sync), ): with pytest.raises(HTTPException) as exc: - method(api, user, pipeline, "draft-workflow") + method(api, RequestContext("test", None, user.id, pipeline.tenant_id), pipeline.id, "draft-workflow") assert exc.value.code == 400 assert exc.value.description == "source workflow must be published" @@ -447,7 +457,7 @@ def test_iteration_node_conversation_not_exists(self, app: Flask) -> None: side_effect=services.errors.conversation.ConversationNotExistsError(), ), ): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): method(api, NodeRunPayload(), user, pipeline, "node") def test_loop_node_success(self, app: Flask) -> None: @@ -479,14 +489,12 @@ def test_execution_not_found(self, app: Flask) -> None: pipeline = make_pipeline() user = make_account() - service = MagicMock() - service.run_draft_workflow_node.return_value = None - with ( app.test_request_context("/", json={"inputs": empty_mapping()}), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, + patch.object( + workflow_controller.application_services().knowledge.pipeline_execution, + "run_draft_workflow_node", + return_value=None, ), ): with pytest.raises(ValueError): @@ -494,35 +502,38 @@ def test_execution_not_found(self, app: Flask) -> None: class TestPublishedPipelineApis: - def test_publish_success(self, app: Flask) -> None: + @pytest.mark.parametrize("outcome", ["success", "conflict", "missing", "invalid"]) + def test_publish( + self, app: Flask, workflow_application: ApplicationServices, monkeypatch: pytest.MonkeyPatch, outcome: str + ) -> None: api = PublishedRagPipelineApi() method = unwrap(api.post) - - tenant_id = str(uuid4()) - pipeline = Pipeline( - tenant_id=tenant_id, - name="test-pipeline", - description="test", - created_by=str(uuid4()), - ) - user = make_account(id="u1") - - workflow = make_workflow(id=str(uuid4()), created_at=naive_utc_now()) - - service = MagicMock() - service.publish_workflow.return_value = workflow - - with ( - app.test_request_context("/"), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, - ), - ): - result = method(api, user, pipeline) - - assert result["result"] == "success" - assert "created_at" in result + context = RequestContext("test", None, "account-1", "tenant-1") + calls: list[tuple[RequestContext, str]] = [] + workflow = workflow_snapshot(make_workflow(id="published", created_at=naive_utc_now())) + + def publish(context: RequestContext, pipeline_id: str) -> WorkflowSnapshot: + calls.append((context, pipeline_id)) + if outcome == "invalid": + raise workflow_controller.RagPipelinePublicationError("Invalid configuration") + if outcome == "conflict": + raise workflow_controller.WorkflowHashNotEqualError() + if outcome == "missing": + raise workflow_controller.WorkflowNotFoundError("Pipeline not found") + return workflow + + monkeypatch.setattr(workflow_controller, "application_services", lambda: workflow_application) + monkeypatch.setattr(workflow_application.knowledge.pipeline_publication, "publish", publish) + with app.test_request_context("/", method="POST"): + if outcome == "success": + result = method(api, context, "pipeline-1") + assert result["result"] == "success" + assert "created_at" in result + else: + with pytest.raises(HTTPException) as error: + method(api, context, "pipeline-1") + assert error.value.code == {"conflict": 409, "missing": 404, "invalid": 400}[outcome] + assert calls == [(context, "pipeline-1")] class TestMiscApis: @@ -565,7 +576,9 @@ def test_recommended_plugins(self, app: Flask) -> None: ): result = method(api, RagPipelineRecommendedPluginQuery(type="all"), tenant_id, user) assert result == recommended_plugins - service.get_recommended_plugins.assert_called_once_with("all", user, tenant_id) + service.get_recommended_plugins.assert_called_once_with( + "all", user, tenant_id, tool_providers=workflow_controller.application_services().tools.tool_providers + ) class TestDefaultBlockConfigApi: @@ -600,7 +613,7 @@ def test_get_block_config_invalid_json(self, app: Flask) -> None: class TestPublishedAllRagPipelineApi: - def test_get_published_workflows_success(self, app: Flask) -> None: + def test_get_published_workflows_success(self, app: Flask, sqlite_session: Session) -> None: api = PublishedAllRagPipelineApi() method = unwrap(api.get) @@ -608,7 +621,10 @@ def test_get_published_workflows_success(self, app: Flask) -> None: user = make_account(id="u1") service = MagicMock() - service.get_all_published_workflow.return_value = ([make_workflow(id="w1")], False) + service.get_all_published_workflow.return_value = ( + [workflow_record(make_workflow(id="w1"), sqlite_session)], + False, + ) with ( app.test_request_context("/"), @@ -633,12 +649,12 @@ def test_get_published_workflows_forbidden(self, app: Flask) -> None: with ( app.test_request_context("/?user_id=u2"), ): - with pytest.raises(Forbidden): + with pytest.raises(AccessDeniedError): method(api, WorkflowListQuery(user_id="u2"), user, pipeline) class TestRagPipelineByIdApi: - def test_patch_success(self, app: Flask) -> None: + def test_patch_success(self, app: Flask, sqlite_session: Session) -> None: api = RagPipelineByIdApi() method = unwrap(api.patch) @@ -648,7 +664,7 @@ def test_patch_success(self, app: Flask) -> None: workflow = make_workflow(id="w1", marked_name="test") service = MagicMock() - service.update_workflow.return_value = workflow + service.update_workflow.return_value = workflow_record(workflow, sqlite_session) payload = {"marked_name": "test"} @@ -676,79 +692,25 @@ def test_patch_no_fields(self, app: Flask) -> None: result, status = method(api, WorkflowUpdatePayload(), user, pipeline, "w1") assert status == 400 - @pytest.mark.parametrize("transaction_fails", [False, True], ids=["commit-succeeds", "commit-fails"]) - def test_delete_retires_candidates_only_after_transaction_exit( - self, - app: Flask, - transaction_fails: bool, + @pytest.mark.parametrize("in_use", [False, True]) + def test_delete_delegates_to_application( + self, app: Flask, workflow_application: ApplicationServices, in_use: bool ) -> None: api = RagPipelineByIdApi() method = unwrap(api.delete) - - pipeline = make_pipeline(tenant_id="t1", workflow_id="active-workflow") - user = make_account() - - events: list[str] = [] - error = RuntimeError("commit failed") - workflow_service = MagicMock() - workflow_service.delete_workflow.side_effect = lambda **_kwargs: events.append("delete") or ["inline-agent"] - transaction_factory = MagicMock() - - @contextmanager - def transaction() -> Generator[Session]: - events.append("transaction-enter") - yield MagicMock(spec=Session) - events.append("transaction-exit") - if transaction_fails: - raise error - - transaction_factory.begin.side_effect = transaction - + context = RequestContext("delete", None, "account", "tenant") + delete = Mock(side_effect=WorkflowInUseError("currently in use by pipeline") if in_use else None) with ( app.test_request_context("/", method="DELETE"), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.WorkflowService", - return_value=workflow_service, - ), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.sessionmaker", - return_value=transaction_factory, - ), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow." - "WorkflowAgentRetirementService.retire_unowned" - ) as retire_unowned, + patch.object(workflow_controller, "application_services", return_value=workflow_application), + patch.object(workflow_application.console_workflows, "delete", delete), ): - retire_unowned.side_effect = lambda **_kwargs: events.append("retire") - if transaction_fails: - with pytest.raises(RuntimeError) as exc_info: - method(api, user, pipeline, "old-workflow") - assert exc_info.value is error + if in_use: + with pytest.raises(InvalidRequestError, match="currently in use by pipeline"): + method(api, context, "pipeline", "version") else: - result = method(api, user, pipeline, "old-workflow") - assert result == (None, 204) - - workflow_service.delete_workflow.assert_called_once() - assert events == ["transaction-enter", "delete", "transaction-exit"] + ([] if transaction_fails else ["retire"]) - if transaction_fails: - retire_unowned.assert_not_called() - else: - retire_unowned.assert_called_once_with( - tenant_id=pipeline.tenant_id, - agent_ids=["inline-agent"], - account_id=user.id, - ) - - def test_delete_active_workflow_rejected(self, app: Flask) -> None: - api = RagPipelineByIdApi() - method = unwrap(api.delete) - - pipeline = make_pipeline(tenant_id="t1", workflow_id="active-workflow") - user = make_account() - - with app.test_request_context("/", method="DELETE"): - with pytest.raises(BadRequest, match="currently in use by pipeline"): - method(api, user, pipeline, "active-workflow") + assert method(api, context, "pipeline", "version") == (None, 204) + delete.assert_called_once_with(context, WorkflowOwner("pipeline", "pipeline"), "version") class TestRagPipelineWorkflowLastRunApi: @@ -792,12 +754,12 @@ def test_last_run_not_found(self, app: Flask) -> None: return_value=service, ), ): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): method(api, pipeline, "node1") class TestRagPipelineWorkflowRunNodeExecutionListApi: - def test_get_node_executions_passes_current_user(self, app: Flask) -> None: + def test_get_node_executions_passes_current_user(self, app: Flask, *, tool_providers) -> None: api = RagPipelineWorkflowRunNodeExecutionListApi() method = unwrap(api.get) @@ -810,7 +772,7 @@ def test_get_node_executions_passes_current_user(self, app: Flask) -> None: session_stub.scalar.return_value = None service = MagicMock() service.get_rag_pipeline_workflow_run_node_executions.return_value = [ - node_execution_response_source(node_exec, session=session_stub) + node_execution_response_source(node_exec, session=session_stub, tool_providers=tool_providers) ] with ( @@ -826,6 +788,7 @@ def test_get_node_executions_passes_current_user(self, app: Flask) -> None: pipeline=pipeline, run_id=str(run_id), user=user, + tool_providers=workflow_controller.application_services().tools.tool_providers, ) assert result["data"][0]["id"] == "node-exec-1" assert result["data"][0]["inputs"] == {"query": "hello"} @@ -847,14 +810,12 @@ def test_set_datasource_variables_success(self, app: Flask) -> None: "start_node_title": "Node", } - service = MagicMock() - service.set_datasource_variables.return_value = make_node_execution(node_id="n1") - with ( app.test_request_context("/", json=payload), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, + patch.object( + workflow_controller.application_services().knowledge.pipeline_execution, + "set_datasource_variables", + return_value=make_node_execution(node_id="n1"), ), ): result = method(api, DatasourceVariablesPayload.model_validate(payload), user, pipeline) diff --git a/api/tests/unit_tests/controllers/console/explore/test_completion.py b/api/tests/unit_tests/controllers/console/explore/test_completion.py index faa349483d344d..21bdd378150174 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_completion.py +++ b/api/tests/unit_tests/controllers/console/explore/test_completion.py @@ -17,6 +17,8 @@ from services.errors.llm import InvokeRateLimitError from services.installed_app_generation_adapters import AppGenerateServiceRuntime from services.installed_app_generation_service import GenerationResponse, InstalledAppGenerationService +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.unit_tests.controllers.console.explore.test_installed_app_admission import ( _assert_json_response, _Harness, @@ -335,6 +337,9 @@ def test_chat_conversation_preflight_returns_404_before_starting_generation_and_ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, rejection: str, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, ) -> None: conversation_id = str(uuid4()) with sqlite_session_factory.begin() as session: @@ -357,7 +362,9 @@ def test_chat_conversation_preflight_returns_404_before_starting_generation_and_ installed_apps=_InstalledAppServices( generation=InstalledAppGenerationService( usage=SQLAlchemyInstalledAppRepository(session_factory=sqlite_session_factory), - runtime=AppGenerateServiceRuntime(session_factory=sqlite_session_factory), + runtime=AppGenerateServiceRuntime( + session_factory=sqlite_session_factory, variables=workflow_variables, runtime=workflow_runtime + ), ) ) ) diff --git a/api/tests/unit_tests/controllers/console/explore/test_installed_app_completion.py b/api/tests/unit_tests/controllers/console/explore/test_installed_app_completion.py index 72ccd03f418c3e..7203e4b1124741 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_installed_app_completion.py +++ b/api/tests/unit_tests/controllers/console/explore/test_installed_app_completion.py @@ -18,6 +18,8 @@ from services.errors.conversation import ConversationCompletedError, ConversationNotExistsError from services.installed_app_generation_adapters import AppGenerateServiceRuntime from services.installed_app_generation_service import GenerationResponse, InstalledAppGenerationService +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.unit_tests.controllers.console.explore.test_installed_app_admission import ( _assert_json_response, _Harness, @@ -391,6 +393,9 @@ def test_resource_removed_after_admission_is_revalidated_before_generation( sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, deleted_resource: str, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, ) -> None: def delete_resource() -> None: with sqlite_session_factory.begin() as session: @@ -406,7 +411,9 @@ def delete_resource() -> None: installed_apps=_InstalledAppServices( generation=InstalledAppGenerationService( usage=SQLAlchemyInstalledAppRepository(session_factory=sqlite_session_factory), - runtime=AppGenerateServiceRuntime(session_factory=sqlite_session_factory), + runtime=AppGenerateServiceRuntime( + session_factory=sqlite_session_factory, variables=workflow_variables, runtime=workflow_runtime + ), ) ) ) diff --git a/api/tests/unit_tests/controllers/console/explore/test_message.py b/api/tests/unit_tests/controllers/console/explore/test_message.py index 84b40deb09703d..0b1184d24b7dc9 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_message.py +++ b/api/tests/unit_tests/controllers/console/explore/test_message.py @@ -31,6 +31,7 @@ from repositories.installed_app_message_repository import SQLAlchemyInstalledAppMessageRepository from repositories.installed_app_repository import SQLAlchemyInstalledAppRepository from services.errors.app import MoreLikeThisDisabledError +from services.errors.app_model_config import AppModelConfigBrokenError from services.errors.conversation import ConversationNotExistsError from services.errors.message import MessageNotExistsError, SuggestedQuestionsAfterAnswerDisabledError from services.installed_app_generation_service import GenerationResponse, InstalledAppGenerationService @@ -814,6 +815,7 @@ def test_generation_failures_keep_provider_errors_and_context( [ ("more-like-this", MessageNotExistsError(), 404, "message_not_found"), ("more-like-this", MoreLikeThisDisabledError(), 403, "app_more_like_this_disabled"), + ("more-like-this", AppModelConfigBrokenError(), 400, "app_unavailable"), ("more-like-this", ValueError("Bad runtime arguments"), 400, "invalid_param"), ("suggested-questions", MessageNotExistsError(), 404, "message_not_found"), ("suggested-questions", ConversationNotExistsError(), 404, "conversation_not_found"), diff --git a/api/tests/unit_tests/controllers/console/explore/test_trial_audio.py b/api/tests/unit_tests/controllers/console/explore/test_trial_audio.py index 319b5c50870f03..19e0c534dc46a9 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_trial_audio.py +++ b/api/tests/unit_tests/controllers/console/explore/test_trial_audio.py @@ -233,6 +233,16 @@ def _assert_error(response: TestResponse, status: int, code: str) -> Mapping[str return body +def test_multipart_body_limit_is_handled_before_service_error_translation(harness: _Harness) -> None: + harness.app.config["MAX_CONTENT_LENGTH"] = 8 + + response = harness.post(_ASR) + + _assert_error(response, 413, "request_entity_too_large") + assert harness.runtime.calls == [] + assert harness.usage() is None + + def test_asr_upload_keeps_transcript_and_owner_tenant(harness: _Harness) -> None: response = harness.post(_ASR) diff --git a/api/tests/unit_tests/controllers/console/explore/test_trial_generation.py b/api/tests/unit_tests/controllers/console/explore/test_trial_generation.py index a66a8bdc18bfa6..2eed6f9133a56f 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_trial_generation.py +++ b/api/tests/unit_tests/controllers/console/explore/test_trial_generation.py @@ -16,7 +16,6 @@ import controllers.console.explore.trial_app_admission as admission_module import controllers.console.wraps as console_wraps import libs.login as login_module -from core.app.apps.agent_app.errors import AgentAppNotPublishedError from core.app.entities.app_invoke_entities import InvokeFrom from core.errors.error import ( AppInvokeQuotaExceededError, @@ -32,6 +31,7 @@ from models.account import AccountStatus from repositories.trial_app_repository import TrialAppRepository from services.account_errors import AccountNotFoundError +from services.app.generation.errors import AgentAppNotPublishedError from services.app_definition_query_service import AppDefinitionUnavailableError from services.errors.app_model_config import AppModelConfigBrokenError from services.errors.conversation import ConversationCompletedError, ConversationNotExistsError diff --git a/api/tests/unit_tests/controllers/console/explore/test_trial_preview_details.py b/api/tests/unit_tests/controllers/console/explore/test_trial_preview_details.py index 259e8d2a66e133..dde68bb212326a 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_trial_preview_details.py +++ b/api/tests/unit_tests/controllers/console/explore/test_trial_preview_details.py @@ -29,7 +29,6 @@ from core.plugin.plugin_service import PluginService from core.tools.__base.tool import Tool from core.tools.entities.tool_entities import ToolParameter -from core.tools.tool_manager import ToolManager from core.workflow.llm_environment_variable import LLMEnvironmentVariable from enums import DeploymentEdition from extensions.ext_login import DifyLoginManager, unauthorized_handler @@ -54,6 +53,8 @@ from models.tools import WorkflowToolProvider from models.workflow import Workflow from repositories.app_preview_query_repository import AppPreviewQueryRepository +from repositories.tools.provider_repository import ToolProviderRepository +from repositories.tools.workflow_repository import WorkflowToolRepository from repositories.trial_app_repository import TrialAppRepository from services.app_preview_details_adapters import AppPreviewDetailsRuntime from services.app_preview_query_service import AppPreviewQueryService @@ -63,6 +64,7 @@ RecommendedAppDetailRecord, RecommendedAppQueryService, ) +from services.tools.tool_manager import ToolManager _Endpoint = Literal["detail", "workflows"] _CREATED_AT = datetime(2024, 1, 1) @@ -115,7 +117,17 @@ def decrypt(self, tenant_id: str, token: str) -> str: assert tenant_id == self.viewer_workspace_id return "tool-secret" - def tool_runtime(self, *, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str) -> Tool: + def tool_runtime( + self, + *, + tenant_id: str, + app_id: str, + agent_tool: AgentToolEntity, + user_id: str, + tool_providers, + workflow_queries, + ) -> Tool: + del workflow_queries, tool_providers assert isinstance(self.engine.pool, QueuePool) assert self.engine.pool.checkedout() == 0 assert tenant_id == self.viewer_workspace_id @@ -214,7 +226,14 @@ def legacy_detail(self) -> dict[str, object]: app = session.get(App, self.target.id) assert app is not None return trial_module.TrialAppDetailResponse.model_validate( - AppResponseView(app, session=session, account=self.viewer), from_attributes=True + AppResponseView( + app, + session=session, + account=self.viewer, + tool_providers=ToolProviderRepository(self.factory), + workflow_queries=WorkflowToolRepository(self.factory), + ), + from_attributes=True, ).model_dump(mode="json") @@ -224,6 +243,9 @@ def harness( config_overrides: Callable[..., None], sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], + *, + tool_providers, + workflow_queries, ) -> _Harness: config_overrides( LOGIN_DISABLED=False, @@ -353,7 +375,11 @@ def track_session(session: Session, _transaction: SessionTransaction, _connectio app_previews=AppPreviewQueryService( apps=AppPreviewQueryRepository(session_factory=factory), is_previewable=recommendations.is_previewable ), - app_preview_details=AppPreviewDetailsRuntime(details=AppPreviewQueryRepository(session_factory=factory)), + app_preview_details=AppPreviewDetailsRuntime( + details=AppPreviewQueryRepository(session_factory=factory), + tool_providers=tool_providers, + workflow_queries=workflow_queries, + ), recommended_app_queries=recommendations, ) for module in (trial_module, admission_module): @@ -514,10 +540,19 @@ def hardcoded_provider(provider_id: str) -> object: return object() raise KeyError(provider_id) - def tool_runtime(*, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str) -> Tool: + def tool_runtime( + *, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str, tool_providers, workflow_queries + ) -> Tool: if agent_tool.provider_id != "example": raise ValueError("Invalid historical provider ID") - return harness.io.tool_runtime(tenant_id=tenant_id, app_id=app_id, agent_tool=agent_tool, user_id=user_id) + return harness.io.tool_runtime( + tenant_id=tenant_id, + app_id=app_id, + agent_tool=agent_tool, + user_id=user_id, + tool_providers=tool_providers, + workflow_queries=workflow_queries, + ) plugin_queries: list[str] = [] @@ -593,9 +628,18 @@ def get_runtime_parameters(self) -> list[ToolParameter]: SimpleNamespace(parameters=[parameter], identity=SimpleNamespace(name="broken")) ) - def tool_runtime(*, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str) -> Tool: + def tool_runtime( + *, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str, tool_providers, workflow_queries + ) -> Tool: if agent_tool.provider_id == "example": - return harness.io.tool_runtime(tenant_id=tenant_id, app_id=app_id, agent_tool=agent_tool, user_id=user_id) + return harness.io.tool_runtime( + tenant_id=tenant_id, + app_id=app_id, + agent_tool=agent_tool, + user_id=user_id, + tool_providers=tool_providers, + workflow_queries=workflow_queries, + ) if failure == "runtime": raise RuntimeError("Tool provider is unavailable") return cast(Tool, broken_runtime) diff --git a/api/tests/unit_tests/controllers/console/explore/test_workflow.py b/api/tests/unit_tests/controllers/console/explore/test_workflow.py index e44c82310f7193..c849bbdb11fed0 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_workflow.py +++ b/api/tests/unit_tests/controllers/console/explore/test_workflow.py @@ -20,6 +20,8 @@ from services.errors.llm import InvokeRateLimitError from services.installed_app_generation_adapters import AppGenerateServiceRuntime from services.installed_app_generation_service import GenerationResponse, InstalledAppGenerationService +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.unit_tests.controllers.console.explore.test_installed_app_admission import ( _assert_json_response, _Harness, @@ -388,6 +390,9 @@ def test_workflow_run_revalidates_app_while_stop_uses_admission_snapshot( sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, action: str, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, ) -> None: def remove_app() -> None: with sqlite_session_factory.begin() as session: @@ -400,7 +405,9 @@ def remove_app() -> None: installed_apps=_InstalledAppServices( generation=InstalledAppGenerationService( usage=SQLAlchemyInstalledAppRepository(session_factory=sqlite_session_factory), - runtime=AppGenerateServiceRuntime(session_factory=sqlite_session_factory), + runtime=AppGenerateServiceRuntime( + session_factory=sqlite_session_factory, variables=workflow_variables, runtime=workflow_runtime + ), ) ), app_tasks=AppTaskControlService(redis_client=stop_redis), diff --git a/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py b/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py index 1cc40426f48593..ddadb86ac5b9e0 100644 --- a/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py +++ b/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py @@ -2,17 +2,23 @@ import json from inspect import unwrap +from typing import NoReturn from unittest.mock import Mock import pytest from flask import Flask from sqlalchemy.orm import Session -from werkzeug.exceptions import HTTPException, NotFound +from werkzeug.exceptions import HTTPException +from controllers.common.errors import NotFoundError from controllers.console.snippets import snippet_workflow as snippet_workflow_module +from extensions.ext_application_services import ApplicationServices +from machinery.context import RequestContext from models.account import Account, TenantAccountRole from models.snippet import CustomizedSnippet from models.workflow import Workflow +from repositories.workflow.definition_repository import workflow_record, workflow_snapshot +from services.workflow.contracts import WorkflowRecord from tests.unit_tests.model_factories import make_account @@ -25,8 +31,8 @@ def _account(account_id: str = "account-1") -> Account: ) -def _snippet(**overrides) -> CustomizedSnippet: - data = { +def _snippet(**overrides: object) -> CustomizedSnippet: + data: dict[str, object] = { "id": "snippet-1", "tenant_id": "tenant-1", "name": "Snippet", @@ -38,7 +44,7 @@ def _snippet(**overrides) -> CustomizedSnippet: return CustomizedSnippet(**data) -def _workflow(**overrides) -> Workflow: +def _workflow(**overrides: object) -> Workflow: workflow = Workflow.new( tenant_id="tenant-1", app_id="snippet-1", @@ -57,9 +63,9 @@ def _workflow(**overrides) -> Workflow: return workflow -def test_get_snippet_requires_snippet_id(app): +def test_get_snippet_requires_snippet_id(app: Flask) -> None: @snippet_workflow_module.get_snippet - def view(**kwargs): + def view(**kwargs: object) -> object: return kwargs with app.test_request_context("/snippets"): @@ -71,7 +77,7 @@ def test_get_snippet_injects_resolved_snippet(app: Flask, monkeypatch: pytest.Mo snippet = _snippet() @snippet_workflow_module.get_snippet - def view(**kwargs): + def view(**kwargs: object) -> object: return kwargs["snippet"] monkeypatch.setattr( @@ -89,7 +95,7 @@ def view(**kwargs): def test_get_snippet_raises_not_found_when_snippet_missing(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: @snippet_workflow_module.get_snippet - def view(**kwargs): + def view(**kwargs: object) -> object: return kwargs monkeypatch.setattr( @@ -100,7 +106,7 @@ def view(**kwargs): monkeypatch.setattr(snippet_workflow_module.SnippetService, "get_snippet_by_id", Mock(return_value=None)) with app.test_request_context("/snippets/snippet-1"): - with pytest.raises(NotFound, match="Snippet not found"): + with pytest.raises(NotFoundError, match="Snippet not found"): view(snippet_id="snippet-1") @@ -108,7 +114,7 @@ def test_draft_workflow_get_raises_when_missing( app: Flask, monkeypatch: pytest.MonkeyPatch, unbound_session: Session ) -> None: snippet = _snippet() - monkeypatch.setattr(snippet_workflow_module.SnippetService, "get_draft_workflow", Mock(return_value=None)) + monkeypatch.setattr(snippet_workflow_module.SnippetService, "get_draft_workflow_record", Mock(return_value=None)) api = snippet_workflow_module.SnippetDraftWorkflowApi() handler = unwrap(api.get) @@ -118,18 +124,17 @@ def test_draft_workflow_get_raises_when_missing( handler(api, unbound_session, snippet=snippet) -def test_draft_workflow_get_uses_session_aware_response_source( +def test_draft_workflow_get_serializes_materialized_record( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session ) -> None: workflow = _workflow(updated_by="account-2") sqlite_session.add_all([_account(), _account("account-2")]) sqlite_session.commit() snippet = _snippet() - monkeypatch.setattr(snippet_workflow_module.SnippetService, "get_draft_workflow", Mock(return_value=workflow)) monkeypatch.setattr( - snippet_workflow_module.WorkflowAgentPublishService, - "project_draft_bindings_to_graph", - Mock(return_value=workflow.graph_dict), + snippet_workflow_module.SnippetService, + "get_draft_workflow_record", + Mock(return_value=workflow_record(workflow, sqlite_session)), ) api = snippet_workflow_module.SnippetDraftWorkflowApi() @@ -144,11 +149,13 @@ def test_draft_workflow_get_uses_session_aware_response_source( assert response["tool_published"] is False -def test_draft_workflow_post_returns_400_for_invalid_graph(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: +def test_draft_workflow_post_returns_400_for_invalid_graph( + app: Flask, monkeypatch: pytest.MonkeyPatch, workflow_application: ApplicationServices +) -> None: user = _account("account-1") snippet = _snippet() sync_draft_workflow = Mock(side_effect=ValueError("invalid graph")) - monkeypatch.setattr(snippet_workflow_module.SnippetService, "sync_draft_workflow", sync_draft_workflow) + monkeypatch.setattr(workflow_application.workflow_drafts, "sync", sync_draft_workflow) api = snippet_workflow_module.SnippetDraftWorkflowApi() handler = unwrap(api.post) @@ -158,20 +165,13 @@ def test_draft_workflow_post_returns_400_for_invalid_graph(app: Flask, monkeypat method="POST", json={"graph": {"nodes": [], "edges": []}, "hash": "hash-1"}, ): - response, status_code = handler( - api, - snippet_workflow_module.SnippetDraftSyncPayload.model_validate( - {"graph": {"nodes": [], "edges": []}, "hash": "hash-1"} - ), - user, - snippet, - ) + response, status_code = handler(api, RequestContext("test", None, user.id, snippet.tenant_id), snippet.id) assert status_code == 400 assert response == {"message": "invalid graph"} -def test_draft_config_returns_parallel_depth_limit(app) -> None: +def test_draft_config_returns_parallel_depth_limit(app: Flask) -> None: api = snippet_workflow_module.SnippetDraftConfigApi() handler = unwrap(api.get) @@ -179,7 +179,7 @@ def test_draft_config_returns_parallel_depth_limit(app) -> None: assert handler(api, snippet=_snippet()) == {"parallel_depth_limit": 3} -def test_published_workflow_get_returns_none_when_not_published(app, unbound_session: Session) -> None: +def test_published_workflow_get_returns_none_when_not_published(app: Flask, unbound_session: Session) -> None: api = snippet_workflow_module.SnippetPublishedWorkflowApi() handler = unwrap(api.get) @@ -187,14 +187,18 @@ def test_published_workflow_get_returns_none_when_not_published(app, unbound_ses assert handler(api, unbound_session, snippet=_snippet(is_published=False)) is None -def test_published_workflow_get_uses_session_aware_response_source( +def test_published_workflow_get_serializes_materialized_record( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session ) -> None: workflow = _workflow(updated_by="account-2") sqlite_session.add_all([_account(), _account("account-2")]) sqlite_session.commit() snippet = _snippet(is_published=True) - monkeypatch.setattr(snippet_workflow_module.SnippetService, "get_published_workflow", Mock(return_value=workflow)) + monkeypatch.setattr( + snippet_workflow_module.SnippetService, + "get_published_workflow_record", + Mock(return_value=workflow_record(workflow, sqlite_session)), + ) api = snippet_workflow_module.SnippetPublishedWorkflowApi() handler = unwrap(api.get) @@ -218,9 +222,7 @@ def test_published_workflow_post_returns_400_when_publish_fails( sqlite_session.add(snippet) sqlite_session.commit() - def fail_publish(*, session: Session, snippet: CustomizedSnippet, account: Account): - snippet.name = "Uncommitted name" - session.add(snippet) + def fail_publish(*, snippet: CustomizedSnippet, account: Account) -> NoReturn: raise ValueError("No valid workflow found.") monkeypatch.setattr(snippet_workflow_module.SnippetService, "publish_workflow", Mock(side_effect=fail_publish)) @@ -229,7 +231,7 @@ def fail_publish(*, session: Session, snippet: CustomizedSnippet, account: Accou handler = unwrap(api.post) with app.test_request_context("/snippets/snippet-1/workflows/publish", method="POST", json={}): - response, status_code = handler(api, sqlite_session, user, snippet) + response, status_code = handler(api, user, snippet) assert status_code == 400 assert response == {"message": "No valid workflow found."} @@ -252,11 +254,25 @@ def test_published_workflow_post_returns_success( api = snippet_workflow_module.SnippetPublishedWorkflowApi() handler = unwrap(api.post) with app.test_request_context("/snippets/snippet-1/workflows/publish", method="POST", json={}): - response = handler(api, sqlite_session, user, snippet) + response = handler(api, user, snippet) assert response["result"] == "success" +def test_published_workflow_post_translates_preparation_conflict(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: + from controllers.console.app.error import DraftWorkflowNotSync + from services.errors.app import WorkflowHashNotEqualError + + monkeypatch.setattr( + snippet_workflow_module.SnippetService, "publish_workflow", Mock(side_effect=WorkflowHashNotEqualError) + ) + api = snippet_workflow_module.SnippetPublishedWorkflowApi() + with app.test_request_context("/snippets/snippet-1/workflows/publish", method="POST", json={}): + with pytest.raises(DraftWorkflowNotSync) as error: + unwrap(api.post)(api, _account("account-1"), _snippet()) + assert error.value.code == 409 + + def test_default_block_configs_delegates_to_service(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: get_default_block_configs = Mock(return_value=[{"type": "llm"}]) monkeypatch.setattr(snippet_workflow_module.SnippetService, "get_default_block_configs", get_default_block_configs) @@ -281,7 +297,9 @@ def test_list_published_snippet_workflows_includes_input_fields( snippet = _snippet(input_fields=json.dumps(input_fields)) monkeypatch.setattr( - snippet_workflow_module.SnippetService, "get_all_published_workflows", Mock(return_value=([workflow], False)) + snippet_workflow_module.SnippetService, + "get_all_published_workflows", + Mock(return_value=([workflow_record(workflow, sqlite_session)], False)), ) api = snippet_workflow_module.SnippetPublishedAllWorkflowApi() @@ -298,14 +316,14 @@ def test_list_published_snippet_workflows_includes_input_fields( assert response["items"][0]["input_fields"] == input_fields -def test_restore_published_snippet_workflow_to_draft_success(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: +def test_restore_published_snippet_workflow_to_draft_success( + app: Flask, monkeypatch: pytest.MonkeyPatch, workflow_application: ApplicationServices +) -> None: workflow = _workflow(updated_at=None) user = _account("account-1") snippet = _snippet() - monkeypatch.setattr( - snippet_workflow_module.SnippetService, "restore_published_workflow_to_draft", Mock(return_value=workflow) - ) + monkeypatch.setattr(workflow_application.workflow_drafts, "restore", Mock(return_value=workflow_snapshot(workflow))) api = snippet_workflow_module.SnippetDraftWorkflowRestoreApi() handler = unwrap(api.post) @@ -314,19 +332,23 @@ def test_restore_published_snippet_workflow_to_draft_success(app: Flask, monkeyp "/snippets/snippet-1/workflows/published-workflow/restore", method="POST", ): - response = handler(api, user, snippet, workflow_id="published-workflow") + response = handler( + api, RequestContext("test", None, user.id, snippet.tenant_id), snippet.id, workflow_id="published-workflow" + ) assert response["result"] == "success" assert response["hash"] == workflow.unique_hash -def test_restore_published_snippet_workflow_to_draft_not_found(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: +def test_restore_published_snippet_workflow_to_draft_not_found( + app: Flask, monkeypatch: pytest.MonkeyPatch, workflow_application: ApplicationServices +) -> None: user = _account("account-1") snippet = _snippet() monkeypatch.setattr( - snippet_workflow_module.SnippetService, - "restore_published_workflow_to_draft", + workflow_application.workflow_drafts, + "restore", Mock(side_effect=snippet_workflow_module.WorkflowNotFoundError("Workflow not found")), ) @@ -337,19 +359,24 @@ def test_restore_published_snippet_workflow_to_draft_not_found(app: Flask, monke "/snippets/snippet-1/workflows/published-workflow/restore", method="POST", ): - with pytest.raises(NotFound): - handler(api, user, snippet, workflow_id="published-workflow") + with pytest.raises(NotFoundError): + handler( + api, + RequestContext("test", None, user.id, snippet.tenant_id), + snippet.id, + workflow_id="published-workflow", + ) def test_restore_published_snippet_workflow_to_draft_returns_400_for_draft_source( - app, monkeypatch: pytest.MonkeyPatch + app: Flask, monkeypatch: pytest.MonkeyPatch, workflow_application: ApplicationServices ) -> None: user = _account("account-1") snippet = _snippet() monkeypatch.setattr( - snippet_workflow_module.SnippetService, - "restore_published_workflow_to_draft", + workflow_application.workflow_drafts, + "restore", Mock(side_effect=snippet_workflow_module.IsDraftWorkflowError("source workflow must be published")), ) @@ -361,21 +388,23 @@ def test_restore_published_snippet_workflow_to_draft_returns_400_for_draft_sourc method="POST", ): with pytest.raises(HTTPException) as exc: - handler(api, user, snippet, workflow_id="draft-workflow") + handler( + api, RequestContext("test", None, user.id, snippet.tenant_id), snippet.id, workflow_id="draft-workflow" + ) assert exc.value.code == 400 assert exc.value.description == snippet_workflow_module.RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE def test_restore_published_snippet_workflow_to_draft_returns_400_for_invalid_graph( - app, monkeypatch: pytest.MonkeyPatch + app: Flask, monkeypatch: pytest.MonkeyPatch, workflow_application: ApplicationServices ) -> None: user = _account("account-1") snippet = _snippet() monkeypatch.setattr( - snippet_workflow_module.SnippetService, - "restore_published_workflow_to_draft", + workflow_application.workflow_drafts, + "restore", Mock(side_effect=ValueError("invalid snippet workflow graph")), ) @@ -387,7 +416,12 @@ def test_restore_published_snippet_workflow_to_draft_returns_400_for_invalid_gra method="POST", ): with pytest.raises(HTTPException) as exc: - handler(api, user, snippet, workflow_id="published-workflow") + handler( + api, + RequestContext("test", None, user.id, snippet.tenant_id), + snippet.id, + workflow_id="published-workflow", + ) assert exc.value.code == 400 assert exc.value.description == "invalid snippet workflow graph" @@ -405,10 +439,10 @@ def test_update_published_snippet_workflow_returns_updated_workflow( sqlite_session.add(snippet) sqlite_session.commit() - def update_persisted_snippet(*, session: Session, snippet: CustomizedSnippet, **_kwargs): + def update_persisted_snippet(*, session: Session, snippet: CustomizedSnippet, **_kwargs: object) -> WorkflowRecord: merged_snippet = session.merge(snippet) merged_snippet.description = "Updated in transaction" - return workflow + return workflow_record(workflow, session) update_workflow = Mock(side_effect=update_persisted_snippet) monkeypatch.setattr(snippet_workflow_module.SnippetService, "update_workflow", update_workflow) @@ -474,7 +508,7 @@ def test_update_published_snippet_workflow_raises_not_found( sqlite_session.add(snippet) sqlite_session.commit() - def update_missing_workflow(*, session: Session, snippet: CustomizedSnippet, **_kwargs): + def update_missing_workflow(*, session: Session, snippet: CustomizedSnippet, **_kwargs: object) -> None: merged_snippet = session.merge(snippet) merged_snippet.name = "Rolled back name" @@ -490,7 +524,7 @@ def update_missing_workflow(*, session: Session, snippet: CustomizedSnippet, **_ method="PATCH", json={"marked_name": "v1"}, ): - with pytest.raises(NotFound, match="Workflow not found"): + with pytest.raises(NotFoundError, match="Workflow not found"): handler( api, snippet_workflow_module.WorkflowUpdatePayload.model_validate({"marked_name": "v1"}), @@ -505,60 +539,33 @@ def update_missing_workflow(*, session: Session, snippet: CustomizedSnippet, **_ assert snippet.name == "Snippet" -def test_delete_published_snippet_workflow_succeeds(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - snippet = _snippet() - delete_workflow = Mock(return_value=True) - monkeypatch.setattr(snippet_workflow_module.SnippetService, "delete_workflow", delete_workflow) - - api = snippet_workflow_module.SnippetWorkflowByIdApi() - handler = unwrap(api.delete) - - with app.test_request_context("/snippets/snippet-1/workflows/workflow-1", method="DELETE"): - response, status_code = handler(api, snippet, workflow_id="workflow-1") - - assert status_code == 204 - assert response is None - delete_workflow.assert_called_once() - delete_call = delete_workflow.call_args.kwargs - assert isinstance(delete_call["session"], Session) - assert delete_call["snippet"] is snippet - assert delete_call["workflow_id"] == "workflow-1" - - -def test_delete_published_snippet_workflow_raises_not_found(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - def delete_missing_workflow(**_kwargs): - raise ValueError("Workflow with ID missing-workflow not found") - - monkeypatch.setattr( - snippet_workflow_module.SnippetService, "delete_workflow", Mock(side_effect=delete_missing_workflow) - ) - - api = snippet_workflow_module.SnippetWorkflowByIdApi() - handler = unwrap(api.delete) - - with app.test_request_context("/snippets/snippet-1/workflows/missing-workflow", method="DELETE"): - with pytest.raises(NotFound): - handler(api, _snippet(), workflow_id="missing-workflow") - - -def test_delete_published_snippet_workflow_raises_bad_request_when_in_use( - app: Flask, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize("outcome", ["success", "missing", "active", "draft"]) +def test_delete_published_snippet_workflow( + app: Flask, monkeypatch: pytest.MonkeyPatch, workflow_application: ApplicationServices, outcome: str ) -> None: - def delete_active_workflow(**_kwargs): - raise snippet_workflow_module.WorkflowInUseError("Cannot delete workflow that is currently in use") - - monkeypatch.setattr( - snippet_workflow_module.SnippetService, "delete_workflow", Mock(side_effect=delete_active_workflow) - ) - + calls: list[tuple[RequestContext, object, str]] = [] + context = RequestContext("test", None, "account-1", "tenant-1") + + def delete(context: RequestContext, owner: snippet_workflow_module.WorkflowOwner, workflow_id: str) -> None: + calls.append((context, owner, workflow_id)) + if outcome == "missing": + raise snippet_workflow_module.WorkflowNotFoundError("Workflow not found") + if outcome == "active": + raise snippet_workflow_module.WorkflowInUseError("Workflow is in use") + if outcome == "draft": + raise snippet_workflow_module.DraftWorkflowDeletionError("Cannot delete draft") + + monkeypatch.setattr(workflow_application.console_workflows, "delete", delete) api = snippet_workflow_module.SnippetWorkflowByIdApi() handler = unwrap(api.delete) - - with app.test_request_context("/snippets/snippet-1/workflows/workflow-1", method="DELETE"): - with pytest.raises(HTTPException) as exc_info: - handler(api, _snippet(), workflow_id="workflow-1") - - assert exc_info.value.code == 400 + with app.test_request_context("/", method="DELETE"): + if outcome == "success": + assert handler(api, context, "snippet-1", "workflow-1") == (None, 204) + else: + with pytest.raises(HTTPException) as error: + handler(api, context, "snippet-1", "workflow-1") + assert error.value.code == (404 if outcome == "missing" else 400) + assert calls == [(context, snippet_workflow_module.WorkflowOwner("snippet-1", "snippet"), "workflow-1")] def test_workflow_run_detail_raises_not_found_when_run_missing( @@ -571,7 +578,7 @@ def test_workflow_run_detail_raises_not_found_when_run_missing( handler = unwrap(api.get) with app.test_request_context("/snippets/snippet-1/workflow-runs/run-1"): - with pytest.raises(NotFound, match="Workflow run not found"): + with pytest.raises(NotFoundError, match="Workflow run not found"): handler(api, sqlite_session, snippet=snippet, run_id="run-1") @@ -587,7 +594,7 @@ def test_draft_node_last_run_raises_not_found_when_execution_missing( handler = unwrap(api.get) with app.test_request_context("/snippets/snippet-1/workflows/draft/nodes/llm-1/last-run"): - with pytest.raises(NotFound, match="Node last run not found"): + with pytest.raises(NotFoundError, match="Node last run not found"): handler(api, snippet=snippet, node_id="llm-1") diff --git a/api/tests/unit_tests/controllers/console/workspace/test_snippets.py b/api/tests/unit_tests/controllers/console/workspace/test_snippets.py index aae1f7a381ed59..f9eb83937af9bd 100644 --- a/api/tests/unit_tests/controllers/console/workspace/test_snippets.py +++ b/api/tests/unit_tests/controllers/console/workspace/test_snippets.py @@ -7,8 +7,8 @@ from flask import Flask from pydantic import ValidationError from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, NotFound +from controllers.common.errors import InvalidRequestError, NotFoundError from controllers.console.workspace import snippets as snippets_module from models.account import Account, TenantAccountRole from models.snippet import CustomizedSnippet @@ -173,10 +173,8 @@ def test_create_snippet_defaults_unknown_type_and_returns_created( assert create_snippet.call_args.kwargs["snippet_type"] == snippets_module.SnippetType.NODE -def test_create_snippet_rejects_forbidden_nodes(app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_create_snippet_rejects_forbidden_nodes(app: Flask, sqlite_session: Session): user = _account("account-1") - create_snippet = Mock() - monkeypatch.setattr(snippets_module.SnippetService, "create_snippet", create_snippet) req_data = snippets_module.CreateSnippetPayload( name="snippet with invalid node", @@ -210,7 +208,7 @@ def test_create_snippet_rejects_forbidden_nodes(app: Flask, monkeypatch: pytest. assert status_code == 400 assert "knowledge-retrieval" in response["message"] - create_snippet.assert_not_called() + assert sqlite_session.query(CustomizedSnippet).count() == 0 def test_get_snippet_detail_raises_when_missing(app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): @@ -220,7 +218,7 @@ def test_get_snippet_detail_raises_when_missing(app: Flask, monkeypatch: pytest. handler = unwrap(api.get) with app.test_request_context("/workspaces/current/customized-snippets/snippet-1"): - with pytest.raises(NotFound, match="Snippet not found"): + with pytest.raises(NotFoundError, match="Snippet not found"): handler(api, sqlite_session, "tenant-1", snippet_id="snippet-1") @@ -384,7 +382,7 @@ def patch_through_decorator(resource, session: Session, snippet_id: str): method="PATCH", json={"name": "New"}, ): - with pytest.raises(BadRequest, match="name already in use"): + with pytest.raises(InvalidRequestError, match="name already in use"): patch_through_decorator(api, snippet_id="snippet-1") assert _persisted_name(sqlite_session) == "Snippet" @@ -454,7 +452,7 @@ def test_export_snippet_returns_yaml_attachment(app: Flask, monkeypatch: pytest. monkeypatch.setattr(snippets_module.SnippetService, "get_snippet_by_id", Mock(return_value=snippet)) monkeypatch.setattr( snippets_module, - "SnippetDslService", + "build_snippet_dsl_service", Mock(return_value=SimpleNamespace(export_snippet_dsl=export_snippet_dsl)), ) @@ -482,7 +480,7 @@ def test_export_snippet_raises_not_found_for_missing_workflow( monkeypatch.setattr(snippets_module.SnippetService, "get_snippet_by_id", Mock(return_value=snippet)) monkeypatch.setattr( snippets_module, - "SnippetDslService", + "build_snippet_dsl_service", Mock( return_value=SimpleNamespace( export_snippet_dsl=Mock(side_effect=ValueError("Missing published workflow workflow-1")) @@ -493,7 +491,7 @@ def test_export_snippet_raises_not_found_for_missing_workflow( handler = unwrap(api.get) with app.test_request_context("/workspaces/current/customized-snippets/snippet-1/export?workflow_id=workflow-1"): - with pytest.raises(NotFound, match="Missing published workflow workflow-1"): + with pytest.raises(NotFoundError, match="Missing published workflow workflow-1"): handler(api, sqlite_session, "tenant-1", snippet_id="snippet-1") @@ -506,7 +504,7 @@ def test_import_snippet_returns_202_for_pending_confirmation( import_snippet = Mock(return_value=result) monkeypatch.setattr( snippets_module, - "SnippetDslService", + "build_snippet_dsl_service", Mock(return_value=SimpleNamespace(import_snippet=import_snippet)), ) @@ -536,7 +534,7 @@ def test_import_snippet_returns_400_for_failed_import( import_snippet = Mock(return_value=result) monkeypatch.setattr( snippets_module, - "SnippetDslService", + "build_snippet_dsl_service", Mock(return_value=SimpleNamespace(import_snippet=import_snippet)), ) @@ -565,7 +563,7 @@ def test_import_confirm_returns_200_for_completed_import( confirm_import = Mock(return_value=result) monkeypatch.setattr( snippets_module, - "SnippetDslService", + "build_snippet_dsl_service", Mock(return_value=SimpleNamespace(confirm_import=confirm_import)), ) @@ -592,7 +590,7 @@ def test_check_dependencies_raises_when_snippet_missing( handler = unwrap(api.get) with app.test_request_context("/workspaces/current/customized-snippets/snippet-1/check-dependencies"): - with pytest.raises(NotFound, match="Snippet not found"): + with pytest.raises(NotFoundError, match="Snippet not found"): handler(api, sqlite_session, "tenant-1", snippet_id="snippet-1") @@ -606,7 +604,7 @@ def test_check_dependencies_returns_dependency_result( monkeypatch.setattr(snippets_module.SnippetService, "get_snippet_by_id", Mock(return_value=snippet)) monkeypatch.setattr( snippets_module, - "SnippetDslService", + "build_snippet_dsl_service", Mock(return_value=SimpleNamespace(check_dependencies=check_dependencies)), ) @@ -633,7 +631,7 @@ def test_increment_use_count_raises_when_snippet_missing( "/workspaces/current/customized-snippets/snippet-1/use-count/increment", method="POST", ): - with pytest.raises(NotFound, match="Snippet not found"): + with pytest.raises(NotFoundError, match="Snippet not found"): handler(api, sqlite_session, "tenant-1", snippet_id="snippet-1") diff --git a/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py b/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py index c0e644e9c731e9..8007a070d9efdc 100644 --- a/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py +++ b/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py @@ -8,11 +8,12 @@ """ import inspect -from unittest.mock import MagicMock, patch +import json +from unittest.mock import MagicMock, Mock, patch import pytest from flask import Flask -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from controllers.inner_api.plugin import plugin as plugin_module from controllers.inner_api.plugin.plugin import ( @@ -33,8 +34,17 @@ PluginInvokeTTSApi, PluginUploadFileRequestApi, ) +from core.app.apps import base_app_queue_manager +from core.ops.ops_trace_manager import TraceQueueManager +from core.plugin.entities.request import RequestInvokeTool from core.workflow.file_reference import build_file_reference +from extensions.ext_application_services import ApplicationServices from models import Account, Tenant +from models.account import TenantAccountJoin, TenantAccountRole +from models.model import App, AppMode +from models.tools import WorkflowToolProvider +from models.workflow import Workflow, WorkflowType +from services.workflow.execution.adapters.workflow import app_generator as generator_module from tests.unit_tests.config_override import apply_config_overrides from tests.unit_tests.model_factories import make_tenant @@ -73,6 +83,92 @@ def _extract_raw_post(cls): return bottom +def test_plugin_invokes_persisted_workflow_tool_through_composed_runtime( + app: Flask, + sqlite_session: Session, + workflow_application: ApplicationServices, + monkeypatch: pytest.MonkeyPatch, +) -> None: + tenant, user = _tenant(), _user() + application = App( + id="nested-app", + tenant_id=tenant.id, + name="Nested", + description="", + mode=AppMode.WORKFLOW, + enable_site=False, + enable_api=True, + max_active_requests=None, + ) + workflow = Workflow.new( + tenant_id=tenant.id, + app_id=application.id, + type=WorkflowType.WORKFLOW.value, + version="1", + graph=json.dumps( + {"nodes": [{"id": "start", "data": {"type": "start", "title": "Start", "variables": []}}], "edges": []} + ), + features="{}", + created_by=user.id, + environment_variables=[], + conversation_variables=[], + rag_pipeline_variables=[], + ) + provider = WorkflowToolProvider( + name="nested", + label="Nested", + icon="icon.svg", + app_id=application.id, + version="1", + user_id=user.id, + tenant_id=tenant.id, + description="", + parameter_configuration="[]", + ) + sqlite_session.add_all( + [ + tenant, + user, + application, + workflow, + provider, + TenantAccountJoin(tenant_id=tenant.id, account_id=user.id, role=TenantAccountRole.OWNER), + ] + ) + sqlite_session.commit() + sqlite_session.close() + # Any fallback to the global database instead of the composed repositories must fail. + monkeypatch.setattr("core.db.session_factory._session_maker", sessionmaker()) + monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", lambda *_args, **_kwargs: None) + monkeypatch.setattr(generator_module, "TraceQueueManager", Mock(return_value=Mock(spec=TraceQueueManager))) + monkeypatch.setattr(generator_module.WorkflowAppGenerator, "_generate_worker", staticmethod(lambda **_kwargs: None)) + handled = [] + + def handle_response(self, **_kwargs): + assert self._runtime is workflow_application.workflow_runtime + handled.append("workflow response") + return {"data": {"outputs": {"result": "nested result"}}} + + monkeypatch.setattr(generator_module.WorkflowAppGenerator, "_handle_response", handle_response) + monkeypatch.setattr( + generator_module.WorkflowAppGenerateResponseConverter, "convert", lambda *, response, **_kwargs: response + ) + with app.test_request_context(): + response = _extract_raw_post(PluginInvokeToolApi)( + PluginInvokeToolApi(), + user_model=user, + tenant_model=tenant, + payload=RequestInvokeTool( + tool_type="workflow", provider=provider.id, tool=provider.name, tool_parameters={} + ), + ) + messages = [json.loads(frame[14:]) for frame in response.response] + assert handled == ["workflow response"] + assert messages + assert all(message["error"] == "" for message in messages) + assert any("nested result" in json.dumps(message["data"]) for message in messages) + + class TestPluginInvokeLLMApi: """Test PluginInvokeLLMApi endpoint structure""" diff --git a/api/tests/unit_tests/controllers/inner_api/test_agent_tools.py b/api/tests/unit_tests/controllers/inner_api/test_agent_tools.py index 186615f3d90853..fbbb324700d6f2 100644 --- a/api/tests/unit_tests/controllers/inner_api/test_agent_tools.py +++ b/api/tests/unit_tests/controllers/inner_api/test_agent_tools.py @@ -2,12 +2,16 @@ from collections.abc import Generator from contextlib import contextmanager -from unittest.mock import patch +from unittest.mock import Mock, create_autospec, patch +import pytest from flask import Flask from controllers.inner_api import bp as inner_api_bp -from services.entities.agent_tool_inner import AgentToolInvokeResponse +from extensions.application_services.agent import AgentAppServices +from extensions.ext_application_services import ApplicationServices +from services.agent.tool_invocation_service import AgentToolInnerService +from services.entities.agent_tool_inner import AgentToolInvokeRequest, AgentToolInvokeResponse from services.errors.agent_tool_inner import AgentToolInnerServiceError from tests.unit_tests.config_override import config_overrides_context @@ -54,6 +58,14 @@ def _agent_inner_auth() -> Generator[None]: yield +@contextmanager +def _tool_service() -> Generator[Mock]: + service = create_autospec(AgentToolInnerService, instance=True, spec_set=True) + services = Mock(spec=ApplicationServices, agent_apps=Mock(spec=AgentAppServices, tools=service)) + with patch("controllers.inner_api.agent.tools.application_services", return_value=services): + yield service.invoke + + def test_post_returns_service_response() -> None: app = Flask(__name__) app.config["TESTING"] = True @@ -61,7 +73,7 @@ def test_post_returns_service_response() -> None: with ( _agent_inner_auth(), - patch("controllers.inner_api.agent.tools.AgentToolInnerService.invoke") as mock_invoke, + _tool_service() as mock_invoke, ): mock_invoke.return_value = AgentToolInvokeResponse( messages=[{"type": "text", "message": {"text": "ok"}}], @@ -79,6 +91,7 @@ def test_post_returns_service_response() -> None: body = response.get_json() assert body["observation"] == "ok" assert body["metadata"]["provider_type"] == "plugin" + mock_invoke.assert_called_once_with(request=AgentToolInvokeRequest.model_validate(_payload())) def test_post_returns_400_for_invalid_body() -> None: @@ -105,7 +118,7 @@ def test_post_preserves_service_error_status_code_and_description() -> None: with ( _agent_inner_auth(), - patch("controllers.inner_api.agent.tools.AgentToolInnerService.invoke") as mock_invoke, + _tool_service() as mock_invoke, ): mock_invoke.side_effect = AgentToolInnerServiceError( error_code="app_tenant_mismatch", @@ -122,3 +135,14 @@ def test_post_preserves_service_error_status_code_and_description() -> None: body = response.get_json() assert body["code"] == "app_tenant_mismatch" assert body["message"] == "App does not belong to the caller tenant." + + +@pytest.mark.parametrize("api_key", [None, "incorrect-key"]) +def test_post_rejects_untrusted_callers_before_service_invocation(api_key: str | None) -> None: + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(inner_api_bp) + with _agent_inner_auth(), _tool_service() as invoke: + response = app.test_client().post("/inner/api/agent/tools/invoke", json=_payload(), headers=_headers(api_key)) + assert response.status_code == 404 + invoke.assert_not_called() diff --git a/api/tests/unit_tests/controllers/mcp/test_mcp.py b/api/tests/unit_tests/controllers/mcp/test_mcp.py index 0dd960de400267..90a9dcae53517f 100644 --- a/api/tests/unit_tests/controllers/mcp/test_mcp.py +++ b/api/tests/unit_tests/controllers/mcp/test_mcp.py @@ -696,7 +696,7 @@ def test_tools_list_json_unchanged_for_legacy_client(self, app): tool = response.get_json()["result"]["tools"][0] assert set(tool) == {"name", "description", "inputSchema"} - @patch("core.mcp.server.streamable_http.AppGenerateService") + @patch("controllers.mcp.mcp.AppGenerateService") def test_tools_call_json_includes_structured_content_for_modern_client(self, mock_app_generate, app): """A 2025-06-18 client receives structuredContent alongside the text content.""" api = self._make_api() @@ -713,7 +713,7 @@ def test_tools_call_json_includes_structured_content_for_modern_client(self, moc assert result["structuredContent"] == {"answer": "test answer"} assert result["content"][0]["text"] == "test answer" - @patch("core.mcp.server.streamable_http.AppGenerateService") + @patch("controllers.mcp.mcp.AppGenerateService") def test_tools_call_json_omits_structured_content_for_legacy_client(self, mock_app_generate, app): """A 2024-11-05 client receives the pre-upgrade tools/call JSON without structuredContent.""" api = self._make_api() @@ -729,3 +729,6 @@ def test_tools_call_json_omits_structured_content_for_legacy_client(self, mock_a result = response.get_json()["result"] assert "structuredContent" not in result assert result["content"][0]["text"] == "test answer" + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/controllers/openapi/test_app_run_rate_limit.py b/api/tests/unit_tests/controllers/openapi/test_app_run_rate_limit.py index 7d94bae5fab979..87ab8e05fbafba 100644 --- a/api/tests/unit_tests/controllers/openapi/test_app_run_rate_limit.py +++ b/api/tests/unit_tests/controllers/openapi/test_app_run_rate_limit.py @@ -5,8 +5,8 @@ """ import pytest -from werkzeug.exceptions import BadRequest, TooManyRequests +from controllers.common.errors import InvalidRequestError, TooManyRequestsError from controllers.openapi.app_run import _translate_service_errors from controllers.service_api.app.error import TriggerWorkflowServiceModeUnavailableError from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError @@ -19,7 +19,7 @@ def test_translate_maps_app_concurrency_to_too_many_requests(): # Regression guard: this used to fall through to a 500 (it was not caught here). - with pytest.raises(TooManyRequests) as exc: + with pytest.raises(TooManyRequestsError) as exc: with _translate_service_errors(): raise AppInvokeQuotaExceededError("internal: client_id=abc max=10") assert exc.value.code == 429 @@ -44,7 +44,7 @@ def test_translate_maps_trigger_workflow_to_stable_unavailable_error(): def test_translate_maps_value_error_to_a_fixed_bad_request(): # Regression guard: an unpublished workflow used to surface as a detail-less 500. # The service message names internals, so the client gets a fixed text. - with pytest.raises(BadRequest) as exc: + with pytest.raises(InvalidRequestError) as exc: with _translate_service_errors(): raise ValueError("variable 'secret_key' of node 42 is missing") assert exc.value.code == 400 diff --git a/api/tests/unit_tests/controllers/openapi/test_app_run_streaming.py b/api/tests/unit_tests/controllers/openapi/test_app_run_streaming.py index 9bccd650ade2d0..c3cabb89174b56 100644 --- a/api/tests/unit_tests/controllers/openapi/test_app_run_streaming.py +++ b/api/tests/unit_tests/controllers/openapi/test_app_run_streaming.py @@ -13,8 +13,8 @@ from flask import Flask from pydantic import BaseModel, ValidationError from werkzeug.datastructures import FileStorage -from werkzeug.exceptions import UnprocessableEntity +from controllers.common.errors import UnprocessableEntityError from controllers.openapi._models import ( AdvancedChatRunPayload, ChatRunPayload, @@ -153,7 +153,7 @@ def test_per_mode_route_refuses_an_app_of_another_mode(app: Flask, monkeypatch: api = ChatRunApi() body = ChatRunPayload(inputs={}, query="hi") with app.test_request_context(f"/openapi/v1/apps/{_TEST_APP_ID}/chat:run", method="POST"): - with pytest.raises(UnprocessableEntity, match="app_mode_mismatch"): + with pytest.raises(UnprocessableEntityError, match="app_mode_mismatch"): api.post.__handler__(api, _ctx(AppMode.WORKFLOW), app_id=_TEST_APP_ID, body=body) generate_mock.assert_not_called() @@ -222,3 +222,6 @@ def test_chat_route_passes_a_message_end_without_conversation_through(app: Flask with app.test_request_context(f"/openapi/v1/apps/{_TEST_APP_ID}/chat:run", method="POST"): response = api.post.__handler__(api, _ctx(AppMode.CHAT), app_id=_TEST_APP_ID, body=body) assert list(response.response) == [end] + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/controllers/service_api/app/test_completion.py b/api/tests/unit_tests/controllers/service_api/app/test_completion.py index 12f05078ba99b3..73259200e9a34e 100644 --- a/api/tests/unit_tests/controllers/service_api/app/test_completion.py +++ b/api/tests/unit_tests/controllers/service_api/app/test_completion.py @@ -22,9 +22,9 @@ from pydantic import ValidationError from sqlalchemy import Engine from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, NotFound import services +from controllers.common.errors import InvalidRequestError, NotFoundError from controllers.service_api.app.completion import ( ChatApi, ChatRequestPayload, @@ -42,7 +42,6 @@ WorkflowVersionExecutionNotAllowedError, ) from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError -from core.app.apps.agent_app.errors import AgentAppNotPublishedError from core.errors.error import QuotaExceededError from enums import CloudPlan, DeploymentEdition from graphon.model_runtime.errors.invoke import InvokeError @@ -50,6 +49,7 @@ from models.base import TypeBase from models.enums import ConversationFromSource, EndUserType from models.model import App, AppMode, Conversation, EndUser, IconType, Message +from services.app.generation.errors import AgentAppNotPublishedError from services.app_generate_service import AppGenerateService from services.app_task_service import AppTaskService from services.billing_service import BillingService @@ -57,6 +57,8 @@ from services.errors.app import IsDraftWorkflowError, WorkflowIdFormatError, WorkflowNotFoundError from services.errors.conversation import ConversationNotExistsError from services.errors.llm import InvokeRateLimitError +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.unit_tests.config_override import apply_config_overrides @@ -318,7 +320,14 @@ def test_generate_method_exists(self): assert callable(AppGenerateService.generate) @patch.object(AppGenerateService, "generate") - def test_generate_returns_response(self, mock_generate, orm_session: Session): + def test_generate_returns_response( + self, + mock_generate, + orm_session: Session, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): """Test that generate returns expected response format.""" expected = {"answer": "Hello!"} mock_generate.return_value = expected @@ -331,12 +340,21 @@ def test_generate_returns_response(self, mock_generate, orm_session: Session): invoke_from=Mock(), session=orm_session, streaming=False, + variables=workflow_variables, + runtime=workflow_runtime, ) assert result == expected @patch.object(AppGenerateService, "generate") - def test_generate_raises_conversation_not_exists(self, mock_generate, orm_session: Session): + def test_generate_raises_conversation_not_exists( + self, + mock_generate, + orm_session: Session, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): """Test generate raises ConversationNotExistsError.""" mock_generate.side_effect = services.errors.conversation.ConversationNotExistsError() app_model, end_user, _, _ = _persist_completion_state(orm_session, AppMode.COMPLETION) @@ -349,10 +367,19 @@ def test_generate_raises_conversation_not_exists(self, mock_generate, orm_sessio invoke_from=Mock(), session=orm_session, streaming=False, + variables=workflow_variables, + runtime=workflow_runtime, ) @patch.object(AppGenerateService, "generate") - def test_generate_raises_quota_exceeded(self, mock_generate, orm_session: Session): + def test_generate_raises_quota_exceeded( + self, + mock_generate, + orm_session: Session, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): """Test generate raises QuotaExceededError.""" mock_generate.side_effect = QuotaExceededError() app_model, end_user, _, _ = _persist_completion_state(orm_session, AppMode.COMPLETION) @@ -365,10 +392,19 @@ def test_generate_raises_quota_exceeded(self, mock_generate, orm_session: Sessio invoke_from=Mock(), session=orm_session, streaming=False, + variables=workflow_variables, + runtime=workflow_runtime, ) @patch.object(AppGenerateService, "generate") - def test_generate_raises_invoke_error(self, mock_generate, orm_session: Session): + def test_generate_raises_invoke_error( + self, + mock_generate, + orm_session: Session, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): """Test generate raises InvokeError.""" mock_generate.side_effect = InvokeError("Model invocation failed") app_model, end_user, _, _ = _persist_completion_state(orm_session, AppMode.COMPLETION) @@ -381,6 +417,8 @@ def test_generate_raises_invoke_error(self, mock_generate, orm_session: Session) invoke_from=Mock(), session=orm_session, streaming=False, + variables=workflow_variables, + runtime=workflow_runtime, ) @@ -526,7 +564,7 @@ def test_conversation_not_found(self, app: Flask, monkeypatch: pytest.MonkeyPatc handler = unwrap(api.post) with app.test_request_context("/completion-messages", method="POST", json={"inputs": {}}): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): handler(api, session=orm_session, app_model=app_model, end_user=end_user) @@ -687,7 +725,7 @@ def test_workflow_not_found(self, app: Flask, monkeypatch: pytest.MonkeyPatch, o app_model, end_user, _, _ = _persist_completion_state(orm_session, AppMode.CHAT) with app.test_request_context("/chat-messages", method="POST", json={"inputs": {}, "query": "hi"}): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): handler(api, session=orm_session, app_model=app_model, end_user=end_user) def test_draft_workflow(self, app: Flask, monkeypatch: pytest.MonkeyPatch, orm_session: Session) -> None: @@ -702,7 +740,7 @@ def test_draft_workflow(self, app: Flask, monkeypatch: pytest.MonkeyPatch, orm_s app_model, end_user, _, _ = _persist_completion_state(orm_session, AppMode.CHAT) with app.test_request_context("/chat-messages", method="POST", json={"inputs": {}, "query": "hi"}): - with pytest.raises(BadRequest): + with pytest.raises(InvalidRequestError): handler(api, session=orm_session, app_model=app_model, end_user=end_user) def test_agent_not_published_error_mapped( @@ -743,7 +781,7 @@ def test_invalid_conversation_id_fails_fast_as_not_found( method="POST", json={"inputs": {}, "query": "hi", "conversation_id": str(uuid.uuid4())}, ): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): handler(api, session=orm_session, app_model=app_model, end_user=end_user) # The lookup must run before generation, so the generator is never started. @@ -760,3 +798,6 @@ def test_wrong_mode(self, app: Flask, orm_session: Session) -> None: with app.test_request_context("/chat-messages/1/stop", method="POST"): with pytest.raises(NotChatAppError): handler(api, app_model=app_model, end_user=end_user, task_id="t1") + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/controllers/service_api/app/test_workflow.py b/api/tests/unit_tests/controllers/service_api/app/test_workflow.py index 1f8e56a348f2e2..79c89d68b13f1b 100644 --- a/api/tests/unit_tests/controllers/service_api/app/test_workflow.py +++ b/api/tests/unit_tests/controllers/service_api/app/test_workflow.py @@ -26,8 +26,8 @@ from flask import Flask from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, sessionmaker -from werkzeug.exceptions import BadRequest, NotFound +from controllers.common.errors import InvalidRequestError, NotFoundError from controllers.service_api.app.error import ( CompletionRequestError, NotWorkflowAppError, @@ -54,7 +54,7 @@ from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom from models.model import App, AppMode, EndUser from models.workflow import WorkflowAppLog, WorkflowAppLogCreatedFrom, WorkflowRun, WorkflowType -from repositories.workflow_app_log_query_repository import WorkflowAppLogQueryRepository +from repositories.workflow.app_log_repository import WorkflowAppLogRepository from services.app_generate_service import AppGenerateService from services.billing_service import BillingService from services.errors.app import ( @@ -65,7 +65,7 @@ TriggerWorkflowServiceModeUnavailableError as TriggerWorkflowServiceModeUnavailableServiceError, ) from services.errors.llm import InvokeRateLimitError -from services.workflow_app_log_query_service import WorkflowAppLogQueryService +from services.workflow.app_log_query_service import WorkflowAppLogQueryService def _default_workflow_inputs() -> dict[str, object]: @@ -682,7 +682,7 @@ def test_not_found( end_user = _make_end_user() with app.test_request_context("/workflows/1/run", method="POST", json={"inputs": {}}): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): handler(api, session=sqlite_session, app_model=app_model, end_user=end_user, workflow_id="w1") @pytest.mark.parametrize("sqlite_session", [()], indirect=True) @@ -707,7 +707,7 @@ def test_draft_workflow( end_user = _make_end_user() with app.test_request_context("/workflows/1/run", method="POST", json={"inputs": {}}): - with pytest.raises(BadRequest): + with pytest.raises(InvalidRequestError): handler(api, session=sqlite_session, app_model=app_model, end_user=end_user, workflow_id="w1") @@ -756,7 +756,7 @@ def test_success( _persist_workflow_log(sqlite_session, tenant_id=app_model.tenant_id, app_id=app_model.id) workflow_app_logs = WorkflowAppLogQueryService( - logs=WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory), + logs=WorkflowAppLogRepository(session_factory=sqlite_session_factory), ) monkeypatch.setattr( sys.modules["controllers.service_api.app.workflow"], @@ -868,3 +868,6 @@ def test_get_workflow_logs_success( result = unwrap(api.get)(api, app_model=workflow_app) assert result == _expected_workflow_log_pagination_payload() + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py b/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py index 4cb3d5ec648531..dcf08f1e70120c 100644 --- a/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py +++ b/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py @@ -27,11 +27,12 @@ from flask import Flask from sqlalchemy.orm import Session, scoped_session, sessionmaker from werkzeug.datastructures import FileStorage -from werkzeug.exceptions import Forbidden, NotFound from controllers.common.errors import ( + AccessDeniedError, FilenameNotExistsError, NoFileUploadedError, + NotFoundError, TooManyFilesError, ) from controllers.common.errors import ( @@ -46,16 +47,13 @@ KnowledgebasePipelineFileUploadApi, PipelineRunApi, ) -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom +from extensions.ext_application_services import ApplicationServices from extensions.storage.storage_type import StorageType from models.account import Account from models.dataset import Dataset, Pipeline from models.enums import CreatorUserRole from models.model import UploadFile -from repositories.credentials.query_repository import CredentialQueryRepository -from repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository -from services.data_source.provider_service import DatasourceProviderService from services.errors.file import FileTooLargeError as FileTooLargeServiceError from services.errors.file import UnsupportedFileTypeError from services.rag_pipeline.entity.pipeline_service_api_entities import ( @@ -63,6 +61,7 @@ PipelineRunApiEntity, ) from services.rag_pipeline.rag_pipeline import RagPipelineService +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator def _persist_dataset(session: Session, *, tenant_id: str, dataset_id: str) -> Dataset: @@ -87,18 +86,14 @@ def _persist_pipeline(session: Session, *, tenant_id: str) -> Pipeline: @pytest.fixture(autouse=True) -def _bind_database(sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch): +def _bind_database( + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + workflow_application: ApplicationServices, +): session_proxy = scoped_session(sqlite_session_factory) monkeypatch.setattr(workflow_module.db, "session", session_proxy) - query = CredentialQueryRepository(session_factory=sqlite_session_factory) - providers = DatasourceProviderService( - credentials=SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory) - ) - monkeypatch.setattr( - workflow_module, - "application_services", - lambda: SimpleNamespace(credential_queries=query, data_sources=SimpleNamespace(providers=providers)), - ) + monkeypatch.setattr(workflow_module, "application_services", lambda: workflow_application) yield session_proxy.remove() @@ -204,10 +199,6 @@ def test_get_pipeline_method_exists(self): """Test RagPipelineService.get_pipeline exists.""" assert hasattr(RagPipelineService, "get_pipeline") - def test_run_datasource_workflow_node_method_exists(self): - """Test RagPipelineService.run_datasource_workflow_node exists.""" - assert hasattr(RagPipelineService, "run_datasource_workflow_node") - def test_get_pipeline_templates_method_exists(self): """Test RagPipelineService.get_pipeline_templates exists.""" assert hasattr(RagPipelineService, "get_pipeline_templates") @@ -494,14 +485,14 @@ def test_get_plugins_parses_false_is_published_query(self, mock_svc_cls, app: Fl ) def test_get_plugins_not_found(self, app: Flask, sqlite_session: Session): - """Test NotFound when dataset check fails.""" + """Test NotFoundError when dataset check fails.""" tenant_id = str(uuid.uuid4()) dataset_id = str(uuid.uuid4()) _persist_dataset(sqlite_session, tenant_id="other-tenant", dataset_id=dataset_id) with app.test_request_context("/datasets/test/pipeline/datasource-plugins"): api = DatasourcePluginsApi() - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): api.get(tenant_id=tenant_id, dataset_id=dataset_id) @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.RagPipelineService") @@ -534,7 +525,7 @@ class TestDatasourceNodeRunApiPost: """ @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.helper") - @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.PipelineGenerator") + @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.convert_to_event_stream") @patch( "controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.current_user", new_callable=lambda: Account(name="Test Account", email="test@example.com"), @@ -559,41 +550,46 @@ def test_post_success( pipeline = _persist_pipeline(sqlite_session, tenant_id=tenant_id) mock_svc_instance = Mock() mock_svc_instance.get_pipeline.return_value = pipeline - mock_svc_instance.run_datasource_workflow_node.return_value = iter(["event1"]) mock_svc_cls.return_value = mock_svc_instance - mock_gen.convert_to_event_stream.return_value = iter(["stream_event"]) + mock_gen.return_value = iter(["stream_event"]) mock_helper.compact_generate_response.return_value = {"result": "ok"} - with app.test_request_context( - "/datasets/test/pipeline/datasource/nodes/node_abc/run", - method="POST", - json={ - "inputs": {"url": "https://example.com"}, - "datasource_type": "online_document", - "is_published": True, - }, + with ( + app.test_request_context( + "/datasets/test/pipeline/datasource/nodes/node_abc/run", + method="POST", + json={ + "inputs": {"url": "https://example.com"}, + "datasource_type": "online_document", + "is_published": True, + }, + ), + patch.object( + workflow_module.application_services().knowledge.pipeline_execution, + "run_datasource_workflow_node", + return_value=iter(["event1"]), + ) as run, ): api = DatasourceNodeRunApi() response = api.post(tenant_id=tenant_id, dataset_id=dataset_id, node_id=node_id) assert response == {"result": "ok"} mock_svc_instance.get_pipeline.assert_called_once_with(tenant_id=tenant_id, dataset_id=dataset_id) - mock_svc_instance.get_pipeline.assert_called_once_with(tenant_id=tenant_id, dataset_id=dataset_id) - mock_svc_instance.run_datasource_workflow_node.assert_called_once() + run.assert_called_once() def test_post_not_found(self, app: Flask, sqlite_session: Session): - """Test NotFound when dataset check fails.""" + """Test NotFoundError when dataset check fails.""" # `@model_validate` parses the body before the ownership guard, so a - # valid payload is required to reach the NotFound branch. + # valid payload is required to reach the NotFoundError branch. with app.test_request_context( "/datasets/test/pipeline/datasource/nodes/n1/run", method="POST", json={"inputs": {}, "datasource_type": "online_document", "is_published": True}, ): api = DatasourceNodeRunApi() - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): api.post(tenant_id=str(uuid.uuid4()), dataset_id=str(uuid.uuid4()), node_id="n1") @patch( @@ -669,13 +665,15 @@ def test_post_success_streaming( mock_svc_cls.assert_called_once_with(sqlite_session) mock_gen_svc.generate.assert_called_once() assert mock_gen_svc.generate.call_args.kwargs["generator"] is pipeline_application - assert mock_gen_svc.generate.call_args.kwargs["session"] is sqlite_session + assert mock_gen_svc.generate.call_args.kwargs["pipeline"] is pipeline + assert mock_gen_svc.generate.call_args.kwargs["invoke_from"] == InvokeFrom.PUBLISHED_PIPELINE + assert mock_gen_svc.generate.call_args.kwargs["streaming"] is True def test_post_not_found(self, app: Flask, sqlite_session: Session): - """Test NotFound when dataset check fails.""" + """Test NotFoundError when dataset check fails.""" with app.test_request_context("/datasets/test/pipeline/run", method="POST"): api = PipelineRunApi() - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): api.post.__wrapped__( api, sqlite_session, @@ -686,7 +684,7 @@ def test_post_not_found(self, app: Flask, sqlite_session: Session): @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.current_user", new="not_account") @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.service_api_ns") def test_post_forbidden_non_account_user(self, mock_ns, app: Flask, sqlite_session: Session): - """Test Forbidden when current_user is not an Account.""" + """Test AccessDeniedError when current_user is not an Account.""" tenant_id = str(uuid.uuid4()) dataset_id = str(uuid.uuid4()) _persist_dataset(sqlite_session, tenant_id=tenant_id, dataset_id=dataset_id) @@ -701,7 +699,7 @@ def test_post_forbidden_non_account_user(self, mock_ns, app: Flask, sqlite_sessi with app.test_request_context("/datasets/test/pipeline/run", method="POST"): api = PipelineRunApi() - with pytest.raises(Forbidden): + with pytest.raises(AccessDeniedError): api.post.__wrapped__( api, sqlite_session, diff --git a/api/tests/unit_tests/controllers/service_api/test_trace_session_id_parsing.py b/api/tests/unit_tests/controllers/service_api/test_trace_session_id_parsing.py index 05fcc5e400d4ef..b816330abd756f 100644 --- a/api/tests/unit_tests/controllers/service_api/test_trace_session_id_parsing.py +++ b/api/tests/unit_tests/controllers/service_api/test_trace_session_id_parsing.py @@ -179,3 +179,6 @@ def test_workflow_run_api_passes_body_trace_session_id( assert response == {"result": "ok"} _assert_generate_trace_session_id(mock_generate_service, "body-session") + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/controllers/trigger/test_trigger.py b/api/tests/unit_tests/controllers/trigger/test_trigger.py index c7079d332c1d28..9b0b2c4627b2c4 100644 --- a/api/tests/unit_tests/controllers/trigger/test_trigger.py +++ b/api/tests/unit_tests/controllers/trigger/test_trigger.py @@ -2,9 +2,9 @@ from unittest.mock import patch import pytest -from werkzeug.exceptions import NotFound import controllers.trigger.trigger as module +from controllers.common.errors import NotFoundError @pytest.fixture(autouse=True) @@ -23,7 +23,7 @@ def mock_jsonify(): class TestTriggerEndpoint: def test_invalid_uuid(self): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): module.trigger_endpoint(INVALID_UUID) @patch.object(module.TriggerService, "process_endpoint") diff --git a/api/tests/unit_tests/controllers/trigger/test_webhook.py b/api/tests/unit_tests/controllers/trigger/test_webhook.py index ac1cfaf1ebb8f0..ad6f55805fe624 100644 --- a/api/tests/unit_tests/controllers/trigger/test_webhook.py +++ b/api/tests/unit_tests/controllers/trigger/test_webhook.py @@ -1,9 +1,12 @@ from unittest.mock import patch import pytest -from werkzeug.exceptions import NotFound, RequestEntityTooLarge +from flask import Flask +from werkzeug.exceptions import RequestEntityTooLarge import controllers.trigger.webhook as module +from configs import dify_config +from controllers.common.errors import NotFoundError from models.trigger import WorkflowWebhookTrigger from models.workflow import Workflow from services.errors.app import QuotaExceededError @@ -57,6 +60,38 @@ def _workflow() -> Workflow: return make_workflow(workflow_id="workflow-1") +@pytest.mark.parametrize("path", ["webhook", "webhook-debug"]) +def test_webhook_body_limit_remains_http_413(monkeypatch, path): + app = Flask(__name__) + monkeypatch.setattr(dify_config, "WEBHOOK_REQUEST_BODY_MAX_SIZE", 8) + monkeypatch.setattr( + module.WebhookService, + "get_webhook_trigger_and_workflow", + lambda *_args, **_kwargs: (_webhook_trigger(), _workflow(), {}), + ) + app.register_blueprint(module.bp) + response = app.test_client().post(f"/triggers/{path}/wh-1", data="long request body") + assert response.status_code == 413 + assert b"Webhook request too large" in response.data + + +@pytest.mark.parametrize("path", ["webhook", "webhook-debug"]) +def test_framework_body_limit_is_not_translated_to_internal_error(monkeypatch, path): + app = Flask(__name__) + app.config["MAX_CONTENT_LENGTH"] = 8 + monkeypatch.setattr(dify_config, "WEBHOOK_REQUEST_BODY_MAX_SIZE", 1024) + monkeypatch.setattr( + module.WebhookService, + "get_webhook_trigger_and_workflow", + lambda *_args, **_kwargs: (_webhook_trigger(), _workflow(), {}), + ) + app.register_blueprint(module.bp) + + response = app.test_client().post(f"/triggers/{path}/wh-1", json={"message": "request too large"}) + + assert response.status_code == 413 + + class TestPrepareWebhookExecution: @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow") @patch.object(module.WebhookService, "extract_and_validate_webhook_data") @@ -171,7 +206,7 @@ def test_rate_limited(self, mock_get): @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=ValueError("missing")) def test_value_error_not_found(self, mock_get): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): module.handle_webhook("wh-1") @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=RequestEntityTooLarge()) @@ -181,10 +216,12 @@ def test_request_entity_too_large(self, mock_get): @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=Exception("boom")) def test_internal_error(self, mock_get): - response, status = module.handle_webhook("wh-1") + app = Flask(__name__) + app.register_blueprint(module.bp) + response = app.test_client().post("/triggers/webhook/wh-1") - assert status == 500 - assert response["error"] == "Internal server error" + assert response.status_code == 500 + assert response.json == {"error": "Internal server error", "message": "boom"} class TestHandleWebhookDebug: @@ -249,7 +286,7 @@ def test_debug_bad_request(self, mock_extract, mock_get): @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=ValueError("missing")) def test_debug_not_found(self, mock_get): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): module.handle_webhook_debug("wh-1") @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=RequestEntityTooLarge()) @@ -259,7 +296,9 @@ def test_debug_request_entity_too_large(self, mock_get): @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=Exception("boom")) def test_debug_internal_error(self, mock_get): - response, status = module.handle_webhook_debug("wh-1") + app = Flask(__name__) + app.register_blueprint(module.bp) + response = app.test_client().post("/triggers/webhook-debug/wh-1") - assert status == 500 - assert response["error"] == "Internal server error" + assert response.status_code == 500 + assert response.json == {"error": "Internal server error", "message": "An internal error has occurred."} diff --git a/api/tests/unit_tests/controllers/web/test_completion.py b/api/tests/unit_tests/controllers/web/test_completion.py index 73f2ce6d20c53e..dd1f6ce20e95f6 100644 --- a/api/tests/unit_tests/controllers/web/test_completion.py +++ b/api/tests/unit_tests/controllers/web/test_completion.py @@ -20,10 +20,10 @@ ProviderNotInitializeError, ProviderQuotaExceededError, ) -from core.app.apps.agent_app.errors import AgentAppNotPublishedError from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from graphon.model_runtime.errors.invoke import InvokeError from models.model import App, AppMode, EndUser +from services.app.generation.errors import AgentAppNotPublishedError from tests.unit_tests.model_factories import make_app, make_end_user @@ -201,3 +201,6 @@ def test_stop_success(self, mock_stop: MagicMock, app: Flask) -> None: assert status == 200 assert result == {"result": "success"} + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/controllers/web/test_message_endpoints.py b/api/tests/unit_tests/controllers/web/test_message_endpoints.py index 9fc9ad2fedc387..68a0c88fb29dc9 100644 --- a/api/tests/unit_tests/controllers/web/test_message_endpoints.py +++ b/api/tests/unit_tests/controllers/web/test_message_endpoints.py @@ -8,11 +8,12 @@ import pytest from flask import Flask -from werkzeug.exceptions import NotFound from controllers.common.controller_schemas import MessageFeedbackPayload +from controllers.common.errors import NotFoundError from controllers.web.error import ( AppMoreLikeThisDisabledError, + AppUnavailableError, NotChatAppError, NotCompletionAppError, ) @@ -24,9 +25,12 @@ ) from models.model import App, AppMode, EndUser from services.errors.app import MoreLikeThisDisabledError +from services.errors.app_model_config import AppModelConfigBrokenError from services.errors.message import MessageNotExistsError from tests.unit_tests.model_factories import make_end_user +pytestmark = pytest.mark.usefixtures("workflow_application") + def _chat_app() -> App: return App(id="app-1", tenant_id="tenant-1", mode=AppMode.CHAT) @@ -81,7 +85,7 @@ def test_feedback_message_not_found(self, mock_create: MagicMock, app: Flask) -> msg_id = uuid4() with app.test_request_context(f"/messages/{msg_id}/feedbacks", method="POST"): - with pytest.raises(NotFound, match="Message Not Exists"): + with pytest.raises(NotFoundError, match="Message Not Exists"): _feedback_post(MessageFeedbackApi(), payload, _chat_app(), _end_user(), msg_id) @@ -121,7 +125,7 @@ def test_message_not_found(self, mock_gen: MagicMock, app: Flask) -> None: query = MessageMoreLikeThisQuery.model_validate({"response_mode": "blocking"}) session = MagicMock() with app.test_request_context(f"/messages/{msg_id}/more-like-this?response_mode=blocking"): - with pytest.raises(NotFound, match="Message Not Exists"): + with pytest.raises(NotFoundError, match="Message Not Exists"): _more_like_this_get(MessageMoreLikeThisApi(), query, session, _completion_app(), _end_user(), msg_id) @patch( @@ -164,5 +168,24 @@ def test_happy_path(self, mock_suggest: MagicMock, app: Flask) -> None: def test_message_not_found(self, mock_suggest: MagicMock, app: Flask) -> None: msg_id = uuid4() with app.test_request_context(f"/messages/{msg_id}/suggested-questions"): - with pytest.raises(NotFound, match="Message not found"): + with pytest.raises(NotFoundError, match="Message not found"): MessageSuggestedQuestionApi().get(_chat_app(), _end_user(), msg_id) + + +def test_more_like_this_maps_missing_historical_configuration(app, sqlite_session, monkeypatch): + def unavailable(**_kwargs): + raise AppModelConfigBrokenError() + + monkeypatch.setattr("controllers.web.message.AppGenerateService.generate_more_like_this", unavailable) + with app.test_request_context("/messages/test/more-like-this?response_mode=blocking"): + with pytest.raises(AppUnavailableError) as raised: + _more_like_this_get( + MessageMoreLikeThisApi(), + MessageMoreLikeThisQuery(response_mode="blocking"), + sqlite_session, + _completion_app(), + _end_user(), + uuid4(), + ) + assert raised.value.code == 400 + assert raised.value.error_code == "app_unavailable" diff --git a/api/tests/unit_tests/controllers/web/test_workflow.py b/api/tests/unit_tests/controllers/web/test_workflow.py index 49c6ad74c6b324..f7bb570d56bc81 100644 --- a/api/tests/unit_tests/controllers/web/test_workflow.py +++ b/api/tests/unit_tests/controllers/web/test_workflow.py @@ -118,3 +118,6 @@ def test_stop_calls_both_mechanisms(self, mock_legacy: MagicMock, mock_graph: Ma assert result == {"result": "success"} mock_legacy.assert_called_once_with("task-1") mock_graph.assert_called_once_with("task-1") + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_resolve_agent.py b/api/tests/unit_tests/core/app/apps/agent_app/test_resolve_agent.py deleted file mode 100644 index 9ed6f108680049..00000000000000 --- a/api/tests/unit_tests/core/app/apps/agent_app/test_resolve_agent.py +++ /dev/null @@ -1,537 +0,0 @@ -"""Unit tests for AgentAppGenerator agent/snapshot resolution. - -Covers the DB-backed resolution helpers (the bound roster Agent + its published -Agent Soul snapshot) including every not-found error path, using persisted rows -in the shared SQLite test database. -""" - -from __future__ import annotations - -from unittest.mock import MagicMock - -import pytest -from sqlalchemy import event, select -from sqlalchemy.orm import Session - -from core.app.apps.agent_app import app_generator -from core.app.apps.agent_app.app_generator import AgentAppGenerator, AgentAppGeneratorError, AgentAppNotPublishedError -from core.app.entities.app_invoke_entities import InvokeFrom -from models.account import Account -from models.agent import ( - Agent, - AgentConfigDraft, - AgentConfigDraftType, - AgentConfigSnapshot, - AgentConfigVersionKind, - AgentScope, - AgentSource, - AgentWorkingResourceStatus, - AgentWorkspaceBinding, -) -from models.agent_config_entities import AgentSoulConfig -from models.model import App, AppMode, Conversation -from services.agent.workspace_service import ( - AgentWorkspaceBindingGenerationMismatchError, - AgentWorkspaceService, -) -from tests.unit_tests.model_factories import make_account, make_app, make_conversation - -_SOUL_DICT = { - "model": { - "plugin_id": "langgenius/openai", - "model_provider": "langgenius/openai/openai", - "model": "gpt-4o-mini", - }, - "prompt": {"system_prompt": "You are Iris."}, -} - - -def _agent( - *, - agent_id: str = "agent-1", - scope: AgentScope = AgentScope.ROSTER, - source: AgentSource = AgentSource.AGENT_APP, - active_snapshot_id: str | None = "snap-1", - published: bool = True, -) -> Agent: - return Agent( - id=agent_id, - tenant_id="t1", - name=f"Agent {agent_id}", - scope=scope, - source=source, - app_id="app-1" if scope == AgentScope.ROSTER else None, - backing_app_id="app-1" if scope == AgentScope.WORKFLOW_ONLY else None, - active_config_snapshot_id=active_snapshot_id, - active_config_has_model=True, - active_config_is_published=published, - created_by="creator-1", - updated_by="updater-1", - ) - - -def _snapshot( - *, snapshot_id: str = "snap-1", agent_id: str = "agent-1", home_snapshot_id: str = "home-1" -) -> AgentConfigSnapshot: - return AgentConfigSnapshot( - id=snapshot_id, - tenant_id="t1", - agent_id=agent_id, - version=1, - config_snapshot=AgentSoulConfig.model_validate(_SOUL_DICT), - home_snapshot_id=home_snapshot_id, - created_by="creator-1", - ) - - -def _app() -> App: - return make_app( - tenant_id="t1", - name="Agent App", - mode=AppMode.AGENT_CHAT, - icon_type=None, - enable_site=False, - enable_api=False, - ) - - -def _account() -> Account: - return make_account(account_id="user-1", name="Agent User", email="agent-user@example.com") - - -def _conversation(*, binding_id: str) -> Conversation: - return make_conversation(agent_workspace_binding_id=binding_id) - - -def _binding( - *, - home_snapshot_id: str = "home-1", - version_id: str = "snap-1", - version_kind: AgentConfigVersionKind = AgentConfigVersionKind.SNAPSHOT, -) -> AgentWorkspaceBinding: - return AgentWorkspaceBinding( - id="binding-1", - tenant_id="t1", - app_id="app-1", - workspace_id="workspace-1", - agent_id="agent-1", - base_home_snapshot_id=home_snapshot_id, - agent_config_version_id=version_id, - agent_config_version_kind=version_kind, - backend_binding_ref="backend-binding-1", - status=AgentWorkingResourceStatus.ACTIVE, - ) - - -def _persist(session: Session, *models: object) -> None: - session.add_all(models) - session.commit() - - -class TestResolveAgentById: - def test_success_returns_agent_snapshot_soul(self, sqlite_session: Session): - agent = _agent() - snapshot = _snapshot() - sqlite_session.add_all([agent, snapshot]) - sqlite_session.commit() - - resolved_agent, resolved_snapshot, soul = AgentAppGenerator._resolve_agent_by_id( - tenant_id="t1", agent_id="agent-1", snapshot_id="snap-1", session=sqlite_session - ) - - assert resolved_agent is agent - assert resolved_snapshot is snapshot - assert soul.prompt.system_prompt == "You are Iris." - assert soul.model is not None - assert soul.model.model == "gpt-4o-mini" - - def test_agent_missing_raises(self, sqlite_session: Session): - with pytest.raises(AgentAppGeneratorError, match="Agent not found"): - AgentAppGenerator._resolve_agent_by_id( - tenant_id="t1", agent_id="x", snapshot_id="snap-1", session=sqlite_session - ) - - def test_no_published_version_raises(self, sqlite_session: Session): - sqlite_session.add(_agent(active_snapshot_id=None)) - sqlite_session.commit() - with pytest.raises(AgentAppGeneratorError, match="no published version"): - AgentAppGenerator._resolve_agent_by_id( - tenant_id="t1", agent_id="agent-1", snapshot_id=None, session=sqlite_session - ) - - def test_snapshot_missing_raises(self, sqlite_session: Session): - sqlite_session.add(_agent()) - sqlite_session.commit() - with pytest.raises(AgentAppGeneratorError, match="published version not found"): - AgentAppGenerator._resolve_agent_by_id( - tenant_id="t1", - agent_id="agent-1", - snapshot_id="snap-1", - session=sqlite_session, - ) - - -class TestResolveDebugDraft: - def test_missing_shared_draft_is_created_with_supplied_session(self, sqlite_session: Session): - agent = _agent() - sqlite_session.add_all([agent, _snapshot()]) - sqlite_session.commit() - flush_count = 0 - - def _record_flush(_session: Session, _context: object) -> None: - nonlocal flush_count - flush_count += 1 - - event.listen(sqlite_session, "after_flush", _record_flush) - - draft = AgentAppGenerator._resolve_debug_draft( - tenant_id="t1", - agent=agent, - draft_type=None, - draft_id=None, - account_id=None, - session=sqlite_session, - ) - - assert draft.draft_type == AgentConfigDraftType.DRAFT - assert draft.base_snapshot_id == "snap-1" - assert sqlite_session.scalar(select(AgentConfigDraft).where(AgentConfigDraft.id == draft.id)) is draft - assert flush_count == 1 - - def test_stale_workflow_only_shared_draft_is_rebased_to_active_snapshot(self, sqlite_session: Session): - agent = _agent(scope=AgentScope.WORKFLOW_ONLY, active_snapshot_id="snap-2") - draft = AgentConfigDraft( - id="draft-1", - tenant_id="t1", - agent_id="agent-1", - draft_type=AgentConfigDraftType.DRAFT, - account_id=None, - draft_owner_key="", - base_snapshot_id="snap-1", - home_snapshot_id="home-1", - config_snapshot=AgentSoulConfig.model_validate({"prompt": {"system_prompt": "old"}}), - ) - active_snapshot = AgentConfigSnapshot( - id="snap-2", - tenant_id="t1", - agent_id="agent-1", - version=2, - config_snapshot=AgentSoulConfig.model_validate({"prompt": {"system_prompt": "new"}}), - home_snapshot_id="home-2", - created_by="creator-1", - ) - sqlite_session.add_all([agent, draft, active_snapshot]) - sqlite_session.commit() - flush_count = 0 - - def _record_flush(_session: Session, _context: object) -> None: - nonlocal flush_count - flush_count += 1 - - event.listen(sqlite_session, "after_flush", _record_flush) - - resolved = AgentAppGenerator._resolve_debug_draft( - tenant_id="t1", - agent=agent, - draft_type=None, - account_id=None, - session=sqlite_session, - ) - - assert resolved is draft - assert resolved.id == "draft-1" - assert resolved.base_snapshot_id == "snap-2" - assert resolved.home_snapshot_id == "home-2" - assert resolved.config_snapshot_dict["prompt"]["system_prompt"] == "new" - assert flush_count == 1 - - def test_build_draft_is_not_rebased_to_active_snapshot(self, sqlite_session: Session): - agent = _agent(scope=AgentScope.WORKFLOW_ONLY, active_snapshot_id="snap-2") - draft = AgentConfigDraft( - id="build-draft-1", - tenant_id="t1", - agent_id="agent-1", - draft_type=AgentConfigDraftType.DEBUG_BUILD, - account_id="account-1", - draft_owner_key="account-1", - base_snapshot_id="snap-1", - home_snapshot_id="home-build", - config_snapshot=AgentSoulConfig.model_validate({"prompt": {"system_prompt": "build edit"}}), - ) - sqlite_session.add_all([agent, draft]) - sqlite_session.commit() - - resolved = AgentAppGenerator._resolve_debug_draft( - tenant_id="t1", - agent=agent, - draft_type=AgentConfigDraftType.DEBUG_BUILD.value, - account_id="account-1", - session=sqlite_session, - ) - - assert resolved is draft - assert resolved.base_snapshot_id == "snap-1" - assert resolved.config_snapshot_dict["prompt"]["system_prompt"] == "build edit" - assert not sqlite_session.dirty - - def test_build_draft_uses_exact_draft_id(self, sqlite_session: Session): - agent = _agent(scope=AgentScope.WORKFLOW_ONLY, active_snapshot_id="snap-2") - draft = AgentConfigDraft( - id="exact-build-draft", - tenant_id="t1", - agent_id="agent-1", - draft_type=AgentConfigDraftType.DEBUG_BUILD, - account_id="account-1", - draft_owner_key="account-1", - base_snapshot_id="snap-1", - home_snapshot_id="home-build", - config_snapshot=AgentSoulConfig(), - ) - newer_decoy = AgentConfigDraft( - id="newer-build-draft", - tenant_id="t1", - agent_id="agent-1", - draft_type=AgentConfigDraftType.DEBUG_BUILD, - account_id="account-1", - draft_owner_key="account-1-decoy", - base_snapshot_id="snap-2", - home_snapshot_id="home-decoy", - config_snapshot=AgentSoulConfig(prompt={"system_prompt": "decoy"}), - ) - sqlite_session.add_all([agent, draft, newer_decoy]) - sqlite_session.commit() - - resolved = AgentAppGenerator._resolve_debug_draft( - tenant_id="t1", - agent=agent, - draft_type=AgentConfigDraftType.DEBUG_BUILD.value, - draft_id="exact-build-draft", - account_id="account-1", - session=sqlite_session, - ) - - assert resolved is draft - assert resolved.id == "exact-build-draft" - - -class TestResolveAgent: - @pytest.fixture(autouse=True) - def _publish_visibility(self, monkeypatch: pytest.MonkeyPatch) -> None: - def is_publish_visible(*, agent: Agent, **_kwargs: object) -> bool: - return agent.active_config_is_published - - monkeypatch.setattr( - app_generator, - "agent_has_workflow_callable_active_snapshot", - is_publish_visible, - ) - - def test_success_chains_to_resolve_by_id(self, sqlite_session: Session): - bound_agent = _agent() - snapshot = _snapshot() - _persist(sqlite_session, bound_agent, snapshot) - app_model = _app() - - agent, config_id, config_version_kind, soul = AgentAppGenerator()._resolve_agent( - app_model, - invoke_from=InvokeFrom.WEB_APP, - draft_type=None, - user=_account(), - session=sqlite_session, - ) - - assert agent is bound_agent - assert config_id == snapshot.id - assert config_version_kind == "snapshot" - assert soul.model is not None - - def test_unpublished_draft_still_resolves_active_snapshot( - self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session - ): - bound_agent = _agent(published=False) - snapshot = _snapshot() - _persist(sqlite_session, bound_agent, snapshot) - app_model = _app() - monkeypatch.setattr(app_generator, "agent_has_workflow_callable_active_snapshot", lambda **_kwargs: True) - - agent, config_id, config_version_kind, soul = AgentAppGenerator()._resolve_agent( - app_model, - invoke_from=InvokeFrom.WEB_APP, - draft_type=None, - user=_account(), - session=sqlite_session, - ) - - assert agent is bound_agent - assert config_id == snapshot.id - assert config_version_kind == "snapshot" - assert soul.prompt.system_prompt == "You are Iris." - - def test_existing_conversation_resolves_binding_snapshot_instead_of_latest_active_snapshot( - self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session - ): - bound_agent = _agent(active_snapshot_id="snap-2") - pinned_snapshot = _snapshot() - _persist(sqlite_session, bound_agent, pinned_snapshot) - conversation = _conversation(binding_id="binding-1") - binding = _binding() - get_active_binding = MagicMock(return_value=binding) - validate_generation = MagicMock() - monkeypatch.setattr(AgentWorkspaceService, "get_active_binding", get_active_binding) - monkeypatch.setattr(AgentWorkspaceService, "validate_binding_generation", validate_generation) - app_model = _app() - - _, config_id, config_version_kind, soul = AgentAppGenerator()._resolve_agent( - app_model, - invoke_from=InvokeFrom.WEB_APP, - draft_type=None, - user=_account(), - session=sqlite_session, - conversation=conversation, - ) - - assert config_id == "snap-1" - assert config_version_kind == "snapshot" - assert soul.prompt.system_prompt == "You are Iris." - assert get_active_binding.call_args.kwargs["binding_id"] == "binding-1" - validate_generation.assert_called_once_with( - binding, - base_home_snapshot_id="home-1", - agent_config_version_id="snap-1", - agent_config_version_kind=AgentConfigVersionKind.SNAPSHOT, - ) - - def test_existing_conversation_rejects_unavailable_binding( - self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session - ): - bound_agent = _agent(active_snapshot_id="snap-active") - _persist(sqlite_session, bound_agent) - conversation = _conversation(binding_id="binding-missing") - monkeypatch.setattr(AgentWorkspaceService, "get_active_binding", MagicMock(return_value=None)) - - with pytest.raises(AgentAppGeneratorError, match="Conversation participant Binding is unavailable"): - AgentAppGenerator()._resolve_agent( - _app(), - invoke_from=InvokeFrom.WEB_APP, - draft_type=None, - user=_account(), - session=sqlite_session, - conversation=conversation, - ) - - @pytest.mark.parametrize( - ("binding_home_id", "binding_version_kind", "snapshot_home_id"), - [ - ("home-binding", AgentConfigVersionKind.SNAPSHOT, "home-other"), - ("home-pinned", AgentConfigVersionKind.DRAFT, "home-pinned"), - ], - ) - def test_existing_conversation_generation_mismatch_does_not_fallback_to_active_snapshot( - self, - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, - binding_home_id: str, - binding_version_kind: AgentConfigVersionKind, - snapshot_home_id: str, - ): - bound_agent = _agent(active_snapshot_id="snap-active") - conversation = _conversation(binding_id="binding-1") - binding = _binding( - home_snapshot_id=binding_home_id, - version_id="snap-pinned", - version_kind=binding_version_kind, - ) - pinned_snapshot = _snapshot(snapshot_id="snap-pinned", home_snapshot_id=snapshot_home_id) - _persist(sqlite_session, bound_agent, pinned_snapshot) - monkeypatch.setattr(AgentWorkspaceService, "get_active_binding", MagicMock(return_value=binding)) - - with pytest.raises(AgentWorkspaceBindingGenerationMismatchError): - AgentAppGenerator()._resolve_agent( - _app(), - invoke_from=InvokeFrom.WEB_APP, - draft_type=None, - user=_account(), - session=sqlite_session, - conversation=conversation, - ) - - def test_unpublished_imported_agent_is_not_available_to_public_runtime(self, sqlite_session: Session): - bound_agent = _agent(source=AgentSource.IMPORTED, published=False) - _persist(sqlite_session, bound_agent) - app_model = _app() - - with pytest.raises(AgentAppNotPublishedError, match="not been published"): - AgentAppGenerator()._resolve_agent( - app_model, - invoke_from=InvokeFrom.WEB_APP, - draft_type=None, - user=_account(), - session=sqlite_session, - ) - - def test_never_published_agent_app_is_not_available_to_public_runtime(self, sqlite_session: Session): - bound_agent = _agent(published=False) - _persist(sqlite_session, bound_agent) - - with pytest.raises(AgentAppNotPublishedError, match="not been published"): - AgentAppGenerator()._resolve_agent( - _app(), - invoke_from=InvokeFrom.WEB_APP, - draft_type=None, - user=_account(), - session=sqlite_session, - ) - - def test_unpublished_imported_agent_remains_available_to_debugger(self, sqlite_session: Session): - bound_agent = _agent(source=AgentSource.IMPORTED, published=False) - draft = AgentConfigDraft( - id="draft-1", - tenant_id="t1", - agent_id="agent-1", - draft_type=AgentConfigDraftType.DRAFT, - account_id=None, - draft_owner_key="", - base_snapshot_id="snap-1", - home_snapshot_id="home-1", - config_snapshot=AgentSoulConfig.model_validate(_SOUL_DICT), - ) - _persist(sqlite_session, bound_agent, draft) - app_model = _app() - - agent, config_id, config_version_kind, soul = AgentAppGenerator()._resolve_agent( - app_model, - invoke_from=InvokeFrom.DEBUGGER, - draft_type=None, - user=_account(), - session=sqlite_session, - ) - - assert agent is bound_agent - assert config_id == draft.id - assert config_version_kind == "draft" - assert soul.prompt.system_prompt == "You are Iris." - - def test_agent_without_active_snapshot_raises_before_model_resolution(self, sqlite_session: Session): - bound_agent = _agent(active_snapshot_id=None, published=False) - _persist(sqlite_session, bound_agent) - app_model = _app() - - with pytest.raises(AgentAppNotPublishedError, match="not been published"): - AgentAppGenerator()._resolve_agent( - app_model, - invoke_from=InvokeFrom.WEB_APP, - draft_type=None, - user=_account(), - session=sqlite_session, - ) - - def test_unbound_app_raises(self, sqlite_session: Session): - app_model = _app() - with pytest.raises(AgentAppGeneratorError, match="has no bound Agent"): - AgentAppGenerator()._resolve_agent( - app_model, - invoke_from=InvokeFrom.WEB_APP, - draft_type=None, - user=_account(), - session=sqlite_session, - ) diff --git a/api/tests/unit_tests/core/app/apps/workflow/test_errors.py b/api/tests/unit_tests/core/app/apps/workflow/test_errors.py deleted file mode 100644 index 7461e06833259c..00000000000000 --- a/api/tests/unit_tests/core/app/apps/workflow/test_errors.py +++ /dev/null @@ -1,9 +0,0 @@ -from core.app.apps.workflow.errors import WorkflowPausedInBlockingModeError - - -class TestWorkflowErrors: - def test_workflow_paused_in_blocking_mode_error_attributes(self): - err = WorkflowPausedInBlockingModeError() - assert err.error_code == "workflow_paused_in_blocking_mode" - assert err.code == 400 - assert "blocking response mode" in err.description diff --git a/api/tests/unit_tests/core/app/features/test_annotation_reply.py b/api/tests/unit_tests/core/app/features/test_annotation_reply.py deleted file mode 100644 index a9001c96c614c5..00000000000000 --- a/api/tests/unit_tests/core/app/features/test_annotation_reply.py +++ /dev/null @@ -1,185 +0,0 @@ -import logging -from types import SimpleNamespace -from unittest.mock import Mock, patch - -import pytest -from sqlalchemy import select -from sqlalchemy.orm import Session - -from core.app.entities.app_invoke_entities import InvokeFrom -from core.app.features.annotation_reply.annotation_reply import AnnotationReplyFeature -from models.dataset import DatasetCollectionBinding -from models.enums import CollectionBindingType, ConversationFromSource -from models.model import App, AppAnnotationHitHistory, AppAnnotationSetting, Message, MessageAnnotation -from tests.unit_tests.model_factories import make_app, make_message - -TABLES = (AppAnnotationSetting, DatasetCollectionBinding, MessageAnnotation, AppAnnotationHitHistory) - - -def _app() -> App: - return make_app(icon_type=None, max_active_requests=0) - - -def _message() -> Message: - return make_message(message_id="msg-1", app_id=None, conversation_id=None) - - -def _persist_binding(session: Session) -> DatasetCollectionBinding: - binding = DatasetCollectionBinding( - provider_name="prov", - model_name="model", - type=CollectionBindingType.ANNOTATION, - collection_name="annotation-collection", - ) - session.add(binding) - session.flush() - return binding - - -def _persist_setting( - session: Session, - *, - app_id: str = "app-1", - collection_binding_id: str, - score_threshold: float = 0.5, -) -> AppAnnotationSetting: - setting = AppAnnotationSetting( - app_id=app_id, - score_threshold=score_threshold, - collection_binding_id=collection_binding_id, - created_user_id="user-1", - updated_user_id="user-1", - ) - session.add(setting) - session.flush() - return setting - - -def _persist_annotation(session: Session) -> MessageAnnotation: - annotation = MessageAnnotation( - app_id="app-1", - question="question", - content="content", - account_id="acct-1", - ) - session.add(annotation) - session.flush() - return annotation - - -@pytest.mark.parametrize("sqlite_session", [TABLES], indirect=True) -class TestAnnotationReplyFeature: - def test_query_returns_none_when_setting_missing(self, sqlite_session: Session): - binding = _persist_binding(sqlite_session) - _persist_setting(sqlite_session, app_id="other-app", collection_binding_id=binding.id) - - result = AnnotationReplyFeature().query( - app_record=_app(), - message=_message(), - query="hi", - user_id="user-1", - invoke_from=InvokeFrom.SERVICE_API, - session=sqlite_session, - ) - - assert result is None - - def test_query_returns_none_when_binding_missing(self, sqlite_session: Session): - _persist_setting(sqlite_session, collection_binding_id="missing-binding") - - result = AnnotationReplyFeature().query( - app_record=_app(), - message=_message(), - query="hi", - user_id="user-1", - invoke_from=InvokeFrom.SERVICE_API, - session=sqlite_session, - ) - - assert result is None - - def test_query_returns_annotation_and_persists_history_for_api(self, sqlite_session: Session): - binding = _persist_binding(sqlite_session) - _persist_setting(sqlite_session, collection_binding_id=binding.id, score_threshold=0) - annotation = _persist_annotation(sqlite_session) - document = SimpleNamespace(metadata={"annotation_id": annotation.id, "score": 0.8}) - vector_instance = Mock() - vector_instance.search_by_vector.return_value = [document] - - with patch( - "core.app.features.annotation_reply.annotation_reply.Vector", return_value=vector_instance - ) as vector_cls: - result = AnnotationReplyFeature().query( - app_record=_app(), - message=_message(), - query="hi", - user_id="user-1", - invoke_from=InvokeFrom.SERVICE_API, - session=sqlite_session, - ) - - assert result is annotation - vector_instance.search_by_vector.assert_called_once_with( - query="hi", top_k=1, score_threshold=1, filter={"group_id": ["app-1"]} - ) - assert vector_cls.call_args.kwargs["session"] is sqlite_session - sqlite_session.refresh(annotation) - assert annotation.hit_count == 1 - history = sqlite_session.scalar(select(AppAnnotationHitHistory)) - assert history is not None - assert history.annotation_id == annotation.id - assert history.app_id == "app-1" - assert history.message_id == "msg-1" - assert history.account_id == "user-1" - assert history.source == ConversationFromSource.API - assert history.score == 0.8 - - def test_query_returns_annotation_and_persists_history_for_console(self, sqlite_session: Session): - binding = _persist_binding(sqlite_session) - _persist_setting(sqlite_session, collection_binding_id=binding.id) - annotation = _persist_annotation(sqlite_session) - document = SimpleNamespace(metadata={"annotation_id": annotation.id, "score": 0.6}) - vector_instance = Mock() - vector_instance.search_by_vector.return_value = [document] - - with patch("core.app.features.annotation_reply.annotation_reply.Vector", return_value=vector_instance): - result = AnnotationReplyFeature().query( - app_record=_app(), - message=_message(), - query="hi", - user_id="user-1", - invoke_from=InvokeFrom.EXPLORE, - session=sqlite_session, - ) - - assert result is annotation - history = sqlite_session.scalar(select(AppAnnotationHitHistory)) - assert history is not None - assert history.source == ConversationFromSource.CONSOLE - - def test_query_logs_and_returns_none_on_exception(self, sqlite_session: Session, caplog: pytest.LogCaptureFixture): - binding = _persist_binding(sqlite_session) - _persist_setting(sqlite_session, collection_binding_id=binding.id) - vector_instance = Mock() - vector_instance.search_by_vector.side_effect = RuntimeError("boom") - - with ( - patch( - "core.app.features.annotation_reply.annotation_reply.Vector", - return_value=vector_instance, - ), - caplog.at_level(logging.WARNING), - ): - result = AnnotationReplyFeature().query( - app_record=_app(), - message=_message(), - query="hi", - user_id="user-1", - invoke_from=InvokeFrom.SERVICE_API, - session=sqlite_session, - ) - - assert result is None - assert "Query annotation failed" in caplog.text - assert sqlite_session.scalar(select(AppAnnotationHitHistory)) is None - assert sqlite_session.is_active diff --git a/api/tests/unit_tests/core/app/layers/test_conversation_variable_persist_layer.py b/api/tests/unit_tests/core/app/layers/test_conversation_variable_persist_layer.py deleted file mode 100644 index ba55e8f6950776..00000000000000 --- a/api/tests/unit_tests/core/app/layers/test_conversation_variable_persist_layer.py +++ /dev/null @@ -1,114 +0,0 @@ -from collections.abc import Sequence -from unittest.mock import Mock - -from core.app.layers.conversation_variable_persist_layer import ConversationVariablePersistenceLayer -from core.workflow.system_variables import SystemVariableKey -from core.workflow.variable_prefixes import CONVERSATION_VARIABLE_NODE_ID -from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatus -from graphon.graph_engine.command_channels import CommandChannel -from graphon.graph_events import NodeRunSucceededEvent, NodeRunVariableUpdatedEvent -from graphon.node_events import NodeRunResult -from graphon.runtime import ReadOnlyGraphRuntimeState -from graphon.variables import StringVariable -from graphon.variables.segments import Segment, StringSegment -from libs.datetime_utils import naive_utc_now - - -class MockReadOnlyVariablePool: - def __init__(self, variables: dict[tuple[str, str], Segment] | None = None) -> None: - self._variables = variables or {} - - def get(self, selector: Sequence[str]) -> Segment | None: - if len(selector) < 2: - return None - return self._variables.get((selector[0], selector[1])) - - def get_all_by_node(self, node_id: str) -> dict[str, object]: - return {key: value for (nid, key), value in self._variables.items() if nid == node_id} - - def get_by_prefix(self, prefix: str) -> dict[str, object]: - return {key: value for (nid, key), value in self._variables.items() if nid == prefix} - - -def _build_graph_runtime_state( - variable_pool: MockReadOnlyVariablePool, - conversation_id: str | None = None, -) -> ReadOnlyGraphRuntimeState: - graph_runtime_state = Mock(spec=ReadOnlyGraphRuntimeState) - if conversation_id is not None: - variable_pool._variables[("sys", SystemVariableKey.CONVERSATION_ID.value)] = StringSegment( - value=conversation_id - ) - graph_runtime_state.variable_pool = variable_pool - return graph_runtime_state - - -def _build_node_run_succeeded_event() -> NodeRunSucceededEvent: - return NodeRunSucceededEvent( - id="node-exec-id", - node_id="assigner", - node_type=BuiltinNodeTypes.LLM, - start_at=naive_utc_now(), - node_run_result=NodeRunResult( - status=WorkflowNodeExecutionStatus.SUCCEEDED, - outputs={}, - process_data={}, - ), - ) - - -def _build_variable_updated_event(variable: StringVariable) -> NodeRunVariableUpdatedEvent: - return NodeRunVariableUpdatedEvent( - id="node-exec-id", - node_id="assigner", - node_type=BuiltinNodeTypes.VARIABLE_ASSIGNER, - variable=variable, - ) - - -def test_persists_conversation_variables_from_variable_update_event(): - conversation_id = "conv-123" - variable = StringVariable( - id="var-1", - name="name", - value="updated", - selector=[CONVERSATION_VARIABLE_NODE_ID, "name"], - ) - updater = Mock() - layer = ConversationVariablePersistenceLayer(updater) - layer.initialize(_build_graph_runtime_state(MockReadOnlyVariablePool(), conversation_id), Mock(spec=CommandChannel)) - - event = _build_variable_updated_event(variable) - layer.on_event(event) - - updater.update.assert_called_once_with(conversation_id=conversation_id, variable=variable) - - -def test_skips_non_variable_update_events(): - conversation_id = "conv-456" - updater = Mock() - layer = ConversationVariablePersistenceLayer(updater) - layer.initialize(_build_graph_runtime_state(MockReadOnlyVariablePool(), conversation_id), Mock(spec=CommandChannel)) - - event = _build_node_run_succeeded_event() - layer.on_event(event) - - updater.update.assert_not_called() - - -def test_skips_non_conversation_variables(): - conversation_id = "conv-789" - non_conversation_variable = StringVariable( - id="var-3", - name="name", - value="updated", - selector=["environment", "name"], - ) - updater = Mock() - layer = ConversationVariablePersistenceLayer(updater) - layer.initialize(_build_graph_runtime_state(MockReadOnlyVariablePool(), conversation_id), Mock(spec=CommandChannel)) - - event = _build_variable_updated_event(non_conversation_variable) - layer.on_event(event) - - updater.update.assert_not_called() diff --git a/api/tests/unit_tests/core/mcp/server/test_streamable_http.py b/api/tests/unit_tests/core/mcp/server/test_streamable_http.py index 32df2a8a9b0113..91e2c5f128249e 100644 --- a/api/tests/unit_tests/core/mcp/server/test_streamable_http.py +++ b/api/tests/unit_tests/core/mcp/server/test_streamable_http.py @@ -50,20 +50,27 @@ def setup_method(self): self.request = types.ClientRequest(root=types.PingRequest()) - def test_handle_ping_request(self): + def test_handle_ping_request(self, *, mock_app_generate: Mock): """Test handling ping request""" # Setup ping request self.request.root = types.PingRequest() result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123 + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) assert result.jsonrpc == "2.0" assert result.id == 123 - def test_handle_initialize_request(self): + def test_handle_initialize_request(self, *, mock_app_generate: Mock): """Test handling initialize request""" # Setup initialize request self.request.root = types.InitializeRequest( @@ -75,32 +82,54 @@ def test_handle_initialize_request(self): ) result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123 + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) assert result.jsonrpc == "2.0" assert result.id == 123 - def test_handle_list_tools_request(self): + def test_handle_list_tools_request(self, *, mock_app_generate: Mock): """Test handling list tools request""" # Setup list tools request self.request.root = types.ListToolsRequest() result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123 + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) assert result.jsonrpc == "2.0" assert result.id == 123 - def test_handle_list_tools_request_threads_protocol_version(self): + def test_handle_list_tools_request_threads_protocol_version(self, *, mock_app_generate: Mock): """The negotiated version reaches handle_list_tools through the dispatcher.""" self.request.root = types.ListToolsRequest() result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123, "2025-06-18" + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + "2025-06-18", + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) @@ -108,19 +137,26 @@ def test_handle_list_tools_request_threads_protocol_version(self): assert tool["outputSchema"] == {"type": "object"} assert tool["title"] == "test_app" - def test_handle_list_tools_request_legacy_serialization_unchanged(self): + def test_handle_list_tools_request_legacy_serialization_unchanged(self, *, mock_app_generate: Mock): """A 2024-11-05 tools/list response serializes without any 2025-06-18 fields.""" self.request.root = types.ListToolsRequest() result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123, "2024-11-05" + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + "2024-11-05", + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) tool = result.result["tools"][0] assert set(tool) == {"name", "description", "inputSchema"} - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool_request(self, mock_app_generate): """Test handling call tool request""" # Setup call tool request @@ -135,7 +171,14 @@ def test_handle_call_tool_request(self, mock_app_generate): mock_app_generate.generate.return_value = mock_response result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123 + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) @@ -145,7 +188,6 @@ def test_handle_call_tool_request(self, mock_app_generate): # Verify AppGenerateService was called mock_app_generate.generate.assert_called_once() - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool_returns_trigger_workflow_business_error(self, mock_app_generate): call_request = types.CallToolRequest( params=types.CallToolRequestParams(name="test_app", arguments={"query": "test question"}) @@ -161,13 +203,13 @@ def test_handle_call_tool_returns_trigger_workflow_business_error(self, mock_app self.mcp_server, self.end_user, 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCError) assert result.error.code == types.INVALID_REQUEST assert result.error.data == {"code": "trigger_workflow_service_mode_unavailable"} - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool_request_threads_protocol_version(self, mock_app_generate): """The negotiated version reaches handle_call_tool through the dispatcher.""" call_request = types.CallToolRequest( @@ -178,13 +220,20 @@ def test_handle_call_tool_request_threads_protocol_version(self, mock_app_genera mock_app_generate.generate.return_value = {"answer": "test answer"} result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123, "2025-06-18" + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + "2025-06-18", + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) assert result.result["structuredContent"] == {"answer": "test answer"} - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool_request_legacy_serialization_unchanged(self, mock_app_generate): """A 2024-11-05 tools/call response serializes without structuredContent.""" call_request = types.CallToolRequest( @@ -195,21 +244,36 @@ def test_handle_call_tool_request_legacy_serialization_unchanged(self, mock_app_ mock_app_generate.generate.return_value = {"answer": "test answer"} result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123, "2024-11-05" + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + "2024-11-05", + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) assert "structuredContent" not in result.result assert result.result["content"][0]["text"] == "test answer" - def test_handle_unknown_request_type(self): + def test_handle_unknown_request_type(self, *, mock_app_generate: Mock): """Test handling unknown request type""" # A valid request type with no application-side handler. self.request.root = types.ListResourcesRequest() result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123 + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCError) @@ -217,18 +281,27 @@ def test_handle_unknown_request_type(self): assert result.id == 123 assert result.error.code == types.METHOD_NOT_FOUND - def test_handle_value_error(self): + def test_handle_value_error(self, *, mock_app_generate: Mock): """Test handling ValueError""" # Setup request that will cause ValueError self.request.root = types.CallToolRequest(params=types.CallToolRequestParams(name="test_app", arguments={})) # Don't provide end_user to cause ValueError - result = handle_mcp_request(Mock(), self.app, self.request, self.user_input_form, self.mcp_server, None, 123) + result = handle_mcp_request( + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + None, + 123, + generate=mock_app_generate.generate, + ) assert isinstance(result, types.JSONRPCError) assert result.error.code == types.INVALID_PARAMS - def test_handle_generic_exception(self): + def test_handle_generic_exception(self, *, mock_app_generate: Mock): """Test handling generic exception""" # Setup request that will cause generic exception self.request.root = types.PingRequest() @@ -236,7 +309,14 @@ def test_handle_generic_exception(self): # Inject the handler failure while preserving normal request dispatch. with patch("core.mcp.server.streamable_http.handle_ping", side_effect=Exception("Test error")): result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123 + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCError) @@ -323,7 +403,6 @@ def test_handle_list_tools_omits_structured_output_for_intermediate_client(self) assert tool.outputSchema is None assert tool.title is None - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool(self, mock_app_generate): """Test call tool handler""" app = App( @@ -342,7 +421,7 @@ def test_handle_call_tool(self, mock_app_generate): mock_response = {"answer": "test answer"} mock_app_generate.generate.return_value = mock_response - result = handle_call_tool(Mock(), app, request, user_input_form, end_user) + result = handle_call_tool(Mock(), app, request, user_input_form, end_user, generate=mock_app_generate.generate) assert isinstance(result, types.CallToolResult) assert len(result.content) == 1 @@ -351,7 +430,6 @@ def test_handle_call_tool(self, mock_app_generate): assert hasattr(text_content, "text") assert text_content.text == "test answer" - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool_structured_output_modern_client(self, mock_app_generate): """structuredContent is attached alongside TextContent for >= 2025-06-18.""" app = App( @@ -365,12 +443,13 @@ def test_handle_call_tool_structured_output_modern_client(self, mock_app_generat mock_app_generate.generate.return_value = {"answer": "test answer"} - result = handle_call_tool(Mock(), app, request, [], EndUser(), "2025-06-18") + result = handle_call_tool( + Mock(), app, request, [], EndUser(), "2025-06-18", generate=mock_app_generate.generate + ) assert result.structuredContent == {"answer": "test answer"} assert result.content[0].text == "test answer" - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool_no_structured_output_legacy_client(self, mock_app_generate): """structuredContent is omitted for 2024-11-05 clients.""" app = App( @@ -384,19 +463,21 @@ def test_handle_call_tool_no_structured_output_legacy_client(self, mock_app_gene mock_app_generate.generate.return_value = {"answer": "test answer"} - result = handle_call_tool(Mock(), app, request, [], EndUser(), "2024-11-05") + result = handle_call_tool( + Mock(), app, request, [], EndUser(), "2024-11-05", generate=mock_app_generate.generate + ) assert result.structuredContent is None assert result.content[0].text == "test answer" - def test_handle_call_tool_no_end_user(self): + def test_handle_call_tool_no_end_user(self, *, mock_app_generate: Mock): """Test call tool handler without end user""" app = App() request = types.ClientRequest(root=types.CallToolRequest(params=types.CallToolRequestParams(name="test_app"))) user_input_form: list[VariableEntity] = [] with pytest.raises(ValueError, match="End user not found"): - handle_call_tool(Mock(), app, request, user_input_form, None) + handle_call_tool(Mock(), app, request, user_input_form, None, generate=mock_app_generate.generate) class TestUtilityFunctions: @@ -833,3 +914,8 @@ def test_supported_header_passes_through(self): def test_unsupported_header_returns_none(self): """An explicit but unsupported header signals an error (None).""" assert negotiate_protocol_version("1999-01-01", False) is None + + +@pytest.fixture +def mock_app_generate() -> Mock: + return Mock() diff --git a/api/tests/unit_tests/core/plugin/test_backwards_invocation_app.py b/api/tests/unit_tests/core/plugin/test_backwards_invocation_app.py index 3734cf1f30bcd0..015420cfdca315 100644 --- a/api/tests/unit_tests/core/plugin/test_backwards_invocation_app.py +++ b/api/tests/unit_tests/core/plugin/test_backwards_invocation_app.py @@ -8,12 +8,13 @@ from sqlalchemy.orm import Session, sessionmaker from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig -from core.plugin.backwards_invocation.app import PluginAppBackwardsInvocation from core.plugin.backwards_invocation.base import BaseBackwardsInvocation from models import Account, Tenant, TenantAccountJoin from models.enums import EndUserType from models.model import App, AppMode, AppModelConfig, EndUser from models.workflow import Workflow +from services.plugin.app_invocation import PluginAppBackwardsInvocation +from services.workflow.execution.ports import WorkflowRuntime from tests.unit_tests.model_factories import make_app, make_end_user, make_workflow @@ -122,7 +123,7 @@ def _real_sessions( self.session_factory = sqlite_session_factory self.sqlite_engine = sqlite_engine self.end_users = _EndUserProvisioner() - mocker.patch("core.plugin.backwards_invocation.app.create_session", side_effect=sqlite_session_factory) + mocker.patch("services.plugin.app_invocation.create_session", side_effect=sqlite_session_factory) def test_fetch_app_info_workflow_path(self, mocker: MockerFixture): variable = {"type": "text-input", "variable": "foo", "label": "Foo", "required": False} @@ -133,7 +134,7 @@ def test_fetch_app_info_workflow_path(self, mocker: MockerFixture): mocker.patch.object(PluginAppBackwardsInvocation, "_get_app", return_value=app) mocker.patch.object(PluginAppBackwardsInvocation, "_get_workflow", return_value=workflow) mapper = mocker.patch( - "core.plugin.backwards_invocation.app.get_parameters_from_feature_dict", + "services.plugin.app_invocation.get_parameters_from_feature_dict", return_value={"mapped": True}, ) @@ -148,7 +149,7 @@ def test_fetch_app_info_model_config_path(self, mocker: MockerFixture): mocker.patch.object(PluginAppBackwardsInvocation, "_get_app", return_value=app) mocker.patch.object(PluginAppBackwardsInvocation, "_get_app_model_config_dict", return_value=model_config_dict) mocker.patch( - "core.plugin.backwards_invocation.app.get_parameters_from_feature_dict", + "services.plugin.app_invocation.get_parameters_from_feature_dict", return_value={"mapped": True}, ) @@ -166,7 +167,9 @@ def test_fetch_app_info_model_config_path(self, mocker: MockerFixture): (AppMode.COMPLETION, "invoke_completion_app"), ], ) - def test_invoke_app_routes_by_mode(self, mocker: MockerFixture, mode, route_method): + def test_invoke_app_routes_by_mode( + self, mocker: MockerFixture, mode, route_method, *, workflow_runtime: WorkflowRuntime + ): app = _app(mode=mode) user = _end_user() workflow = _workflow() @@ -186,12 +189,15 @@ def test_invoke_app_routes_by_mode(self, mocker: MockerFixture, mode, route_meth files=[], session=self.session, end_users=self.end_users, + runtime=workflow_runtime, ) assert result == {"routed": True} assert route.call_count == 1 - def test_invoke_app_uses_end_user_when_user_id_missing(self, mocker: MockerFixture): + def test_invoke_app_uses_end_user_when_user_id_missing( + self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + ): app = _app(mode=AppMode.WORKFLOW) end_user = _end_user() workflow = _workflow() @@ -211,6 +217,7 @@ def test_invoke_app_uses_end_user_when_user_id_missing(self, mocker: MockerFixtu files=[], session=self.session, end_users=self.end_users, + runtime=workflow_runtime, ) assert result == {"ok": True} @@ -218,7 +225,9 @@ def test_invoke_app_uses_end_user_when_user_id_missing(self, mocker: MockerFixtu assert route.call_args.args[1] is workflow assert route.call_args.args[2] is end_user - def test_invoke_app_missing_query_for_chat_raises(self, mocker: MockerFixture): + def test_invoke_app_missing_query_for_chat_raises( + self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + ): mocker.patch.object(PluginAppBackwardsInvocation, "_get_app", return_value=_app(mode=AppMode.CHAT)) mocker.patch.object(PluginAppBackwardsInvocation, "_get_user", return_value=_end_user()) @@ -234,9 +243,10 @@ def test_invoke_app_missing_query_for_chat_raises(self, mocker: MockerFixture): files=[], session=self.session, end_users=self.end_users, + runtime=workflow_runtime, ) - def test_invoke_app_unexpected_mode_raises(self, mocker: MockerFixture): + def test_invoke_app_unexpected_mode_raises(self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime): mocker.patch.object( PluginAppBackwardsInvocation, "_get_app", @@ -256,16 +266,19 @@ def test_invoke_app_unexpected_mode_raises(self, mocker: MockerFixture): files=[], session=self.session, end_users=self.end_users, + runtime=workflow_runtime, ) @pytest.mark.parametrize( ("mode", "generator_path"), [ - (AppMode.AGENT_CHAT, "core.plugin.backwards_invocation.app.AgentChatAppGenerator.generate"), - (AppMode.CHAT, "core.plugin.backwards_invocation.app.ChatAppGenerator.generate"), + (AppMode.AGENT_CHAT, "services.plugin.app_invocation.AgentChatAppGenerator.generate"), + (AppMode.CHAT, "services.plugin.app_invocation.ChatAppGenerator.generate"), ], ) - def test_invoke_chat_app_agent_and_chat(self, mocker: MockerFixture, mode, generator_path): + def test_invoke_chat_app_agent_and_chat( + self, mocker: MockerFixture, mode, generator_path, *, workflow_runtime: WorkflowRuntime + ): app = _app(mode=mode) spy = mocker.patch(generator_path, return_value={"result": "ok"}) @@ -278,12 +291,15 @@ def test_invoke_chat_app_agent_and_chat(self, mocker: MockerFixture, mode, gener inputs={"k": "v"}, files=[], session=self.session, + runtime=workflow_runtime, ) assert result == {"result": "ok"} assert spy.call_count == 1 - def test_invoke_chat_app_advanced_chat_injects_pause_state_config(self, mocker: MockerFixture): + def test_invoke_chat_app_advanced_chat_injects_pause_state_config( + self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + ): workflow = _workflow() workflow.created_by = "owner-id" @@ -291,11 +307,11 @@ def test_invoke_chat_app_advanced_chat_injects_pause_state_config(self, mocker: mocker.patch.object(PluginAppBackwardsInvocation, "_get_workflow", return_value=workflow) mocker.patch( - "core.plugin.backwards_invocation.app.db", + "services.plugin.app_invocation.db", _DatabaseWithEngine(self.sqlite_engine), ) generator_spy = mocker.patch( - "core.plugin.backwards_invocation.app.AdvancedChatAppGenerator.generate", + "services.plugin.app_invocation.AdvancedChatAppGenerator.generate", return_value={"result": "ok"}, ) session = self.session @@ -309,16 +325,19 @@ def test_invoke_chat_app_advanced_chat_injects_pause_state_config(self, mocker: inputs={"k": "v"}, files=[], session=session, + runtime=workflow_runtime, ) assert result == {"result": "ok"} call_kwargs = generator_spy.call_args.kwargs - assert call_kwargs["session"] is session + assert "session" not in call_kwargs pause_state_config = call_kwargs.get("pause_state_config") assert isinstance(pause_state_config, PauseStateLayerConfig) assert pause_state_config.state_owner_user_id == "owner-id" - def test_invoke_chat_app_advanced_chat_without_workflow_raises(self, mocker: MockerFixture): + def test_invoke_chat_app_advanced_chat_without_workflow_raises( + self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + ): app = _app(mode=AppMode.ADVANCED_CHAT) mocker.patch.object(PluginAppBackwardsInvocation, "_get_workflow", return_value=None) with pytest.raises(ValueError, match="unexpected app type"): @@ -331,9 +350,10 @@ def test_invoke_chat_app_advanced_chat_without_workflow_raises(self, mocker: Moc inputs={}, files=[], session=self.session, + runtime=workflow_runtime, ) - def test_invoke_chat_app_unexpected_mode_raises(self): + def test_invoke_chat_app_unexpected_mode_raises(self, *, workflow_runtime: WorkflowRuntime): app = _app(mode=cast(AppMode, "invalid")) with pytest.raises(ValueError, match="unexpected app type"): PluginAppBackwardsInvocation.invoke_chat_app( @@ -345,20 +365,23 @@ def test_invoke_chat_app_unexpected_mode_raises(self): inputs={}, files=[], session=self.session, + runtime=workflow_runtime, ) - def test_invoke_workflow_app_injects_pause_state_config(self, mocker: MockerFixture): + def test_invoke_workflow_app_injects_pause_state_config( + self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + ): workflow = _workflow() workflow.created_by = "owner-id" app = _app(mode=AppMode.WORKFLOW) mocker.patch( - "core.plugin.backwards_invocation.app.db", + "services.plugin.app_invocation.db", _DatabaseWithEngine(self.sqlite_engine), ) generator_spy = mocker.patch( - "core.plugin.backwards_invocation.app.WorkflowAppGenerator.generate", + "services.plugin.app_invocation.WorkflowAppGenerator.generate", return_value={"result": "ok"}, ) @@ -369,6 +392,7 @@ def test_invoke_workflow_app_injects_pause_state_config(self, mocker: MockerFixt stream=False, inputs={"k": "v"}, files=[], + runtime=workflow_runtime, ) assert result == {"result": "ok"} @@ -377,7 +401,9 @@ def test_invoke_workflow_app_injects_pause_state_config(self, mocker: MockerFixt assert isinstance(pause_state_config, PauseStateLayerConfig) assert pause_state_config.state_owner_user_id == "owner-id" - def test_invoke_app_workflow_without_workflow_raises(self, mocker: MockerFixture): + def test_invoke_app_workflow_without_workflow_raises( + self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + ): app = _app(mode=AppMode.WORKFLOW) mocker.patch.object(PluginAppBackwardsInvocation, "_get_app", return_value=app) mocker.patch.object(PluginAppBackwardsInvocation, "_get_user", return_value=_end_user()) @@ -394,15 +420,24 @@ def test_invoke_app_workflow_without_workflow_raises(self, mocker: MockerFixture files=[], session=self.session, end_users=self.end_users, + runtime=workflow_runtime, ) - def test_invoke_completion_app(self, mocker: MockerFixture): - spy = mocker.patch( - "core.plugin.backwards_invocation.app.CompletionAppGenerator.generate", return_value={"ok": 1} - ) + def test_invoke_completion_app(self, workflow_runtime, mocker: MockerFixture, app_records, annotation_replies): + spy = mocker.patch("services.plugin.app_invocation.CompletionAppGenerator.generate", return_value={"ok": 1}) app = _app(mode=AppMode.COMPLETION) - result = PluginAppBackwardsInvocation.invoke_completion_app(app, _end_user(), False, {"x": 1}, [], self.session) + result = PluginAppBackwardsInvocation.invoke_completion_app( + app, + _end_user(), + False, + {"x": 1}, + [], + self.session, + records=app_records, + annotations=annotation_replies, + retrieval=workflow_runtime.retrieval, + ) assert result == {"ok": 1} assert spy.call_count == 1 @@ -523,7 +558,9 @@ def test_get_user_raises_when_user_not_found(self): with pytest.raises(ValueError, match="user not found"): PluginAppBackwardsInvocation._get_user("uid", app) - def test_invoke_app_creates_end_user_for_unknown_external_user_id(self, mocker: MockerFixture): + def test_invoke_app_creates_end_user_for_unknown_external_user_id( + self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + ): app = _app(mode=AppMode.WORKFLOW) end_user = _end_user(session_id="wecom-sender-1") workflow = _workflow() @@ -544,6 +581,7 @@ def test_invoke_app_creates_end_user_for_unknown_external_user_id(self, mocker: files=[], session=self.session, end_users=self.end_users, + runtime=workflow_runtime, ) assert result == {"ok": True} @@ -610,7 +648,7 @@ def test_get_app_model_config_dict_uses_explicit_session_for_annotation_reply(se self.session.add(app_model_config) self.session.commit() load_annotation_reply_config = mocker.patch( - "core.plugin.backwards_invocation.app.load_annotation_reply_config", + "services.plugin.app_invocation.load_annotation_reply_config", return_value=annotation_reply, ) app = _app(app_model_config_id="config-1") diff --git a/api/tests/unit_tests/core/tools/test_base_tool.py b/api/tests/unit_tests/core/tools/test_base_tool.py index 8cbd538b66eba6..b46b763ee6ca44 100644 --- a/api/tests/unit_tests/core/tools/test_base_tool.py +++ b/api/tests/unit_tests/core/tools/test_base_tool.py @@ -5,7 +5,6 @@ from typing import Any, cast import pytest -from sqlalchemy.orm import Session from core.app.entities.app_invoke_entities import InvokeFrom from core.tools.__base.tool import Tool @@ -52,7 +51,6 @@ def tool_provider_type(self) -> ToolProviderType: def _invoke( self, - session: Any, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, @@ -94,7 +92,7 @@ def _build_tool(runtime: ToolRuntime | None = None) -> DummyTool: return DummyTool(entity=entity, runtime=runtime) -def test_invoke_supports_single_message_and_parameter_casting(sqlite_session: Session): +def test_invoke_supports_single_message_and_parameter_casting(): runtime = ToolRuntime( tenant_id="tenant-1", invoke_from=InvokeFrom.DEBUGGER, @@ -112,7 +110,6 @@ def test_invoke_supports_single_message_and_parameter_casting(sqlite_session: Se messages = list( tool.invoke( - session=sqlite_session, user_id="user-1", tool_parameters={"age": "18", "raw": "keep"}, conversation_id="conv-1", @@ -132,7 +129,7 @@ def test_invoke_supports_single_message_and_parameter_casting(sqlite_session: Se } -def test_invoke_preserves_multiple_select_values(sqlite_session: Session): +def test_invoke_preserves_multiple_select_values(): tool = _build_tool() parameter = ToolParameter.get_simple_instance( name="choice", @@ -144,18 +141,18 @@ def test_invoke_preserves_multiple_select_values(sqlite_session: Session): parameter.multiple = True tool.entity.parameters = [parameter] - list(tool.invoke(session=sqlite_session, user_id="user-1", tool_parameters={"choice": ["a", "b"]})) + list(tool.invoke(user_id="user-1", tool_parameters={"choice": ["a", "b"]})) assert tool.last_invocation is not None assert tool.last_invocation["tool_parameters"] == {"choice": ["a", "b"]} with pytest.raises(ValueError, match="must be a list"): - tool.invoke(session=sqlite_session, user_id="user-1", tool_parameters={"choice": "a"}) + tool.invoke(user_id="user-1", tool_parameters={"choice": "a"}) -def test_invoke_supports_list_and_generator_results(sqlite_session: Session): +def test_invoke_supports_list_and_generator_results(): tool = _build_tool() tool.result = [tool.create_text_message("a"), tool.create_text_message("b")] - list_messages = list(tool.invoke(session=sqlite_session, user_id="user-1", tool_parameters={})) + list_messages = list(tool.invoke(user_id="user-1", tool_parameters={})) assert [msg.message.text for msg in list_messages] == ["a", "b"] def _message_generator() -> Generator[ToolInvokeMessage, None, None]: @@ -163,7 +160,7 @@ def _message_generator() -> Generator[ToolInvokeMessage, None, None]: yield tool.create_text_message("g2") tool.result = _message_generator() - generated_messages = list(tool.invoke(session=sqlite_session, user_id="user-2", tool_parameters={})) + generated_messages = list(tool.invoke(user_id="user-2", tool_parameters={})) assert [msg.message.text for msg in generated_messages] == ["g1", "g2"] @@ -372,6 +369,6 @@ def test_message_factory_helpers(): assert variable_message.message.stream is False -def test_base_abstract_invoke_placeholder_returns_none(sqlite_session: Session): +def test_base_abstract_invoke_placeholder_returns_none(): tool = _build_tool() - assert Tool._invoke(tool, session=sqlite_session, user_id="u", tool_parameters={}) is None + assert Tool._invoke(tool, user_id="u", tool_parameters={}) is None diff --git a/api/tests/unit_tests/core/tools/test_builtin_tools_extra.py b/api/tests/unit_tests/core/tools/test_builtin_tools_extra.py index 09da0178560201..f1c5b5f1f0f127 100644 --- a/api/tests/unit_tests/core/tools/test_builtin_tools_extra.py +++ b/api/tests/unit_tests/core/tools/test_builtin_tools_extra.py @@ -7,7 +7,6 @@ from zoneinfo import ZoneInfo import pytest -from sqlalchemy.orm import Session from core.app.entities.app_invoke_entities import InvokeFrom from core.tools.__base.tool_runtime import ToolRuntime @@ -51,24 +50,19 @@ def _raise_runtime_error(*_args: object, **_kwargs: object) -> None: raise RuntimeError("boom") -def test_current_time_tool(sqlite_session: Session): +def test_current_time_tool(): current_tool = _build_builtin_tool(CurrentTimeTool) - utc_text = list(current_tool.invoke(session=sqlite_session, user_id="u", tool_parameters={"timezone": "UTC"}))[ - 0 - ].message.text + utc_text = list(current_tool.invoke(user_id="u", tool_parameters={"timezone": "UTC"}))[0].message.text assert utc_text - invalid_tz = list( - current_tool.invoke(session=sqlite_session, user_id="u", tool_parameters={"timezone": "Invalid/TZ"}) - )[0].message.text + invalid_tz = list(current_tool.invoke(user_id="u", tool_parameters={"timezone": "Invalid/TZ"}))[0].message.text assert "Invalid timezone" in invalid_tz -def test_localtime_to_timestamp_tool(sqlite_session: Session): +def test_localtime_to_timestamp_tool(): localtime_tool = _build_builtin_tool(LocaltimeToTimestampTool) ts_message = list( localtime_tool.invoke( - session=sqlite_session, user_id="u", tool_parameters={"localtime": "2024-01-01 10:00:00", "timezone": "UTC"}, ) @@ -94,23 +88,20 @@ def test_localtime_to_timestamp_tool(sqlite_session: Session): LocaltimeToTimestampTool.localtime_to_timestamp("bad", "%Y-%m-%d %H:%M:%S", "UTC") -def test_timestamp_to_localtime_tool(sqlite_session: Session): +def test_timestamp_to_localtime_tool(): to_local_tool = _build_builtin_tool(TimestampToLocaltimeTool) - local_text = list( - to_local_tool.invoke( - session=sqlite_session, user_id="u", tool_parameters={"timestamp": 1704067200, "timezone": "UTC"} - ) - )[0].message.text + local_text = list(to_local_tool.invoke(user_id="u", tool_parameters={"timestamp": 1704067200, "timezone": "UTC"}))[ + 0 + ].message.text assert "2024" in local_text with pytest.raises(ToolInvokeError): TimestampToLocaltimeTool.timestamp_to_localtime("bad", "UTC") # type: ignore[arg-type] -def test_timezone_conversion_tool(sqlite_session: Session): +def test_timezone_conversion_tool(): timezone_tool = _build_builtin_tool(TimezoneConversionTool) converted = list( timezone_tool.invoke( - session=sqlite_session, user_id="u", tool_parameters={ "current_time": "2024-01-01 08:00:00", @@ -124,11 +115,9 @@ def test_timezone_conversion_tool(sqlite_session: Session): TimezoneConversionTool.timezone_convert("bad", "UTC", "Asia/Tokyo") -def test_weekday_tool(sqlite_session: Session): +def test_weekday_tool(): weekday_tool = _build_builtin_tool(WeekdayTool) - valid = list( - weekday_tool.invoke(session=sqlite_session, user_id="u", tool_parameters={"year": 2024, "month": 1, "day": 1}) - )[0].message.text + valid = list(weekday_tool.invoke(user_id="u", tool_parameters={"year": 2024, "month": 1, "day": 1}))[0].message.text expected_date = date(2024, 1, 1) expected_message = ( f"{calendar.month_name[expected_date.month]} " @@ -136,19 +125,17 @@ def test_weekday_tool(sqlite_session: Session): f"is {calendar.day_name[expected_date.weekday()]}." ) assert valid == expected_message - invalid = list( - weekday_tool.invoke(session=sqlite_session, user_id="u", tool_parameters={"year": 2024, "month": 2, "day": 31}) - )[0].message.text + invalid = list(weekday_tool.invoke(user_id="u", tool_parameters={"year": 2024, "month": 2, "day": 31}))[ + 0 + ].message.text assert "Invalid date" in invalid with pytest.raises(ValueError, match="Month is required"): - list(weekday_tool.invoke(session=sqlite_session, user_id="u", tool_parameters={"year": 2024, "day": 1})) + list(weekday_tool.invoke(user_id="u", tool_parameters={"year": 2024, "day": 1})) # LLMs often send numeric parameters as strings; these must be accepted. - string_params = list( - weekday_tool.invoke( - session=sqlite_session, user_id="u", tool_parameters={"year": "2024", "month": "3", "day": "5"} - ) - )[0].message.text + string_params = list(weekday_tool.invoke(user_id="u", tool_parameters={"year": "2024", "month": "3", "day": "5"}))[ + 0 + ].message.text expected_date = date(2024, 3, 5) assert string_params == ( f"{calendar.month_name[expected_date.month]} " @@ -157,11 +144,11 @@ def test_weekday_tool(sqlite_session: Session): ) # Missing or non-numeric values must yield an "Invalid date" message, not crash. for params in ({"year": 2024, "month": 3}, {"year": "abc", "month": 3, "day": 5}): - result = list(weekday_tool.invoke(session=sqlite_session, user_id="u", tool_parameters=params))[0].message.text + result = list(weekday_tool.invoke(user_id="u", tool_parameters=params))[0].message.text assert "Invalid date" in result -def test_simple_code_valid_execution(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_simple_code_valid_execution(monkeypatch: pytest.MonkeyPatch): simple_code = _build_builtin_tool(SimpleCode) monkeypatch.setattr( @@ -170,7 +157,6 @@ def test_simple_code_valid_execution(monkeypatch: pytest.MonkeyPatch, sqlite_ses ) result = list( simple_code.invoke( - session=sqlite_session, user_id="u", tool_parameters={"language": "python3", "code": "print(1)"}, ) @@ -178,18 +164,14 @@ def test_simple_code_valid_execution(monkeypatch: pytest.MonkeyPatch, sqlite_ses assert result == "ok" -def test_simple_code_invalid_language(sqlite_session: Session): +def test_simple_code_invalid_language(): simple_code = _build_builtin_tool(SimpleCode) with pytest.raises(ValueError, match="Only python3 and javascript"): - list( - simple_code.invoke( - session=sqlite_session, user_id="u", tool_parameters={"language": "go", "code": "fmt.Println(1)"} - ) - ) + list(simple_code.invoke(user_id="u", tool_parameters={"language": "go", "code": "fmt.Println(1)"})) -def test_simple_code_execution_error(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_simple_code_execution_error(monkeypatch: pytest.MonkeyPatch): simple_code = _build_builtin_tool(SimpleCode) monkeypatch.setattr( @@ -199,35 +181,31 @@ def test_simple_code_execution_error(monkeypatch: pytest.MonkeyPatch, sqlite_ses with pytest.raises(ToolInvokeError, match="boom"): list( simple_code.invoke( - session=sqlite_session, user_id="u", tool_parameters={"language": "python3", "code": "print(1)"}, ) ) -def test_webscraper_empty_url(sqlite_session: Session): +def test_webscraper_empty_url(): webscraper = _build_builtin_tool(WebscraperTool) - empty = list(webscraper.invoke(session=sqlite_session, user_id="u", tool_parameters={"url": ""}))[0].message.text + empty = list(webscraper.invoke(user_id="u", tool_parameters={"url": ""}))[0].message.text assert empty == "Please input url" -def test_webscraper_fetch(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_webscraper_fetch(monkeypatch: pytest.MonkeyPatch): webscraper = _build_builtin_tool(WebscraperTool) monkeypatch.setattr("core.tools.builtin_tool.providers.webscraper.tools.webscraper.get_url", lambda *a, **k: "page") - full = list(webscraper.invoke(session=sqlite_session, user_id="u", tool_parameters={"url": "https://example.com"}))[ - 0 - ].message.text + full = list(webscraper.invoke(user_id="u", tool_parameters={"url": "https://example.com"}))[0].message.text assert full == "page" -def test_webscraper_summary(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_webscraper_summary(monkeypatch: pytest.MonkeyPatch): webscraper = _build_builtin_tool(WebscraperTool) monkeypatch.setattr("core.tools.builtin_tool.providers.webscraper.tools.webscraper.get_url", lambda *a, **k: "page") monkeypatch.setattr(webscraper, "summary", lambda user_id, content: "summary") summarized = list( webscraper.invoke( - session=sqlite_session, user_id="u", tool_parameters={"url": "https://example.com", "generate_summary": True}, ) @@ -235,26 +213,24 @@ def test_webscraper_summary(monkeypatch: pytest.MonkeyPatch, sqlite_session: Ses assert summarized == "summary" -def test_webscraper_fetch_error(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_webscraper_fetch_error(monkeypatch: pytest.MonkeyPatch): webscraper = _build_builtin_tool(WebscraperTool) monkeypatch.setattr( "core.tools.builtin_tool.providers.webscraper.tools.webscraper.get_url", _raise_runtime_error, ) with pytest.raises(ToolInvokeError, match="boom"): - list(webscraper.invoke(session=sqlite_session, user_id="u", tool_parameters={"url": "https://example.com"})) + list(webscraper.invoke(user_id="u", tool_parameters={"url": "https://example.com"})) -def test_asr_invalid_file(sqlite_session: Session): +def test_asr_invalid_file(): asr = _build_builtin_tool(ASRTool) file_obj = SimpleNamespace(type=FileType.DOCUMENT) - invalid_file = list(asr.invoke(session=sqlite_session, user_id="u", tool_parameters={"audio_file": file_obj}))[ - 0 - ].message.text + invalid_file = list(asr.invoke(user_id="u", tool_parameters={"audio_file": file_obj}))[0].message.text assert "not a valid audio file" in invalid_file -def test_asr_valid_file_invocation(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_asr_valid_file_invocation(monkeypatch: pytest.MonkeyPatch): asr = _build_builtin_tool(ASRTool) model_instance = type("M", (), {"invoke_speech2text": lambda self, file: "transcript"})() model_manager = type("Mgr", (), {"get_model_instance": lambda *a, **k: model_instance})() @@ -266,9 +242,7 @@ def test_asr_valid_file_invocation(monkeypatch: pytest.MonkeyPatch, sqlite_sessi lambda **kwargs: captured_manager_kwargs.update(kwargs) or model_manager, ) audio_file = SimpleNamespace(type=FileType.AUDIO) - ok = list( - asr.invoke(session=sqlite_session, user_id="u", tool_parameters={"audio_file": audio_file, "model": "p#m"}) - )[0].message.text + ok = list(asr.invoke(user_id="u", tool_parameters={"audio_file": audio_file, "model": "p#m"}))[0].message.text assert ok == "transcript" assert captured_manager_kwargs == {"tenant_id": "tenant-1", "user_id": "u"} @@ -284,7 +258,7 @@ def test_asr_available_models_and_runtime_parameters(monkeypatch: pytest.MonkeyP assert asr.get_runtime_parameters()[0].name == "model" -def test_tts_invoke_returns_messages(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_tts_invoke_returns_messages(monkeypatch: pytest.MonkeyPatch): tts = _build_builtin_tool(TTSTool) captured_manager_kwargs = {} voices_model_instance = type( @@ -302,7 +276,7 @@ def test_tts_invoke_returns_messages(monkeypatch: pytest.MonkeyPatch, sqlite_ses or type("M", (), {"get_model_instance": lambda *a, **k: voices_model_instance})() ), ) - messages = list(tts.invoke(session=sqlite_session, user_id="u", tool_parameters={"model": "p#m", "text": "hello"})) + messages = list(tts.invoke(user_id="u", tool_parameters={"model": "p#m", "text": "hello"})) assert [m.type for m in messages] == [ToolInvokeMessage.MessageType.TEXT, ToolInvokeMessage.MessageType.BLOB] assert messages[1].meta == {"mime_type": "audio/mpeg"} assert captured_manager_kwargs == {"tenant_id": "tenant-1", "user_id": "u"} @@ -315,18 +289,18 @@ def test_tts_get_available_models_requires_runtime(): tts.get_available_models() -def test_tts_tool_raises_when_runtime_missing(sqlite_session: Session): +def test_tts_tool_raises_when_runtime_missing(): tts = _build_builtin_tool(TTSTool) tts.runtime = None with pytest.raises(ValueError, match="Runtime is required"): - list(tts.invoke(session=sqlite_session, user_id="u", tool_parameters={"model": "p#m", "text": "hello"})) + list(tts.invoke(user_id="u", tool_parameters={"model": "p#m", "text": "hello"})) @pytest.mark.parametrize( "voices", [[{"value": None}], []], ) -def test_tts_tool_raises_when_voice_unavailable(monkeypatch, voices, sqlite_session: Session): +def test_tts_tool_raises_when_voice_unavailable(monkeypatch, voices): tts = _build_builtin_tool(TTSTool) tts.runtime = ToolRuntime(tenant_id="tenant-1", invoke_from=InvokeFrom.DEBUGGER) model_without_voice = type( @@ -342,7 +316,7 @@ def test_tts_tool_raises_when_voice_unavailable(monkeypatch, voices, sqlite_sess lambda **_: type("Manager", (), {"get_model_instance": lambda *args, **kwargs: model_without_voice})(), ) with pytest.raises(ValueError, match="no voice available"): - list(tts.invoke(session=sqlite_session, user_id="u", tool_parameters={"model": "p#m", "text": "hello"})) + list(tts.invoke(user_id="u", tool_parameters={"model": "p#m", "text": "hello"})) def test_tts_tool_get_available_models_and_runtime_parameters(monkeypatch: pytest.MonkeyPatch): @@ -394,14 +368,13 @@ def test_provider_classes_and_builtin_sort(monkeypatch: pytest.MonkeyPatch): assert [p.name for p in sorted_providers] == ["a", "b"] -def test_localtime_to_timestamp_tool_epoch_zero(sqlite_session: Session): +def test_localtime_to_timestamp_tool_epoch_zero(): localtime_tool = _build_builtin_tool(LocaltimeToTimestampTool) # 1970-01-01 08:00:00 in Asia/Shanghai is exactly Unix epoch 0. A valid # conversion to timestamp 0 must not be reported as "Invalid localtime" # just because 0 is falsy. epoch_message = list( localtime_tool.invoke( - session=sqlite_session, user_id="u", tool_parameters={"localtime": "1970-01-01 08:00:00", "timezone": "Asia/Shanghai"}, ) diff --git a/api/tests/unit_tests/core/tools/test_custom_tool.py b/api/tests/unit_tests/core/tools/test_custom_tool.py index 9dd68334b6c754..4ea71deb4bbafe 100644 --- a/api/tests/unit_tests/core/tools/test_custom_tool.py +++ b/api/tests/unit_tests/core/tools/test_custom_tool.py @@ -281,11 +281,11 @@ def _fake_post(url, **kwargs): monkeypatch.setattr(tool, "assembling_request", lambda parameters: {}) monkeypatch.setattr(tool, "do_http_request", lambda *args, **kwargs: httpx.Response(200, text='{"a":1}')) monkeypatch.setattr(tool, "validate_and_parse_response", lambda _: ParsedResponse({"a": 1}, True)) - messages = list(tool.invoke(session=sqlite_session, user_id="u1", tool_parameters={})) + messages = list(tool.invoke(user_id="u1", tool_parameters={})) assert [m.type for m in messages] == [ToolInvokeMessage.MessageType.JSON, ToolInvokeMessage.MessageType.TEXT] # _invoke text path monkeypatch.setattr(tool, "validate_and_parse_response", lambda _: ParsedResponse("plain", False)) - messages = list(tool.invoke(session=sqlite_session, user_id="u1", tool_parameters={})) + messages = list(tool.invoke(user_id="u1", tool_parameters={})) assert len(messages) == 1 assert messages[0].message.text == "plain" diff --git a/api/tests/unit_tests/core/tools/test_dataset_retriever_tool.py b/api/tests/unit_tests/core/tools/test_dataset_retriever_tool.py index 71b17c966988e8..d682f4f4d4030c 100644 --- a/api/tests/unit_tests/core/tools/test_dataset_retriever_tool.py +++ b/api/tests/unit_tests/core/tools/test_dataset_retriever_tool.py @@ -1,163 +1,95 @@ -"""Unit tests for DatasetRetrieverTool behavior and retrieval wiring.""" +"""Dataset tools query injected records and preserve their dependencies when forked.""" -from __future__ import annotations - -from types import SimpleNamespace -from unittest.mock import Mock, patch +from dataclasses import dataclass, field +from uuid import uuid4 import pytest from sqlalchemy.orm import Session from core.app.app_config.entities import DatasetRetrieveConfigEntity from core.app.entities.app_invoke_entities import InvokeFrom -from core.tools.utils.dataset_retriever_tool import DatasetRetrieverTool +from core.tools.__base.tool_runtime import ToolRuntime +from models.dataset import Dataset, Document +from repositories.knowledge.retrieval_repository import KnowledgeRetrievalRepository +from services.tools.dataset.tool import DatasetRetrieverTool +from tests.unit_tests.model_factories import make_dataset -def _retrieve_config() -> DatasetRetrieveConfigEntity: - return DatasetRetrieveConfigEntity(retrieve_strategy=DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE) +@dataclass +class Retrieval: + calls: list = field(default_factory=list) + def retrieve_dataset(self, **kwargs): + self.calls.append(kwargs) + return f"result:{kwargs['query']}" -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_get_dataset_tools_returns_empty_for_empty_dataset_ids(sqlite_session: Session) -> None: - # Arrange - retrieve_config = _retrieve_config() - - # Act - tools = DatasetRetrieverTool.get_dataset_tools( - session=sqlite_session, - tenant_id="tenant", - dataset_ids=[], - retrieve_config=retrieve_config, - return_resource=False, - invoke_from=InvokeFrom.DEBUGGER, - hit_callback=Mock(), - user_id="u", - inputs={}, - ) - - # Assert - assert tools == [] +class NoResources: + def return_retriever_resource_info(self, _resources): + raise AssertionError("unexpected resources") -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_get_dataset_tools_returns_empty_for_missing_retrieve_config(sqlite_session: Session) -> None: - # Arrange - dataset_ids = ["d1"] - # Act +@pytest.fixture +def dataset_tools(sqlite_session: Session, sqlite_session_factory): + dataset_id = str(uuid4()) + sqlite_session.add( + make_dataset(dataset_id=dataset_id, tenant_id="tenant", provider="external", description="one\ntwo") + ) + sqlite_session.commit() + retrieval = Retrieval() + config = DatasetRetrieveConfigEntity(retrieve_strategy="multiple") tools = DatasetRetrieverTool.get_dataset_tools( - session=sqlite_session, + records=KnowledgeRetrievalRepository(sqlite_session_factory), + retrieval=lambda: retrieval, tenant_id="tenant", - dataset_ids=dataset_ids, - retrieve_config=None, # type: ignore[arg-type] - return_resource=False, + app_id="app", + dataset_ids=[dataset_id], + retrieve_config=config, + return_resource=True, invoke_from=InvokeFrom.DEBUGGER, - hit_callback=Mock(), - user_id="u", - inputs={}, + hit_callback=NoResources(), + user_id="user", + inputs={"x": 1}, ) - - # Assert - assert tools == [] - - -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_get_dataset_tools_builds_tool_and_restores_strategy(sqlite_session: Session) -> None: - # Arrange - retrieve_config = _retrieve_config() - retrieval_tool = SimpleNamespace(name="dataset_tool", description="desc", run=lambda query: f"result:{query}") - feature = Mock() - feature.to_dataset_retriever_tool.return_value = [retrieval_tool] - - # Act - with patch("core.tools.utils.dataset_retriever_tool.DatasetRetrieval", return_value=feature): - tools = DatasetRetrieverTool.get_dataset_tools( - session=sqlite_session, - tenant_id="tenant", - dataset_ids=["d1"], - retrieve_config=retrieve_config, - return_resource=True, - invoke_from=InvokeFrom.DEBUGGER, - hit_callback=Mock(), - user_id="u", - inputs={"x": 1}, - ) - - # Assert - assert len(tools) == 1 - assert tools[0].entity.identity.name == "dataset_tool" - assert retrieve_config.retrieve_strategy == DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE + return tools, retrieval, config -def _build_dataset_tool(sqlite_session: Session) -> tuple[DatasetRetrieverTool, SimpleNamespace]: - retrieval_tool = SimpleNamespace( - name="dataset_tool", - description="desc", - run=lambda session, query: f"result:{query}", - ) - feature = Mock() - feature.to_dataset_retriever_tool.return_value = [retrieval_tool] - with patch("core.tools.utils.dataset_retriever_tool.DatasetRetrieval", return_value=feature): - tools = DatasetRetrieverTool.get_dataset_tools( - session=sqlite_session, - tenant_id="tenant", - dataset_ids=["d1"], - retrieve_config=_retrieve_config(), - return_resource=False, - invoke_from=InvokeFrom.DEBUGGER, - hit_callback=Mock(), - user_id="u", - inputs={}, - ) - return tools[0], retrieval_tool - - -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_runtime_parameters_shape(sqlite_session: Session) -> None: - # Arrange - tool, _ = _build_dataset_tool(sqlite_session) - - # Act - params = tool.get_runtime_parameters() - - # Assert - assert len(params) == 1 - assert params[0].name == "query" - - -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_empty_query_behavior(sqlite_session: Session) -> None: - # Arrange - tool, _ = _build_dataset_tool(sqlite_session) - - # Act - empty_query = list(tool.invoke(session=sqlite_session, user_id="u", tool_parameters={})) - - # Assert - assert len(empty_query) == 1 - assert empty_query[0].message.text == "please input query" - - -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_query_invocation_result(sqlite_session: Session) -> None: - # Arrange - tool, _ = _build_dataset_tool(sqlite_session) - - # Act - result = list(tool.invoke(session=sqlite_session, user_id="u", tool_parameters={"query": "hello"})) - - # Assert - assert len(result) == 1 +def test_tool_metadata_and_fork_preserve_injected_retrieval(dataset_tools): + tools, retrieval, config = dataset_tools + assert len(tools) == 1 + tool = tools[0] + assert tool.entity.description.llm == "onetwo" + assert [parameter.name for parameter in tool.get_runtime_parameters()] == ["query"] + assert config.retrieve_strategy == "multiple" + forked = tool.fork_tool_runtime(ToolRuntime(tenant_id="tenant")) + result = list(forked.invoke(user_id="user", tool_parameters={"query": "hello"})) assert result[0].message.text == "result:hello" + assert retrieval.calls[0]["inputs"] == {"x": 1} + assert retrieval.calls[0]["app_id"] == "app" -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_validate_credentials(sqlite_session: Session) -> None: - # Arrange - tool, _ = _build_dataset_tool(sqlite_session) +def test_empty_query_does_not_start_retrieval(dataset_tools): + tools, retrieval, _ = dataset_tools + assert list(tools[0].invoke(user_id="user", tool_parameters={}))[0].message.text == "please input query" + assert not retrieval.calls - # Act - result = tool.validate_credentials(credentials={}, parameters={}, format_only=False) - # Assert - assert result is None +@pytest.mark.parametrize("sqlite_session", [(Dataset, Document)], indirect=True) +@pytest.mark.parametrize("case", ["no-datasets", "no-config", "foreign-tenant"]) +def test_tool_preparation_rejects_unavailable_datasets(sqlite_session, sqlite_session_factory, case): + sqlite_session.add(make_dataset(dataset_id="dataset", tenant_id="other", provider="external")) + sqlite_session.commit() + result = DatasetRetrieverTool.get_dataset_tools( + records=KnowledgeRetrievalRepository(sqlite_session_factory), + retrieval=Retrieval, + tenant_id="tenant", + app_id="app", + dataset_ids=[] if case == "no-datasets" else ["dataset"], + retrieve_config=None if case == "no-config" else DatasetRetrieveConfigEntity(retrieve_strategy="single"), + return_resource=False, + invoke_from=InvokeFrom.DEBUGGER, + hit_callback=NoResources(), + user_id="user", + inputs={}, + ) + assert result == [] diff --git a/api/tests/unit_tests/core/tools/test_mcp_tool.py b/api/tests/unit_tests/core/tools/test_mcp_tool.py index e5df4634527bb9..206e673f62370f 100644 --- a/api/tests/unit_tests/core/tools/test_mcp_tool.py +++ b/api/tests/unit_tests/core/tools/test_mcp_tool.py @@ -6,7 +6,6 @@ from unittest.mock import patch import pytest -from sqlalchemy.orm import Session from core.app.entities.app_invoke_entities import InvokeFrom from core.mcp.types import ( @@ -101,8 +100,7 @@ def test_mcp_tool_usage_extraction_helpers(): assert derived.total_tokens == 0 -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_mcp_tool_invoke_handles_content_types_and_structured_output(sqlite_session: Session): +def test_mcp_tool_invoke_handles_content_types_and_structured_output(): tool = _build_mcp_tool() img_data = base64.b64encode(b"img").decode() blob_data = base64.b64encode(b"blob").decode() @@ -128,7 +126,7 @@ def test_mcp_tool_invoke_handles_content_types_and_structured_output(sqlite_sess ) with patch.object(MCPTool, "invoke_remote_mcp_tool", return_value=result): - messages = list(tool.invoke(session=sqlite_session, user_id="user-1", tool_parameters={"a": 1})) + messages = list(tool.invoke(user_id="user-1", tool_parameters={"a": 1})) types = [m.type for m in messages] assert ToolInvokeMessage.MessageType.JSON in types @@ -138,8 +136,7 @@ def test_mcp_tool_invoke_handles_content_types_and_structured_output(sqlite_sess assert tool.latest_usage.total_tokens == 5 -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_mcp_tool_invoke_raises_for_unsupported_embedded_resource(sqlite_session: Session): +def test_mcp_tool_invoke_raises_for_unsupported_embedded_resource(): tool = _build_mcp_tool() # Use model_construct to bypass pydantic validation and force unsupported resource path. bad_resource = EmbeddedResource.model_construct(type="resource", resource=object()) @@ -147,7 +144,7 @@ def test_mcp_tool_invoke_raises_for_unsupported_embedded_resource(sqlite_session with patch.object(MCPTool, "invoke_remote_mcp_tool", return_value=result): with pytest.raises(ToolInvokeError, match="Unsupported embedded resource type"): - list(tool.invoke(session=sqlite_session, user_id="user-1", tool_parameters={})) + list(tool.invoke(user_id="user-1", tool_parameters={})) def test_mcp_tool_handle_none_parameter_filters_empty_values(): @@ -284,7 +281,7 @@ def test_invoke_skips_forwarding_outside_enterprise_edition(config_overrides): _meta=None, ) with patch.object(tool, "_inject_forwarded_identity") as inject: - with patch("services.tools.legacy_mcp_tools_manage_service.MCPToolManageService"): + with patch("services.tools.mcp_tools_manage_service.MCPToolManageService"): with patch("core.entities.mcp_provider.MCPProviderEntity.decrypt_server_url", return_value="u"): with patch("core.entities.mcp_provider.MCPProviderEntity.decrypt_headers", return_value={}): # Should not raise; should not call enterprise. diff --git a/api/tests/unit_tests/core/tools/test_plugin_tool.py b/api/tests/unit_tests/core/tools/test_plugin_tool.py index 2ab3e57a8adcb1..4378432a0f01fa 100644 --- a/api/tests/unit_tests/core/tools/test_plugin_tool.py +++ b/api/tests/unit_tests/core/tools/test_plugin_tool.py @@ -2,9 +2,6 @@ from unittest.mock import Mock, patch -import pytest -from sqlalchemy.orm import Session - from core.app.entities.app_invoke_entities import InvokeFrom from core.tools.__base.tool_runtime import ToolRuntime from core.tools.entities.common_entities import I18nObject @@ -40,8 +37,7 @@ def _build_plugin_tool(*, has_runtime_parameters: bool) -> PluginTool: ) -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_plugin_tool_invoke_and_fork_runtime(sqlite_session: Session): +def test_plugin_tool_invoke_and_fork_runtime(): tool = _build_plugin_tool(has_runtime_parameters=False) manager = Mock() manager.invoke.return_value = iter([tool.create_text_message("ok")]) @@ -51,7 +47,7 @@ def test_plugin_tool_invoke_and_fork_runtime(sqlite_session: Session): "core.tools.plugin_tool.tool.convert_parameters_to_plugin_format", return_value={"converted": 1}, ): - messages = list(tool.invoke(session=sqlite_session, user_id="user-1", tool_parameters={"raw": 1})) + messages = list(tool.invoke(user_id="user-1", tool_parameters={"raw": 1})) assert [m.message.text for m in messages] == ["ok"] manager.invoke.assert_called_once() diff --git a/api/tests/unit_tests/core/tools/test_tool_engine.py b/api/tests/unit_tests/core/tools/test_tool_engine.py index 941dbb22a7db3a..de388475cec837 100644 --- a/api/tests/unit_tests/core/tools/test_tool_engine.py +++ b/api/tests/unit_tests/core/tools/test_tool_engine.py @@ -7,7 +7,6 @@ import pytest from sqlalchemy import select -from sqlalchemy.engine import Engine from sqlalchemy.orm import Session from core.app.entities.app_invoke_entities import InvokeFrom @@ -28,15 +27,9 @@ ToolInvokeError, ToolParameterValidationError, ) -from core.tools.tool_engine import ToolEngine -from models.model import AppMode, Message, MessageFile - - -class _DatabaseBinding: - engine: Engine - - def __init__(self, engine: Engine) -> None: - self.engine = engine +from models.model import App, AppMode, Conversation, Message, MessageFile +from services.tools.tool_engine import ToolEngine +from tests.unit_tests.model_factories import make_app, make_conversation def _message() -> Message: @@ -84,7 +77,6 @@ def tool_provider_type(self) -> ToolProviderType: def _invoke( self, - session: Any, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, @@ -246,41 +238,52 @@ def test_convert_tool_response_deduplicates_only_matching_json(): assert ToolEngine.tool_response_to_str(messages) == text + "result" + text + '{"c": 3}{"c": 3}' -@pytest.mark.parametrize("sqlite_session", [(MessageFile,)], indirect=True) -def test_create_message_files_and_invoke_generator(sqlite_engine: Engine, sqlite_session: Session): +@pytest.mark.parametrize("sqlite_session", [(App, Conversation, Message, MessageFile)], indirect=True) +def test_create_message_files_and_invoke_generator(sqlite_session: Session, *, app_records): binaries = [ ToolInvokeMessageBinary(mimetype="image/png", url="https://example.com/abc.png"), ToolInvokeMessageBinary(mimetype="audio/wav", url="https://example.com/def.wav"), ] agent_message = _message() - with patch("core.tools.tool_engine.db", _DatabaseBinding(sqlite_engine)): - ids = ToolEngine._create_message_files( - tool_messages=binaries, - agent_message=agent_message, - invoke_from=InvokeFrom.DEBUGGER, - user_id=str(uuid4()), - ) - + sqlite_session.add_all( + [ + make_app(app_id=agent_message.app_id, tenant_id="tenant-1"), + make_conversation( + conversation_id=agent_message.conversation_id, + app_id=agent_message.app_id, + inputs={}, + from_source="console", + ), + agent_message, + ] + ) + sqlite_session.commit() + ids = ToolEngine._create_message_files( + records=app_records, + tenant_id="tenant-1", + tool_messages=binaries, + agent_message=agent_message, + invoke_from=InvokeFrom.DEBUGGER, + user_id=str(uuid4()), + ) message_files = list(sqlite_session.scalars(select(MessageFile).order_by(MessageFile.created_at)).all()) assert ids == [message_file.id for message_file in message_files] assert len(message_files) == 2 assert {message_file.message_id for message_file in message_files} == {agent_message.id} tool = _build_tool() - invoked = list(ToolEngine._invoke(sqlite_session, tool, {"a": 1}, user_id="u")) + invoked = list(ToolEngine._invoke(tool, {"a": 1}, user_id="u")) assert invoked[0].type == ToolInvokeMessage.MessageType.TEXT assert isinstance(invoked[-1], ToolInvokeMeta) assert invoked[-1].error is None -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_generic_invoke_success_and_error_paths(sqlite_session: Session): +def test_generic_invoke_success_and_error_paths(): tool = _build_tool() callback = Mock() callback.on_tool_execution.side_effect = lambda **kwargs: kwargs["tool_outputs"] response = list( ToolEngine.generic_invoke( - session=sqlite_session, tool=tool, tool_parameters={"x": 1}, user_id="u1", @@ -301,7 +304,6 @@ def test_generic_invoke_success_and_error_paths(sqlite_session: Session): with pytest.raises(RuntimeError, match="boom"): list( ToolEngine.generic_invoke( - session=sqlite_session, tool=tool, tool_parameters={"x": 1}, user_id="u1", @@ -312,8 +314,7 @@ def test_generic_invoke_success_and_error_paths(sqlite_session: Session): error_callback.on_tool_error.assert_called_once() -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_agent_invoke_success(sqlite_session: Session): +def test_agent_invoke_success(app_records): tool = _build_tool(with_llm_parameter=True) callback = Mock() message = _message() @@ -321,13 +322,13 @@ def test_agent_invoke_success(sqlite_session: Session): with patch.object(ToolEngine, "_invoke", return_value=iter([tool.create_text_message("ok"), meta])): with patch( - "core.tools.tool_engine.ToolFileMessageTransformer.transform_tool_invoke_messages", + "services.tools.tool_engine.ToolFileMessageTransformer.transform_tool_invoke_messages", side_effect=lambda messages, **kwargs: messages, ): with patch.object(ToolEngine, "_extract_tool_response_binary_and_text", return_value=iter([])): with patch.object(ToolEngine, "_create_message_files", return_value=[]): result_text, message_files, result_meta = ToolEngine.agent_invoke( - session=sqlite_session, + records=app_records, tool=tool, tool_parameters="hello", user_id="u1", @@ -344,15 +345,14 @@ def test_agent_invoke_success(sqlite_session: Session): callback.on_tool_end.assert_called_once() -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_agent_invoke_param_validation_error(sqlite_session: Session): +def test_agent_invoke_param_validation_error(app_records): tool = _build_tool(with_llm_parameter=True) callback = Mock() message = _message() with patch.object(ToolEngine, "_invoke", side_effect=ToolParameterValidationError("bad-param")): error_text, files, error_meta = ToolEngine.agent_invoke( - session=sqlite_session, + records=app_records, tool=tool, tool_parameters={"a": 1}, user_id="u1", @@ -367,8 +367,7 @@ def test_agent_invoke_param_validation_error(sqlite_session: Session): assert error_meta.error -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_agent_invoke_engine_meta_error(sqlite_session: Session): +def test_agent_invoke_engine_meta_error(app_records): tool = _build_tool(with_llm_parameter=True) callback = Mock() message = _message() @@ -376,7 +375,7 @@ def test_agent_invoke_engine_meta_error(sqlite_session: Session): with patch.object(ToolEngine, "_invoke", side_effect=engine_error): error_text, files, error_meta = ToolEngine.agent_invoke( - session=sqlite_session, + records=app_records, tool=tool, tool_parameters={"a": 1}, user_id="u1", @@ -413,15 +412,14 @@ def test_convert_tool_response_excludes_variable_messages(): assert "variable_name" not in result -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_agent_invoke_tool_invoke_error(sqlite_session: Session): +def test_agent_invoke_tool_invoke_error(app_records): tool = _build_tool(with_llm_parameter=True) callback = Mock() message = _message() with patch.object(ToolEngine, "_invoke", side_effect=ToolInvokeError("invoke boom")): error_text, files, _ = ToolEngine.agent_invoke( - session=sqlite_session, + records=app_records, tool=tool, tool_parameters={"a": 1}, user_id="u1", diff --git a/api/tests/unit_tests/core/tools/utils/test_misc_utils_extra.py b/api/tests/unit_tests/core/tools/utils/test_misc_utils_extra.py index c0123332876216..61b0004d4af6cc 100644 --- a/api/tests/unit_tests/core/tools/utils/test_misc_utils_extra.py +++ b/api/tests/unit_tests/core/tools/utils/test_misc_utils_extra.py @@ -1,139 +1,14 @@ from __future__ import annotations import uuid -from contextlib import nullcontext -from datetime import datetime from pathlib import Path -from unittest.mock import patch import pytest -from sqlalchemy.orm import Session, scoped_session, sessionmaker from yaml import YAMLError -from core.app.app_config.entities import DatasetRetrieveConfigEntity -from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler -from core.model_manager import ModelInstance, ModelManager -from core.provider_manager import ProviderManager -from core.rag.embedding.retrieval import RetrievalSegments -from core.rag.models.document import Document as RagDocument -from core.rag.rerank.rerank_model import RerankModelRunner -from core.tools.utils.dataset_retriever import dataset_multi_retriever_tool as multi_retriever_module -from core.tools.utils.dataset_retriever import dataset_retriever_tool as single_retriever_module -from core.tools.utils.dataset_retriever.dataset_multi_retriever_tool import DatasetMultiRetrieverTool -from core.tools.utils.dataset_retriever.dataset_retriever_tool import DatasetRetrieverTool as SingleDatasetRetrieverTool from core.tools.utils.text_processing_utils import remove_leading_symbols from core.tools.utils.uuid_utils import is_valid_uuid from core.tools.utils.yaml_utils import _load_yaml_file, load_yaml_file_cached -from models.dataset import Dataset, Document, DocumentSegment -from models.enums import DataSourceType, DocumentCreatedFrom, SegmentStatus -from tests.unit_tests.model_factories import upload_reader - - -def _retrieve_config() -> DatasetRetrieveConfigEntity: - return DatasetRetrieveConfigEntity(retrieve_strategy=DatasetRetrieveConfigEntity.RetrieveStrategy.SINGLE) - - -def _persist_dataset( - session: Session, - *, - dataset_id: str | None = None, - tenant_id: str | None = None, - name: str = "Knowledge Base", - provider: str = "vendor", - retrieval_model: dict | None = None, -) -> Dataset: - dataset = Dataset( - id=dataset_id or str(uuid.uuid4()), - tenant_id=tenant_id or str(uuid.uuid4()), - name=name, - provider=provider, - data_source_type=DataSourceType.UPLOAD_FILE, - indexing_technique="high_quality", - retrieval_model=retrieval_model, - created_by=str(uuid.uuid4()), - ) - session.add(dataset) - session.commit() - return dataset - - -def _persist_document( - session: Session, - *, - dataset: Dataset, - name: str, - data_source_type: DataSourceType = DataSourceType.UPLOAD_FILE, - doc_metadata: dict | None = None, -) -> Document: - document = Document( - id=str(uuid.uuid4()), - tenant_id=dataset.tenant_id, - dataset_id=dataset.id, - position=1, - data_source_type=data_source_type, - batch="batch-1", - name=name, - created_from=DocumentCreatedFrom.WEB, - created_by=dataset.created_by, - doc_metadata=doc_metadata, - ) - session.add(document) - session.commit() - return document - - -class _FakeFlaskApp: - def app_context(self): - return nullcontext() - - -class _FakeCurrentApp: - def _get_current_object(self) -> _FakeFlaskApp: - return _FakeFlaskApp() - - -class _DatabaseWithSession: - def __init__(self, session: scoped_session[Session]) -> None: - self.session = session - - -class _UnusedProviderManager(ProviderManager): - def __init__(self) -> None: - pass - - -class _UnusedModelInstance(ModelInstance): - def __init__(self) -> None: - pass - - -class _ImmediateThread: - def __init__(self, target=None, kwargs=None, **_kwargs): - self._target = target - self._kwargs = kwargs or {} - - def start(self): - if self._target is not None: - self._target(**self._kwargs) - - def join(self): - return None - - -class _TestHitCallback(DatasetIndexToolCallbackHandler): - def __init__(self): - self.queries: list[tuple[str, str]] = [] - self.documents: list[RagDocument] | None = None - self.resources = None - - def on_query(self, query: str, dataset_id: str, session=None): - self.queries.append((query, dataset_id)) - - def on_tool_end(self, documents: list[RagDocument], session=None): - self.documents = documents - - def return_retriever_resource_info(self, resource): - self.resources = list(resource) def test_remove_leading_symbols_preserves_markdown_link_and_strips_punctuation(): @@ -181,417 +56,3 @@ def test_load_yaml_file_cached_hits(tmp_path: Path): assert load_yaml_file_cached(str(valid_file)) == {"a": 1, "b": "two"} assert load_yaml_file_cached.cache_info().hits == 1 - - -def test_single_dataset_retriever_from_dataset_builds_name_and_description(sqlite_session: Session): - dataset = _persist_dataset( - sqlite_session, - dataset_id="dataset-1", - tenant_id="tenant-1", - name="Knowledge", - ) - - tool = SingleDatasetRetrieverTool.from_dataset( - dataset=dataset, - retrieve_config=_retrieve_config(), - return_resource=False, - retriever_from="prod", - inputs={}, - ) - - assert tool.name == "dataset_dataset_1" - assert tool.description == "useful for when you want to answer queries about the Knowledge" - - -def test_single_dataset_retriever_external_run_returns_content_and_resources(sqlite_session: Session): - dataset = _persist_dataset(sqlite_session, provider="external", retrieval_model={}) - callback = _TestHitCallback() - metadata_filter_result = ( - {dataset.id: ["doc-a"]}, - {"logical_operator": "and"}, - ) - external_documents = [ - {"content": "first", "metadata": {"document_id": "doc-a"}, "score": 0.9, "title": "Doc A"}, - {"content": "second", "metadata": {"document_id": "doc-b"}, "score": 0.8, "title": "Doc B"}, - ] - - tool = SingleDatasetRetrieverTool( - tenant_id=dataset.tenant_id, - dataset_id=dataset.id, - retrieve_config=_retrieve_config(), - return_resource=True, - retriever_from="dev", - hit_callbacks=[callback], - inputs={"x": 1}, - ) - - with patch.object( - single_retriever_module.DatasetRetrieval, - "get_metadata_filter_condition", - return_value=metadata_filter_result, - ): - with patch.object( - single_retriever_module.ExternalDatasetService, - "fetch_external_knowledge_retrieval", - return_value=external_documents, - ) as fetch_mock: - result = tool.run(session=sqlite_session, query="hello") - - assert result == "first\nsecond" - assert callback.queries == [("hello", dataset.id)] - assert callback.resources is not None - resource_info = callback.resources - assert [item.position for item in resource_info] == [1, 2] - assert resource_info[0].dataset_id == dataset.id - fetch_mock.assert_called_once() - assert fetch_mock.call_args.kwargs["session"] is sqlite_session - - -def test_single_dataset_retriever_returns_empty_when_metadata_filter_finds_no_documents( - sqlite_session: Session, -): - dataset = _persist_dataset(sqlite_session) - tool = SingleDatasetRetrieverTool( - tenant_id=dataset.tenant_id, - dataset_id=dataset.id, - retrieve_config=_retrieve_config(), - return_resource=False, - retriever_from="prod", - hit_callbacks=[_TestHitCallback()], - inputs={}, - ) - - with patch.object( - single_retriever_module.DatasetRetrieval, - "get_metadata_filter_condition", - return_value=({dataset.id: []}, {"logical_operator": "and"}), - ): - with patch.object(single_retriever_module.RetrievalService, "retrieve") as retrieve_mock: - result = tool.run(session=sqlite_session, query="hello") - - assert result == "" - retrieve_mock.assert_not_called() - - -def test_single_dataset_retriever_non_economy_run_sorts_context_and_resources(sqlite_session: Session): - dataset = _persist_dataset( - sqlite_session, - retrieval_model={ - "search_method": "semantic_search", - "score_threshold_enabled": True, - "score_threshold": 0.2, - "reranking_enable": True, - "reranking_model": {"reranking_provider_name": "provider", "reranking_model_name": "model"}, - "reranking_mode": "reranking_model", - "weights": {"vector_setting": {"vector_weight": 0.6}}, - }, - ) - document_low = _persist_document( - sqlite_session, - dataset=dataset, - name="Document Low", - doc_metadata={"lang": "en"}, - ) - document_high = _persist_document( - sqlite_session, - dataset=dataset, - name="Document High", - data_source_type=DataSourceType.NOTION_IMPORT, - doc_metadata={"lang": "fr"}, - ) - callback = _TestHitCallback() - low_segment = DocumentSegment( - tenant_id=dataset.tenant_id, - dataset_id=dataset.id, - document_id=document_low.id, - index_node_id="node-low", - content="raw low", - hit_count=1, - word_count=10, - position=3, - index_node_hash="hash-low", - tokens=10, - created_by=dataset.created_by, - answer="low answer", - status=SegmentStatus.COMPLETED, - completed_at=datetime.now(), - ) - high_segment = DocumentSegment( - tenant_id=dataset.tenant_id, - dataset_id=dataset.id, - document_id=document_high.id, - index_node_id="node-high", - content="raw high", - hit_count=9, - word_count=25, - position=1, - index_node_hash="hash-high", - tokens=25, - created_by=dataset.created_by, - status=SegmentStatus.COMPLETED, - completed_at=datetime.now(), - ) - sqlite_session.add_all([low_segment, high_segment]) - sqlite_session.commit() - records = [ - RetrievalSegments(segment=low_segment, score=0.2, summary="summary low"), - RetrievalSegments(segment=high_segment, score=0.9), - ] - documents = [ - RagDocument(page_content="first", metadata={"doc_id": "node-low", "score": 0.2}), - RagDocument(page_content="second", metadata={"doc_id": "node-high", "score": 0.9}), - ] - tool = SingleDatasetRetrieverTool( - tenant_id=dataset.tenant_id, - dataset_id=dataset.id, - retrieve_config=_retrieve_config(), - return_resource=True, - retriever_from="dev", - hit_callbacks=[callback], - inputs={}, - top_k=2, - ) - - with ( - patch.object( - single_retriever_module.DatasetRetrieval, - "get_metadata_filter_condition", - return_value=(None, None), - ), - patch.object(single_retriever_module.RetrievalService, "retrieve", return_value=documents), - patch.object( - single_retriever_module.RetrievalService, - "format_retrieval_documents", - return_value=records, - ), - patch.object( - single_retriever_module, - "sign_segment_content", - lambda segment, **_kwargs: segment.content.replace("raw", "signed"), - ), - ): - result = tool.run(session=sqlite_session, query="hello") - - assert result == "signed high\nsummary low\nquestion:signed low answer:low answer" - assert callback.documents == documents - assert callback.resources is not None - resource_info = callback.resources - assert [item.position for item in resource_info] == [1, 2] - assert resource_info[0].segment_id == high_segment.id - assert resource_info[0].hit_count == 9 - assert resource_info[1].summary == "summary low" - assert resource_info[1].content == "question:raw low \nanswer:low answer" - - -def test_multi_dataset_retriever_from_dataset_sets_tool_name(): - tool = DatasetMultiRetrieverTool.from_dataset( - dataset_ids=["dataset-1"], - tenant_id="tenant-1", - reranking_provider_name="provider", - reranking_model_name="model", - return_resource=False, - retriever_from="prod", - ) - - assert tool.name == "dataset_tenant_1" - - -def test_multi_dataset_retriever_retriever_returns_early_when_dataset_is_missing( - sqlite_session_factory: sessionmaker[Session], -): - callback = _TestHitCallback() - all_documents: list[RagDocument] = [] - db_session = scoped_session(sqlite_session_factory) - tool = DatasetMultiRetrieverTool( - tenant_id="tenant-1", - dataset_ids=["dataset-1"], - reranking_provider_name="provider", - reranking_model_name="model", - return_resource=False, - retriever_from="prod", - ) - - try: - with patch.object(multi_retriever_module, "db", _DatabaseWithSession(db_session)): - with patch.object(multi_retriever_module.RetrievalService, "retrieve") as retrieve_mock: - result = tool._retriever( - flask_app=_FakeFlaskApp(), - dataset_id=str(uuid.uuid4()), - query="hello", - all_documents=all_documents, - hit_callbacks=[callback], - ) - finally: - db_session.remove() - - assert result == [] - assert all_documents == [] - assert callback.queries == [] - retrieve_mock.assert_not_called() - - -def test_multi_dataset_retriever_retriever_non_economy_uses_retrieval_model( - sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], -): - dataset = _persist_dataset( - sqlite_session, - retrieval_model={ - "search_method": "semantic_search", - "top_k": 6, - "score_threshold_enabled": True, - "score_threshold": 0.4, - "reranking_enable": False, - "reranking_mode": None, - "weights": {"balanced": True}, - }, - ) - callback = _TestHitCallback() - documents = [RagDocument(page_content="retrieved", metadata={"doc_id": "node-1", "score": 0.4})] - all_documents: list[RagDocument] = [] - db_session = scoped_session(sqlite_session_factory) - tool = DatasetMultiRetrieverTool( - tenant_id=dataset.tenant_id, - dataset_ids=[dataset.id], - reranking_provider_name="provider", - reranking_model_name="model", - return_resource=False, - retriever_from="prod", - top_k=2, - ) - - try: - with patch.object(multi_retriever_module, "db", _DatabaseWithSession(db_session)): - with patch.object( - multi_retriever_module.RetrievalService, "retrieve", return_value=documents - ) as retrieve_mock: - tool._retriever( - flask_app=_FakeFlaskApp(), - dataset_id=dataset.id, - query="hello", - all_documents=all_documents, - hit_callbacks=[callback], - ) - finally: - db_session.remove() - - assert all_documents == documents - assert callback.queries == [("hello", dataset.id)] - retrieve_mock.assert_called_once_with( - retrieval_method="semantic_search", - dataset_id=dataset.id, - query="hello", - top_k=6, - score_threshold=0.4, - reranking_model=None, - reranking_mode="reranking_model", - weights={"balanced": True}, - ) - - -def test_multi_dataset_retriever_run_orders_segments_and_returns_resources(sqlite_session: Session): - dataset_one = _persist_dataset(sqlite_session, name="Dataset One") - dataset_two = _persist_dataset(sqlite_session, tenant_id=dataset_one.tenant_id, name="Dataset Two") - document_two = _persist_document( - sqlite_session, - dataset=dataset_one, - name="Doc Two", - data_source_type=DataSourceType.NOTION_IMPORT, - doc_metadata={"p": 2}, - ) - document_one = _persist_document( - sqlite_session, - dataset=dataset_two, - name="Doc One", - doc_metadata={"p": 1}, - ) - callback = _TestHitCallback() - tool = DatasetMultiRetrieverTool( - tenant_id=dataset_one.tenant_id, - dataset_ids=[dataset_one.id, dataset_two.id], - reranking_provider_name="provider", - reranking_model_name="model", - return_resource=True, - retriever_from="dev", - hit_callbacks=[callback], - top_k=2, - score_threshold=0.1, - ) - first_doc = RagDocument(page_content="first", metadata={"doc_id": "node-2", "score": 0.4}) - second_doc = RagDocument(page_content="second", metadata={"doc_id": "node-1", "score": 0.9}) - - def fake_retriever(**kwargs): - if kwargs["dataset_id"] == dataset_one.id: - kwargs["all_documents"].append(first_doc) - else: - kwargs["all_documents"].append(second_doc) - - segment_for_node_2 = DocumentSegment( - tenant_id=dataset_one.tenant_id, - dataset_id=dataset_one.id, - document_id=document_two.id, - index_node_id="node-2", - content="raw two", - hit_count=2, - word_count=20, - position=2, - index_node_hash="hash-2", - tokens=20, - created_by=dataset_one.created_by, - answer="answer two", - status=SegmentStatus.COMPLETED, - completed_at=datetime.now(), - ) - segment_for_node_1 = DocumentSegment( - tenant_id=dataset_two.tenant_id, - dataset_id=dataset_two.id, - document_id=document_one.id, - index_node_id="node-1", - content="raw one", - hit_count=7, - word_count=30, - position=1, - index_node_hash="hash-1", - tokens=30, - created_by=dataset_two.created_by, - status=SegmentStatus.COMPLETED, - completed_at=datetime.now(), - ) - sqlite_session.add_all([segment_for_node_2, segment_for_node_1]) - sqlite_session.commit() - model_manager = ModelManager(provider_manager=_UnusedProviderManager()) - model_instance = _UnusedModelInstance() - rerank_runner = RerankModelRunner( - model_instance, load_upload=upload_reader(sessionmaker(bind=sqlite_session.get_bind())) - ) - fake_current_app = _FakeCurrentApp() - - with ( - patch.object( - multi_retriever_module, - "sign_segment_content", - lambda segment, **_kwargs: segment.content.replace("raw", "signed"), - ), - patch.object(tool, "_retriever", side_effect=fake_retriever) as retriever_mock, - patch.object(multi_retriever_module, "current_app", fake_current_app), - patch.object(multi_retriever_module.threading, "Thread", _ImmediateThread), - patch.object(multi_retriever_module.ModelManager, "for_tenant", return_value=model_manager), - patch.object(model_manager, "get_model_instance", return_value=model_instance), - patch.object(multi_retriever_module, "RerankModelRunner", return_value=rerank_runner) as rerank_runner_class, - patch.object(rerank_runner, "run", return_value=[second_doc, first_doc]), - ): - result = tool.run(session=sqlite_session, query="hello") - - assert result == "signed one\nquestion:signed two answer:answer two" - rerank_runner_class.assert_called_once() - assert rerank_runner_class.call_args.args == (model_instance,) - assert rerank_runner_class.call_args.kwargs["load_upload"].keywords == {"workspace_id": dataset_one.tenant_id} - assert retriever_mock.call_count == 2 - assert callback.documents == [second_doc, first_doc] - assert callback.resources is not None - resource_info = callback.resources - assert [item.position for item in resource_info] == [1, 2] - assert resource_info[0].score == 0.9 - assert resource_info[0].content == "raw one" - assert resource_info[1].score == 0.4 - assert resource_info[1].content == "question:raw two \nanswer:answer two" diff --git a/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py b/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py index b64eea358d81a8..657b65c5234dc7 100644 --- a/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py +++ b/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py @@ -6,12 +6,13 @@ from dataclasses import dataclass from types import SimpleNamespace from typing import Any -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, Mock, patch import pytest from sqlalchemy import Engine, inspect from sqlalchemy.orm import Session, sessionmaker +from core.app.apps.draft_variable_saver import DraftVariableSaverFactory from core.app.entities.app_invoke_entities import InvokeFrom from core.tools.__base.tool_runtime import ToolRuntime from core.tools.entities.common_entities import I18nObject @@ -22,15 +23,15 @@ ToolParameter, ToolProviderType, ) -from core.tools.errors import ToolInvokeError -from core.tools.workflow_as_tool import tool as workflow_tool_module -from core.tools.workflow_as_tool.tool import WorkflowTool +from core.tools.errors import ToolInvokeError, ToolNotFoundError from graphon.file import FILE_MODEL_IDENTITY, FileTransferMethod, FileType from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole from models.base import TypeBase from models.enums import EndUserType from models.model import App, AppMode, EndUser from models.workflow import Workflow, WorkflowType +from services.tools.workflow.tool import WorkflowTool +from services.workflow.execution.ports import WorkflowRuntime TENANT_ID = "00000000-0000-0000-0000-000000000001" OTHER_TENANT_ID = "00000000-0000-0000-0000-000000000002" @@ -49,14 +50,12 @@ class SqliteToolDb: @pytest.fixture def sqlite_tool_db( - monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine, ) -> Iterator[SqliteToolDb]: """Bind service-owned sessions to SQLite.""" models = (App, Workflow, EndUser, Account, Tenant, TenantAccountJoin) TypeBase.metadata.create_all(sqlite_engine, tables=[model.__table__ for model in models]) session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False) - monkeypatch.setattr(workflow_tool_module.session_factory, "create_session", session_maker) with session_maker() as caller_session: yield SqliteToolDb(engine=sqlite_engine, session_maker=session_maker, caller_session=caller_session) @@ -147,7 +146,13 @@ def _persist_workflow(db: SqliteToolDb, *, version: str, workflow_id: str | None return workflow -def _build_tool(*, tenant_id: str = "test_tool", workflow_app_id: str = "app-1", version: str = "1") -> WorkflowTool: +def _build_tool( + *, + tenant_id: str = "test_tool", + workflow_app_id: str = "app-1", + version: str = "1", + workflow_runtime: WorkflowRuntime, +) -> WorkflowTool: entity = ToolEntity( identity=ToolIdentity(author="test", name="test tool", label=I18nObject(en_US="test tool"), provider="test"), parameters=[], @@ -156,6 +161,7 @@ def _build_tool(*, tenant_id: str = "test_tool", workflow_app_id: str = "app-1", ) runtime = ToolRuntime(tenant_id=tenant_id, invoke_from=InvokeFrom.EXPLORE) return WorkflowTool( + draft_variable_saver=Mock(return_value=Mock(spec=DraftVariableSaverFactory)), workflow_app_id=workflow_app_id, workflow_as_tool_id="wf-tool-1", version=version, @@ -163,18 +169,19 @@ def _build_tool(*, tenant_id: str = "test_tool", workflow_app_id: str = "app-1", workflow_call_depth=1, entity=entity, runtime=runtime, + workflow_runtime=workflow_runtime, + queries=workflow_runtime.tools, ) def test_workflow_tool_should_raise_tool_invoke_error_when_result_has_error_field( - monkeypatch: pytest.MonkeyPatch, - sqlite_tool_db: SqliteToolDb, + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime ): """Ensure that WorkflowTool will throw a `ToolInvokeError` exception when `WorkflowAppGenerator.generate` returns a result with `error` key inside the `data` element. """ - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) # needs to patch those methods to avoid database access. monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) @@ -186,23 +193,22 @@ def test_workflow_tool_should_raise_tool_invoke_error_when_result_has_error_fiel # replace `WorkflowAppGenerator.generate` 's return value. monkeypatch.setattr( - "core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", lambda *args, **kwargs: {"data": {"error": "oops"}}, ) with pytest.raises(ToolInvokeError) as exc_info: # WorkflowTool always returns a generator, so we need to iterate to # actually `run` the tool. - list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {})) + list(tool.invoke("test_user", {})) assert exc_info.value.args == ("oops",) def test_workflow_tool_does_not_use_pause_state_config( - monkeypatch: pytest.MonkeyPatch, - sqlite_tool_db: SqliteToolDb, + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime ): """Ensure pause_state_config is passed as None.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) @@ -211,10 +217,12 @@ def test_workflow_tool_does_not_use_pause_state_config( monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate_mock = MagicMock(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock + ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {})) + list(tool.invoke("test_user", {})) call_kwargs = generate_mock.call_args.kwargs assert "pause_state_config" in call_kwargs @@ -222,11 +230,10 @@ def test_workflow_tool_does_not_use_pause_state_config( def test_workflow_tool_passes_parent_trace_context_from_runtime( - monkeypatch: pytest.MonkeyPatch, - sqlite_tool_db: SqliteToolDb, + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime ): """Ensure nested workflow runtime metadata is forwarded as parent trace context.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool.set_parent_trace_context( parent_workflow_run_id="outer-workflow-run-1", parent_node_execution_id="outer-node-execution-1", @@ -239,10 +246,12 @@ def test_workflow_tool_passes_parent_trace_context_from_runtime( monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate_mock = MagicMock(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock + ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {})) + list(tool.invoke("test_user", {})) call_kwargs = generate_mock.call_args.kwargs assert call_kwargs["args"]["parent_trace_context"].model_dump() == { @@ -252,11 +261,10 @@ def test_workflow_tool_passes_parent_trace_context_from_runtime( def test_workflow_tool_passes_parent_trace_session_id( - monkeypatch: pytest.MonkeyPatch, - sqlite_tool_db: SqliteToolDb, + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime ): """Ensure nested workflows inherit the parent observability session ID.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool.entity.parameters = [ ToolParameter.get_simple_instance( name="trace_session_id", @@ -274,10 +282,12 @@ def test_workflow_tool_passes_parent_trace_session_id( monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate_mock = MagicMock(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock + ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {"trace_session_id": "user-input-session"})) + list(tool.invoke("test_user", {"trace_session_id": "user-input-session"})) call_kwargs = generate_mock.call_args.kwargs assert call_kwargs["args"]["inputs"]["trace_session_id"] == "user-input-session" @@ -285,11 +295,10 @@ def test_workflow_tool_passes_parent_trace_session_id( def test_workflow_tool_keeps_user_inputs_named_like_trace_runtime_keys( - monkeypatch: pytest.MonkeyPatch, - sqlite_tool_db: SqliteToolDb, + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime ): """Ensure private trace context does not overwrite same-named workflow inputs.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool.entity.parameters = [ ToolParameter.get_simple_instance( name="outer_workflow_run_id", @@ -316,12 +325,13 @@ def test_workflow_tool_keeps_user_inputs_named_like_trace_runtime_keys( monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate_mock = MagicMock(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock + ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) list( tool.invoke( - sqlite_tool_db.caller_session, "test_user", { "outer_workflow_run_id": "user-workflow-input", @@ -340,11 +350,10 @@ def test_workflow_tool_keeps_user_inputs_named_like_trace_runtime_keys( def test_workflow_tool_can_clear_parent_trace_context( - monkeypatch: pytest.MonkeyPatch, - sqlite_tool_db: SqliteToolDb, + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime ): """Ensure reused WorkflowTool instances do not keep stale parent trace context.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool.set_parent_trace_context( parent_workflow_run_id="outer-workflow-run-1", parent_node_execution_id="outer-node-execution-1", @@ -358,21 +367,22 @@ def test_workflow_tool_can_clear_parent_trace_context( monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate_mock = MagicMock(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock + ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {})) + list(tool.invoke("test_user", {})) call_kwargs = generate_mock.call_args.kwargs assert "parent_trace_context" not in call_kwargs["args"] def test_workflow_tool_can_clear_trace_session_id( - monkeypatch: pytest.MonkeyPatch, - sqlite_tool_db: SqliteToolDb, + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime ): """Ensure reused WorkflowTool instances do not keep stale trace session IDs.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool.set_trace_session_id("session-1") tool.clear_trace_session_id() @@ -383,10 +393,12 @@ def test_workflow_tool_can_clear_trace_session_id( monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate_mock = MagicMock(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock + ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {})) + list(tool.invoke("test_user", {})) call_kwargs = generate_mock.call_args.kwargs assert "trace_session_id" not in call_kwargs["args"] @@ -405,9 +417,11 @@ def test_workflow_tool_omits_parent_trace_context_when_runtime_is_incomplete( monkeypatch: pytest.MonkeyPatch, runtime_parameters: dict[str, Any], sqlite_tool_db: SqliteToolDb, + *, + workflow_runtime: WorkflowRuntime, ): """Ensure incomplete runtime metadata does not leak parent trace context into generator args.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool.runtime.runtime_parameters = runtime_parameters monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) @@ -417,21 +431,22 @@ def test_workflow_tool_omits_parent_trace_context_when_runtime_is_incomplete( monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate_mock = MagicMock(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock + ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {})) + list(tool.invoke("test_user", {})) call_kwargs = generate_mock.call_args.kwargs assert "parent_trace_context" not in call_kwargs["args"] def test_workflow_tool_should_generate_variable_messages_for_outputs( - monkeypatch: pytest.MonkeyPatch, - sqlite_tool_db: SqliteToolDb, + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime ): """Test that WorkflowTool should generate variable messages when there are outputs""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) # Mock workflow outputs mock_outputs = {"result": "success", "count": 42, "data": {"key": "value"}} @@ -446,13 +461,13 @@ def test_workflow_tool_should_generate_variable_messages_for_outputs( # replace `WorkflowAppGenerator.generate` 's return value. monkeypatch.setattr( - "core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", lambda *args, **kwargs: {"data": {"outputs": mock_outputs}}, ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) # Execute tool invocation - messages = list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {})) + messages = list(tool.invoke("test_user", {})) # Verify variable messages variable_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.VARIABLE] @@ -476,11 +491,10 @@ def test_workflow_tool_should_generate_variable_messages_for_outputs( def test_workflow_tool_should_handle_empty_outputs( - monkeypatch: pytest.MonkeyPatch, - sqlite_tool_db: SqliteToolDb, + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime ): """Test that WorkflowTool should handle empty outputs correctly""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) # needs to patch those methods to avoid database access. monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) @@ -492,13 +506,13 @@ def test_workflow_tool_should_handle_empty_outputs( # replace `WorkflowAppGenerator.generate` 's return value. monkeypatch.setattr( - "core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", lambda *args, **kwargs: {"data": {}}, ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) # Execute tool invocation - messages = list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {})) + messages = list(tool.invoke("test_user", {})) # Verify generated messages # Should contain: 0 variable messages + 1 text message + 1 JSON message = 2 messages @@ -530,9 +544,9 @@ def test_workflow_tool_should_handle_empty_outputs( ("dict_var", {"key": "value"}), ], ) -def test_create_variable_message(var_name, var_value): +def test_create_variable_message(var_name, var_value, *, workflow_runtime: WorkflowRuntime): """Create variable messages for multiple value types.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) message = tool.create_variable_message(var_name, var_value) @@ -542,9 +556,9 @@ def test_create_variable_message(var_name, var_value): assert message.message.stream is False -def test_create_file_message_should_include_file_marker(): +def test_create_file_message_should_include_file_marker(*, workflow_runtime: WorkflowRuntime): """Ensure file message includes marker and meta payload.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) file_obj = object() message = tool.create_file_message(file_obj) # type: ignore[arg-type] @@ -554,7 +568,7 @@ def test_create_file_message_should_include_file_marker(): assert message.meta == {"file": file_obj} -def test_resolve_user_from_database_falls_back_to_end_user(sqlite_tool_db: SqliteToolDb): +def test_resolve_tenant_user_falls_back_to_end_user(sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime): """Ensure worker context can resolve EndUser when Account is missing.""" _persist_tenant(sqlite_tool_db) end_user = _persist_end_user(sqlite_tool_db) @@ -564,31 +578,33 @@ def test_resolve_user_from_database_falls_back_to_end_user(sqlite_tool_db: Sqlit tenant_id=OTHER_TENANT_ID, ) - tool = _build_tool(tenant_id=TENANT_ID) + tool = _build_tool(tenant_id=TENANT_ID, workflow_runtime=workflow_runtime) tool.runtime.invoke_from = InvokeFrom.SERVICE_API - resolved_user = tool._resolve_user_from_database(user_id=end_user.id) + resolved_user = tool._resolve_user(user_id=end_user.id) assert isinstance(resolved_user, EndUser) assert resolved_user.id == end_user.id assert resolved_user.tenant_id == TENANT_ID assert inspect(resolved_user).detached is True - assert tool._resolve_user_from_database(user_id=other_tenant_end_user.id) is None + assert tool._resolve_user(user_id=other_tenant_end_user.id) is None -def test_resolve_user_from_database_returns_none_when_no_tenant(sqlite_tool_db: SqliteToolDb): +def test_resolve_tenant_user_returns_none_when_no_tenant( + sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +): """Return None if tenant cannot be found in worker context.""" - tool = _build_tool(tenant_id=OTHER_TENANT_ID) + tool = _build_tool(tenant_id=OTHER_TENANT_ID, workflow_runtime=workflow_runtime) tool.runtime.invoke_from = InvokeFrom.SERVICE_API - resolved_user = tool._resolve_user_from_database(user_id="any") + resolved_user = tool._resolve_user(user_id="any") assert resolved_user is None -def test_workflow_tool_provider_type_and_fork_runtime(): +def test_workflow_tool_provider_type_and_fork_runtime(*, workflow_runtime: WorkflowRuntime): """Verify provider type and forked runtime behavior.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) assert tool.tool_provider_type() == ToolProviderType.WORKFLOW assert tool.latest_usage.total_tokens == 0 @@ -632,39 +648,38 @@ def test_extract_usage_from_nested(): def test_invoke_raises_when_user_not_found( - monkeypatch: pytest.MonkeyPatch, - sqlite_tool_db: SqliteToolDb, + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime ): """Raise ToolInvokeError when user resolution fails.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: None) with pytest.raises(ToolInvokeError, match="User not found"): - list(tool.invoke(sqlite_tool_db.caller_session, "missing", {})) + list(tool.invoke("missing", {})) -def test_resolve_user_from_database_returns_account(sqlite_tool_db: SqliteToolDb): +def test_resolve_tenant_user_returns_account(sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime): """Resolve Account and set tenant in worker context.""" tenant = _persist_tenant(sqlite_tool_db) account = _persist_account(sqlite_tool_db) - tool = _build_tool(tenant_id=TENANT_ID) + tool = _build_tool(tenant_id=TENANT_ID, workflow_runtime=workflow_runtime) - resolved = tool._resolve_user_from_database(user_id=account.id) + resolved = tool._resolve_user(user_id=account.id) assert isinstance(resolved, Account) assert resolved.id == account.id assert resolved.current_tenant_id == tenant.id assert inspect(resolved).detached is True -def test_get_workflow_and_get_app_db_branches(sqlite_tool_db: SqliteToolDb): +def test_get_workflow_and_get_app_db_branches(sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime): """Cover workflow/app retrieval branches and error cases.""" app = _persist_app(sqlite_tool_db) specific_workflow = _persist_workflow(sqlite_tool_db, version="1") latest_workflow = _persist_workflow(sqlite_tool_db, version="2") _persist_workflow(sqlite_tool_db, version=Workflow.VERSION_DRAFT) - tool = _build_tool(tenant_id=TENANT_ID, workflow_app_id=APP_ID) + tool = _build_tool(tenant_id=TENANT_ID, workflow_app_id=APP_ID, workflow_runtime=workflow_runtime) latest = tool._get_workflow(APP_ID, "") specific = tool._get_workflow(APP_ID, "1") @@ -677,15 +692,15 @@ def test_get_workflow_and_get_app_db_branches(sqlite_tool_db: SqliteToolDb): assert inspect(specific).detached is True assert inspect(resolved_app).detached is True - with pytest.raises(ValueError, match="workflow not found"): + with pytest.raises(ToolNotFoundError, match="workflow not found"): tool._get_workflow(APP_ID, "missing") - with pytest.raises(ValueError, match="app not found"): + with pytest.raises(ToolNotFoundError, match="app not found"): tool._get_app("00000000-0000-0000-0000-000000000099") -def _setup_transform_args_tool(monkeypatch: pytest.MonkeyPatch) -> WorkflowTool: +def _setup_transform_args_tool(monkeypatch: pytest.MonkeyPatch, *, workflow_runtime: WorkflowRuntime) -> WorkflowTool: """Build a WorkflowTool and stub merged runtime parameters for files/query.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) files_param = ToolParameter.get_simple_instance( name="files", llm_description="files", @@ -705,9 +720,9 @@ def _setup_transform_args_tool(monkeypatch: pytest.MonkeyPatch) -> WorkflowTool: return tool -def test_transform_args_valid_files(monkeypatch: pytest.MonkeyPatch): +def test_transform_args_valid_files(monkeypatch: pytest.MonkeyPatch, *, workflow_runtime: WorkflowRuntime): """Transform args into parameters and files payloads.""" - tool = _setup_transform_args_tool(monkeypatch) + tool = _setup_transform_args_tool(monkeypatch, workflow_runtime=workflow_runtime) build_file_from_stored_mapping = MagicMock( side_effect=[ SimpleNamespace( @@ -731,7 +746,7 @@ def test_transform_args_valid_files(monkeypatch: pytest.MonkeyPatch): ] ) monkeypatch.setattr( - "core.tools.workflow_as_tool.tool.build_file_from_stored_mapping", + "services.tools.workflow.tool.build_file_from_stored_mapping", build_file_from_stored_mapping, ) @@ -769,9 +784,9 @@ def test_transform_args_valid_files(monkeypatch: pytest.MonkeyPatch): assert all(call.kwargs["tenant_id"] == "test_tool" for call in build_file_from_stored_mapping.call_args_list) -def test_transform_args_invalid_files(monkeypatch: pytest.MonkeyPatch): +def test_transform_args_invalid_files(monkeypatch: pytest.MonkeyPatch, *, workflow_runtime: WorkflowRuntime): """Ignore invalid file entries while keeping params.""" - tool = _setup_transform_args_tool(monkeypatch) + tool = _setup_transform_args_tool(monkeypatch, workflow_runtime=workflow_runtime) invalid_params, invalid_files = tool._transform_args({"query": "hello", "files": [{"invalid": True}]}) assert invalid_params == {"query": "hello"} assert invalid_files == [] @@ -779,11 +794,10 @@ def test_transform_args_invalid_files(monkeypatch: pytest.MonkeyPatch): @pytest.mark.parametrize("empty_value", [None, "", [], [None], [""]]) def test_transform_args_normalizes_optional_files_parameter( - monkeypatch: pytest.MonkeyPatch, - empty_value: Any, + monkeypatch: pytest.MonkeyPatch, empty_value: Any, *, workflow_runtime: WorkflowRuntime ): """Pass optional workflow file-list inputs as an empty list when no files were provided.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) images_param = ToolParameter.get_simple_instance( name="images", llm_description="images", @@ -800,11 +814,10 @@ def test_transform_args_normalizes_optional_files_parameter( def test_workflow_tool_invocation_normalizes_optional_files_parameter( - monkeypatch: pytest.MonkeyPatch, - sqlite_tool_db: SqliteToolDb, + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime ): """Ensure casted empty FILES values do not reach workflow input validation as [None].""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) images_param = ToolParameter.get_simple_instance( name="images", llm_description="images", @@ -820,22 +833,24 @@ def test_workflow_tool_invocation_normalizes_optional_files_parameter( monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate_mock = MagicMock(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate_mock + ) - list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {"images": None})) + list(tool.invoke("test_user", {"images": None})) call_kwargs = generate_mock.call_args.kwargs assert call_kwargs["args"]["inputs"]["images"] == [] -def test_extract_files(): +def test_extract_files(*, workflow_runtime: WorkflowRuntime): """Extract file outputs into result and file list.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) built_files = [ SimpleNamespace(id="file-1"), SimpleNamespace(id="file-2"), ] - with patch("core.tools.workflow_as_tool.tool.build_from_mapping", side_effect=built_files): + with patch("services.tools.workflow.tool.build_from_mapping", side_effect=built_files): outputs = { "attachments": [ { @@ -857,10 +872,242 @@ def test_extract_files(): assert len(extracted_files) == 2 -def test_update_file_mapping(): +def test_update_file_mapping(*, workflow_runtime: WorkflowRuntime): """Map tool/local file transfer methods into output shape.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool_file = tool._update_file_mapping({"transfer_method": "tool_file", "related_id": "tool-1"}) assert tool_file["tool_file_id"] == "tool-1" local_file = tool._update_file_mapping({"transfer_method": "local_file", "related_id": "upload-1"}) assert local_file["upload_file_id"] == "upload-1" + + +@pytest.mark.parametrize("invoke_from", [InvokeFrom.DEBUGGER, InvokeFrom.EXPLORE]) +@pytest.mark.parametrize("entry", ["transform", "workflow"]) +def test_composed_tool_fork_runs_real_generator_with_injected_draft_saver( + monkeypatch: pytest.MonkeyPatch, + sqlite_tool_db: SqliteToolDb, + invoke_from: InvokeFrom, + entry: str, + workflow_runtime: WorkflowRuntime, + *, + workflow_queries, +) -> None: + from flask import Flask + from sqlalchemy import select + + from core.app.apps import base_app_queue_manager + from core.app.entities.app_invoke_entities import UserFrom, build_dify_run_context + from core.ops.ops_trace_manager import TraceQueueManager + from extensions.application_services.workflow_variables import build_workflow_variable_service + from graphon.nodes import BuiltinNodeTypes + from graphon.nodes.tool.entities import ToolNodeData + from graphon.runtime import GraphRuntimeState, VariablePool + from models.tools import WorkflowToolProvider + from models.workflow import WorkflowDraftVariable + from services.tools import tools_transform_service + from services.workflow.execution.adapters.node_factory import DifyGraphInitContext, DifyNodeFactory + from services.workflow.execution.adapters.workflow import app_generator as generator_module + from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator + + account = _persist_account(sqlite_tool_db) + app = _persist_app(sqlite_tool_db) + workflow = _persist_workflow(sqlite_tool_db, version="1") + workflow.graph = json.dumps( + {"nodes": [{"id": "start", "data": {"type": "start", "title": "Start", "variables": []}}], "edges": []} + ) + provider = WorkflowToolProvider( + name="nested_workflow", + label="Nested workflow", + icon="icon.svg", + app_id=app.id, + version="1", + user_id=account.id, + tenant_id=app.tenant_id, + description="", + parameter_configuration="[]", + ) + sqlite_tool_db.caller_session.add(provider) + sqlite_tool_db.caller_session.commit() + variables = build_workflow_variable_service(database_client=sqlite_tool_db.session_maker) + saver = variables.saver_factory + resolver = workflow_runtime.agent_bindings + if entry == "transform": + controller = tools_transform_service.ToolTransformService.workflow_provider_to_controller( + provider, + draft_variable_saver=saver, + workflow_runtime=workflow_runtime, + queries=workflow_queries, + ) + original = controller.get_tools(app.tenant_id)[0] + tool = original.fork_tool_runtime(ToolRuntime(tenant_id=app.tenant_id, invoke_from=invoke_from)) + else: + run_context = build_dify_run_context( + tenant_id=app.tenant_id, + app_id=app.id, + user_id=account.id, + user_from=UserFrom.ACCOUNT, + invoke_from=invoke_from, + ) + state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0) + factory = DifyNodeFactory.from_graph_init_context( + graph_init_context=DifyGraphInitContext( + workflow_id=workflow.id, graph_config=workflow.graph_dict, run_context=run_context, call_depth=0 + ), + graph_runtime_state=state, + draft_variable_saver=saver, + agent_binding_resolver=resolver, + workflow_runtime=workflow_runtime, + ).with_runtime_state(state) + runtime = factory._tool_runtime + handle = runtime.get_runtime( + node_id="outer-tool-node", + node_data=ToolNodeData( + title="Nested workflow", + provider_id=provider.id, + provider_type="workflow", + provider_name=provider.name, + tool_name=provider.name, + tool_label=provider.label, + tool_configurations={}, + tool_parameters={}, + ), + variable_pool=state.variable_pool, + ) + tool = runtime._tool_from_handle(handle) + assert isinstance(tool, WorkflowTool) + assert tool._draft_variable_saver is saver + + monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", lambda *_args, **_kwargs: None) + monkeypatch.setattr(generator_module, "TraceQueueManager", Mock(return_value=Mock(spec=TraceQueueManager))) + monkeypatch.setattr(WorkflowAppGenerator, "_generate_worker", staticmethod(lambda **_kwargs: None)) + outputs = {"result": "nested result"} + + def handle_response( + self: WorkflowAppGenerator, *, draft_var_saver_factory: DraftVariableSaverFactory, **_kwargs: Any + ) -> dict[str, Any]: + draft_var_saver_factory(app.id, "nested-node", BuiltinNodeTypes.LLM, "nested-execution").save( + process_data=None, outputs=outputs + ) + return {"data": {"outputs": outputs}} + + monkeypatch.setattr(WorkflowAppGenerator, "_handle_response", handle_response) + monkeypatch.setattr( + generator_module.WorkflowAppGenerateResponseConverter, "convert", lambda *, response, invoke_from: response + ) + with Flask(__name__).app_context(): + messages = list(tool._invoke(user_id=account.id, tool_parameters={})) + assert messages + with sqlite_tool_db.session_maker() as session: + saved = session.scalar(select(WorkflowDraftVariable).where(WorkflowDraftVariable.node_id == "nested-node")) + if invoke_from == InvokeFrom.DEBUGGER: + assert saved is not None + assert saved.user_id == account.id + assert saved.app_id == app.id + assert saved.get_value().value == outputs["result"] + else: + assert saved is None + + +@pytest.mark.parametrize("invoke_from", [InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API]) +def test_nested_workflow_preserves_admitted_end_user_and_resolves_agent_binding( + sqlite_tool_db, + workflow_runtime, + monkeypatch, + invoke_from, + config_overrides, +): + """Exercise tool admission, generator, worker, runner and node construction across apps.""" + from flask import Flask + + from core.ops.ops_trace_manager import OpsTraceManager + from enums.agent import WorkflowAgentBindingType + from models.tools import WorkflowToolProvider + from services.tools.workflow.provider import WorkflowToolProviderController + from services.workflow.execution.adapters.workflow import app_generator as generator_module + from services.workflow.execution.adapters.workflow_entry import WorkflowEntry + from tests.unit_tests.core.workflow.nodes.agent_v2.test_binding_resolver import _agent, _binding, _snapshot + from tests.unit_tests.model_factories import make_app + + config_overrides(AGENT_BACKEND_USE_FAKE=True) + account = _persist_account(sqlite_tool_db) + child = _persist_app(sqlite_tool_db) + outer = make_app(app_id=str(uuid.uuid4()), tenant_id=child.tenant_id) + user = _persist_end_user(sqlite_tool_db) + user.app_id = outer.id + user.session_id = "customer-42" + workflow = _persist_workflow(sqlite_tool_db, version="1") + workflow.graph = json.dumps( + { + "nodes": [ + {"id": "start", "data": {"type": "start", "title": "Start", "variables": []}}, + { + "id": "agent-node", + "data": {"type": "agent", "version": "2", "title": "Agent", "agent_node_kind": "dify_agent"}, + }, + ], + "edges": [{"source": "start", "target": "agent-node", "sourceHandle": "source", "targetHandle": "target"}], + } + ) + agent = _agent(tenant_id=child.tenant_id) + sqlite_tool_db.caller_session.add_all([outer, agent]) + sqlite_tool_db.caller_session.flush() + snapshot = _snapshot(tenant_id=child.tenant_id, agent_id=agent.id) + sqlite_tool_db.caller_session.add(snapshot) + sqlite_tool_db.caller_session.flush() + ids = {"tenant_id": child.tenant_id, "app_id": child.id, "workflow_id": workflow.id, "node_id": "agent-node"} + sqlite_tool_db.caller_session.add( + _binding( + ids=ids, + agent_id=agent.id, + snapshot_id=snapshot.id, + binding_type=WorkflowAgentBindingType.INLINE_AGENT, + ) + ) + provider = WorkflowToolProvider( + tenant_id=child.tenant_id, + app_id=child.id, + user_id=account.id, + name="child", + label="Child", + icon="", + description="", + version="1", + parameter_configuration="[]", + ) + sqlite_tool_db.caller_session.add(provider) + sqlite_tool_db.caller_session.commit() + controller = WorkflowToolProviderController.from_db( + provider, + queries=workflow_runtime.tools, + workflow_runtime=workflow_runtime, + draft_variable_saver=None, + ) + tool = controller.get_tools(child.tenant_id)[0].fork_tool_runtime( + ToolRuntime(tenant_id=child.tenant_id, invoke_from=invoke_from) + ) + monkeypatch.setattr(OpsTraceManager, "get_ops_trace_instance", lambda *_args: None) + resolved = [] + + def probe_engine(entry): + engine = entry.graph_engine + pool = engine._graph_runtime_state.variable_pool + node = engine._graph.nodes["agent-node"] + bundle = node._binding_resolver.resolve(**ids) + resolved.append((pool.get(["sys", "user_id"]).value, bundle.snapshot.id)) + return iter(()) + + monkeypatch.setattr(WorkflowEntry, "run", probe_engine) + + def execute_worker(*, worker, respond): + worker.run() + assert resolved == [("customer-42", snapshot.id)] + return {"data": {"outputs": {"caller": resolved[0][0]}}} + + monkeypatch.setattr(generator_module, "generate_response", execute_worker) + with Flask(__name__).app_context(): + result = list(tool._invoke(user_id=user.id, tool_parameters={}, app_id=outer.id)) + assert any(item.type == ToolInvokeMessage.MessageType.VARIABLE for item in result) + # Tool admission must still reject an external identity from another tenant. + foreign_user = _persist_end_user(sqlite_tool_db, end_user_id=str(uuid.uuid4()), tenant_id=OTHER_TENANT_ID) + with pytest.raises(ToolInvokeError, match="User not found"): + list(tool._invoke(user_id=foreign_user.id, tool_parameters={}, app_id=outer.id)) diff --git a/api/tests/unit_tests/core/workflow/nodes/agent/test_message_transformer.py b/api/tests/unit_tests/core/workflow/nodes/agent/test_message_transformer.py index 26204b040ca002..86b2237d1a44dc 100644 --- a/api/tests/unit_tests/core/workflow/nodes/agent/test_message_transformer.py +++ b/api/tests/unit_tests/core/workflow/nodes/agent/test_message_transformer.py @@ -34,7 +34,7 @@ def _message_stream(messages: list[ToolInvokeMessage]) -> Generator[ToolInvokeMe def _run_transform(messages: list[ToolInvokeMessage]) -> tuple[str, ArrayFileSegment]: events = list( - AgentMessageTransformer().transform( + AgentMessageTransformer(lambda _name, default: (default, None)).transform( messages=_message_stream(messages), tool_info={}, parameters_for_log={}, @@ -54,7 +54,7 @@ def _run_transform(messages: list[ToolInvokeMessage]) -> tuple[str, ArrayFileSeg def test_transform_passes_conversation_id_to_tool_file_message_transformer() -> None: messages = _message_stream([]) - transformer = AgentMessageTransformer() + transformer = AgentMessageTransformer(lambda _name, default: (default, None)) with patch.object(ToolFileMessageTransformer, "transform_tool_invoke_messages", return_value=iter(())) as transform: result = list( @@ -240,7 +240,7 @@ def test_transform_closes_think_tag_when_tool_log_interrupts() -> None: def test_transform_streams_close_tag_before_post_tool_text() -> None: events = list( - AgentMessageTransformer().transform( + AgentMessageTransformer(lambda _name, default: (default, None)).transform( messages=_message_stream( [ _text("need to search"), diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py index f4632563260e8e..e5a0454e7a3471 100644 --- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py @@ -2,7 +2,7 @@ from datetime import UTC, datetime from types import SimpleNamespace from typing import cast -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, Mock, patch from uuid import UUID, uuid4 import pytest @@ -35,15 +35,9 @@ ) from core.app.entities.app_invoke_entities import DIFY_RUN_CONTEXT_KEY, DifyRunContext, InvokeFrom, UserFrom from core.workflow.file_reference import build_file_reference -from core.workflow.nodes.agent_v2 import DifyAgentNode from core.workflow.nodes.agent_v2.ask_human_resume import AskHumanResumeOutcome -from core.workflow.nodes.agent_v2.binding_resolver import WorkflowAgentBindingBundle, WorkflowAgentBindingResolver from core.workflow.nodes.agent_v2.entities import DifyAgentNodeData from core.workflow.nodes.agent_v2.output_adapter import WorkflowAgentOutputAdapter -from core.workflow.nodes.agent_v2.runtime_request_builder import ( - WorkflowAgentRuntimeBuildContext, - WorkflowAgentRuntimeRequestBuilder, -) from core.workflow.nodes.agent_v2.session_store import ( StoredWorkflowAgentSession, WorkflowAgentSessionScope, @@ -71,13 +65,22 @@ DeclaredOutputType, WorkflowNodeJobConfig, ) +from models.agent_runtime_contracts import WorkflowAgentBindingBundle +from models.tool_runtime_contracts import WorkflowToolQueries +from repositories.agent.runtime_repository import WorkflowAgentBindingResolver from services.agent.workspace_service import AgentWorkspaceNotFoundError +from services.workflow.execution.adapters.agent_v2.agent_node import DifyAgentNode +from services.workflow.execution.adapters.agent_v2.dify_tools_builder import WorkflowAgentDifyToolsBuilder +from services.workflow.execution.adapters.agent_v2.runtime_request_builder import ( + WorkflowAgentRuntimeBuildContext, + WorkflowAgentRuntimeRequestBuilder, +) @pytest.fixture(autouse=True) def _stub_model_context_window(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( - "core.workflow.nodes.agent_v2.runtime_request_builder.resolve_model_context_window", + "services.workflow.execution.adapters.agent_v2.runtime_request_builder.resolve_model_context_window", lambda **_kwargs: None, ) @@ -404,6 +407,7 @@ def cancel_run(self, run_id: str, request: CancelRunRequest | None = None) -> Ca def _node( *, + tool_providers, scenario: FakeAgentBackendScenario = FakeAgentBackendScenario.SUCCESS, session_store: FakeSessionStore | None = None, declared_outputs: list[dict[str, object]] | None = None, @@ -467,7 +471,12 @@ def is_accessible_file_mapping(self, *, file_id: str, tenant_id: str, transfer_m ), ), binding_resolver=binding_resolver, - runtime_request_builder=runtime_request_builder or WorkflowAgentRuntimeRequestBuilder(), + runtime_request_builder=runtime_request_builder + or WorkflowAgentRuntimeRequestBuilder( + dify_tools_builder=WorkflowAgentDifyToolsBuilder( + tool_providers=tool_providers, workflow_queries=Mock(spec=WorkflowToolQueries) + ) + ), agent_backend_client=client, event_adapter=AgentBackendRunEventAdapter(), output_adapter=WorkflowAgentOutputAdapter(), @@ -496,8 +505,8 @@ def test_extract_variable_selector_to_variable_mapping_uses_frontend_agent_task_ } -def test_agent_node_run_maps_successful_agent_backend_run_to_node_result(): - events = list(_node(agent_backend_client=PlainTextOutputBackendClient())._run()) +def test_agent_node_run_maps_successful_agent_backend_run_to_node_result(tool_providers): + events = list(_node(tool_providers=tool_providers, agent_backend_client=PlainTextOutputBackendClient())._run()) assert len(events) == 1 result = cast(StreamCompletedEvent, events[0]).node_run_result @@ -513,9 +522,10 @@ def test_agent_node_run_maps_successful_agent_backend_run_to_node_result(): assert "output_type_check" not in agent_log -def test_agent_node_structured_success_preserves_text_and_checks_only_custom_outputs(): +def test_agent_node_structured_success_preserves_text_and_checks_only_custom_outputs(tool_providers): events = list( _node( + tool_providers=tool_providers, declared_outputs=[{"name": "summary", "type": DeclaredOutputType.STRING}], agent_backend_client=FileOutputBackendClient( output_payload={"text": "hello agent", "summary": "Short summary"} @@ -540,9 +550,10 @@ def test_agent_node_structured_success_preserves_text_and_checks_only_custom_out } -def test_agent_node_structured_output_type_failure_stops_the_node(): +def test_agent_node_structured_output_type_failure_stops_the_node(tool_providers): events = list( _node( + tool_providers=tool_providers, declared_outputs=[{"name": "summary", "type": DeclaredOutputType.STRING}], agent_backend_client=FileOutputBackendClient(output_payload={"summary": 42}), )._run() @@ -555,11 +566,11 @@ def test_agent_node_structured_output_type_failure_stops_the_node(): assert agent_log["output_failure_decision"] == "fail_node" -def test_agent_node_uses_resolved_backend_binding_before_backend_invocation() -> None: +def test_agent_node_uses_resolved_backend_binding_before_backend_invocation(tool_providers) -> None: client = FakeAgentBackendRunClient() store = FakeSessionStore(binding_id="binding-2", backend_binding_ref="backend-binding-2") - events = list(_node(agent_backend_client=client, session_store=store)._run()) + events = list(_node(tool_providers=tool_providers, agent_backend_client=client, session_store=store)._run()) assert len(events) == 1 assert client.request is not None @@ -569,7 +580,7 @@ def test_agent_node_uses_resolved_backend_binding_before_backend_invocation() -> assert len(store.resolved_scopes) == 1 -def test_agent_node_resume_resolves_the_generation_from_the_persisted_execution() -> None: +def test_agent_node_resume_resolves_the_generation_from_the_persisted_execution(tool_providers) -> None: binding_resolver = FakeBindingResolver() store = FakeSessionStore() store.loaded_session = StoredWorkflowAgentSession( @@ -589,7 +600,7 @@ def test_agent_node_resume_resolves_the_generation_from_the_persisted_execution( backend_binding_ref="backend-binding-1", session_snapshot=None, ) - node = _node(binding_resolver=binding_resolver, session_store=store) + node = _node(tool_providers=tool_providers, binding_resolver=binding_resolver, session_store=store) events = list(node._run()) @@ -600,13 +611,13 @@ def test_agent_node_resume_resolves_the_generation_from_the_persisted_execution( assert binding_resolver.calls[0]["conversation_id"] == "conversation-1" -def test_agent_node_maps_persisted_participant_lookup_error_to_node_failure() -> None: +def test_agent_node_maps_persisted_participant_lookup_error_to_node_failure(tool_providers) -> None: class _UnavailableParticipantStore(FakeSessionStore): def load_existing_node_execution_scope(self, **kwargs: object) -> WorkflowAgentSessionScope | None: del kwargs raise AgentWorkspaceNotFoundError("Workflow node participant Binding is unavailable") - events = list(_node(session_store=_UnavailableParticipantStore())._run()) + events = list(_node(tool_providers=tool_providers, session_store=_UnavailableParticipantStore())._run()) assert len(events) == 1 result = cast(StreamCompletedEvent, events[0]).node_run_result @@ -615,10 +626,14 @@ def load_existing_node_execution_scope(self, **kwargs: object) -> WorkflowAgentS assert result.error_type == "agent_workflow_node_runtime_error" -def test_agent_node_passes_execution_id_to_session_store_and_runtime_request_builder() -> None: +def test_agent_node_passes_execution_id_to_session_store_and_runtime_request_builder(tool_providers) -> None: store = FakeSessionStore() - request_builder = WorkflowAgentRuntimeRequestBuilder() - node = _node(session_store=store, runtime_request_builder=request_builder) + request_builder = WorkflowAgentRuntimeRequestBuilder( + dify_tools_builder=WorkflowAgentDifyToolsBuilder( + tool_providers=tool_providers, workflow_queries=Mock(spec=WorkflowToolQueries) + ) + ) + node = _node(tool_providers=tool_providers, session_store=store, runtime_request_builder=request_builder) execution_id = node.execution_id with patch.object(request_builder, "build", wraps=request_builder.build) as build: @@ -635,8 +650,8 @@ def test_agent_node_passes_execution_id_to_session_store_and_runtime_request_bui assert context.node_execution_id == execution_id -def test_agent_node_run_ignores_agent_message_delta_until_terminal_result(): - events = list(_node(agent_backend_client=AgentMessageDeltaBackendClient())._run()) +def test_agent_node_run_ignores_agent_message_delta_until_terminal_result(tool_providers): + events = list(_node(tool_providers=tool_providers, agent_backend_client=AgentMessageDeltaBackendClient())._run()) assert len(events) == 1 result = cast(StreamCompletedEvent, events[0]).node_run_result @@ -648,7 +663,7 @@ def test_agent_node_run_ignores_agent_message_delta_until_terminal_result(): assert agent_backend["agent_message_delta_length"] == len("hello ") -def test_agent_node_run_normalizes_declared_file_output_with_canonical_mapping(): +def test_agent_node_run_normalizes_declared_file_output_with_canonical_mapping(tool_providers): tool_reference = build_file_reference(record_id="tool-file-1") with patch( "core.workflow.nodes.agent_v2.output_adapter.build_from_mapping", @@ -656,6 +671,7 @@ def test_agent_node_run_normalizes_declared_file_output_with_canonical_mapping() ): events = list( _node( + tool_providers=tool_providers, declared_outputs=[{"name": "report", "type": DeclaredOutputType.FILE}], agent_backend_client=FileOutputBackendClient( output_payload={"report": {"transfer_method": "tool_file", "reference": tool_reference}} @@ -669,7 +685,7 @@ def test_agent_node_run_normalizes_declared_file_output_with_canonical_mapping() assert report.value.reference == tool_reference -def test_agent_node_run_normalizes_declared_datasource_file_output_with_canonical_mapping(): +def test_agent_node_run_normalizes_declared_datasource_file_output_with_canonical_mapping(tool_providers): datasource_reference = build_file_reference(record_id="datasource-file-1") with patch( "core.workflow.nodes.agent_v2.output_adapter.build_from_mapping", @@ -680,6 +696,7 @@ def test_agent_node_run_normalizes_declared_datasource_file_output_with_canonica ): events = list( _node( + tool_providers=tool_providers, declared_outputs=[{"name": "report", "type": DeclaredOutputType.FILE}], agent_backend_client=FileOutputBackendClient( output_payload={"report": {"transfer_method": "datasource_file", "reference": datasource_reference}} @@ -694,11 +711,12 @@ def test_agent_node_run_normalizes_declared_datasource_file_output_with_canonica assert report.value.reference == datasource_reference -def test_agent_node_run_normalizes_declared_remote_url_file_output_with_canonical_mapping(): +def test_agent_node_run_normalizes_declared_remote_url_file_output_with_canonical_mapping(tool_providers): remote_url = "https://example.com/report.pdf" events = list( _node( + tool_providers=tool_providers, declared_outputs=[{"name": "report", "type": DeclaredOutputType.FILE}], agent_backend_client=FileOutputBackendClient( output_payload={"report": {"transfer_method": "remote_url", "url": remote_url}} @@ -713,7 +731,7 @@ def test_agent_node_run_normalizes_declared_remote_url_file_output_with_canonica assert report.value.remote_url == remote_url -def test_agent_node_run_normalizes_declared_array_file_output_with_canonical_mappings(): +def test_agent_node_run_normalizes_declared_array_file_output_with_canonical_mappings(tool_providers): first_reference = build_file_reference(record_id="tool-file-1") second_reference = build_file_reference(record_id="tool-file-2") with patch( @@ -725,6 +743,7 @@ def test_agent_node_run_normalizes_declared_array_file_output_with_canonical_map ): events = list( _node( + tool_providers=tool_providers, declared_outputs=[ { "name": "attachments", @@ -749,9 +768,11 @@ def test_agent_node_run_normalizes_declared_array_file_output_with_canonical_map assert [item.reference for item in attachments.value] == [first_reference, second_reference] -def test_agent_node_run_maps_failed_agent_backend_run_to_node_result(): +def test_agent_node_run_maps_failed_agent_backend_run_to_node_result(tool_providers): store = FakeSessionStore() - events = list(_node(scenario=FakeAgentBackendScenario.FAILED, session_store=store)._run()) + events = list( + _node(tool_providers=tool_providers, scenario=FakeAgentBackendScenario.FAILED, session_store=store)._run() + ) assert len(events) == 1 result = cast(StreamCompletedEvent, events[0]).node_run_result @@ -761,11 +782,11 @@ def test_agent_node_run_maps_failed_agent_backend_run_to_node_result(): assert store.saved[0][2] == CompositorSessionSnapshot(layers=[]) -def test_agent_node_saves_success_snapshot_and_reuses_existing_snapshot(): +def test_agent_node_saves_success_snapshot_and_reuses_existing_snapshot(tool_providers): existing_snapshot = CompositorSessionSnapshot(layers=[]) store = FakeSessionStore(snapshot=existing_snapshot) client = FakeAgentBackendRunClient() - node = _node(agent_backend_client=client, session_store=store) + node = _node(tool_providers=tool_providers, agent_backend_client=client, session_store=store) events = list(node._run()) @@ -782,13 +803,13 @@ def test_agent_node_saves_success_snapshot_and_reuses_existing_snapshot(): assert client.request.session_snapshot is existing_snapshot -def test_agent_node_run_when_session_store_save_raises_records_persist_error_in_metadata(): +def test_agent_node_run_when_session_store_save_raises_records_persist_error_in_metadata(tool_providers): """A DB-side write failure must not crash the node; it should set ``session_snapshot_persist_error`` in the agent_backend metadata so the incident is observable from the workflow_node_executions record.""" store = ExplodingSessionStore() - events = list(_node(session_store=store)._run()) + events = list(_node(tool_providers=tool_providers, session_store=store)._run()) assert len(events) == 1 result = cast(StreamCompletedEvent, events[0]).node_run_result @@ -799,7 +820,7 @@ def test_agent_node_run_when_session_store_save_raises_records_persist_error_in_ @pytest.mark.parametrize("failure_kind", ["backend", "transport"]) -def test_agent_node_snapshot_save_failure_preserves_original_failure(failure_kind: str) -> None: +def test_agent_node_snapshot_save_failure_preserves_original_failure(tool_providers, failure_kind: str) -> None: store = ExplodingSessionStore() client = ( FakeAgentBackendRunClient(scenario=FakeAgentBackendScenario.FAILED) @@ -807,7 +828,7 @@ def test_agent_node_snapshot_save_failure_preserves_original_failure(failure_kin else FailingStreamBackendClient() ) - events = list(_node(agent_backend_client=client, session_store=store)._run()) + events = list(_node(tool_providers=tool_providers, agent_backend_client=client, session_store=store)._run()) result = cast(StreamCompletedEvent, events[0]).node_run_result assert result.status == WorkflowNodeExecutionStatus.FAILED @@ -823,11 +844,14 @@ def test_agent_node_snapshot_save_failure_preserves_original_failure(failure_kin @pytest.mark.parametrize("terminal_type", ["failed", "cancelled"]) -def test_agent_node_terminal_without_snapshot_preserves_prior_session_without_write(terminal_type: str) -> None: +def test_agent_node_terminal_without_snapshot_preserves_prior_session_without_write( + tool_providers, terminal_type: str +) -> None: store = FakeSessionStore() events = list( _node( + tool_providers=tool_providers, agent_backend_client=TerminalWithoutSnapshotBackendClient(terminal_type=terminal_type), session_store=store, )._run() @@ -838,9 +862,9 @@ def test_agent_node_terminal_without_snapshot_preserves_prior_session_without_wr assert store.saved == [] -def test_agent_node_paused_run_requests_workflow_pause_and_persists_snapshot(): +def test_agent_node_paused_run_requests_workflow_pause_and_persists_snapshot(tool_providers): store = FakeSessionStore() - node = _node(scenario=FakeAgentBackendScenario.PAUSED, session_store=store) + node = _node(tool_providers=tool_providers, scenario=FakeAgentBackendScenario.PAUSED, session_store=store) # ENG-636: the PAUSED scenario emits a dify.ask_human deferred call, so the # node now builds a HITL form and pauses with HitlRequired. Stub the @@ -891,7 +915,9 @@ def _pending_session(snapshot: CompositorSessionSnapshot) -> StoredWorkflowAgent ) -def test_agent_node_resumes_with_deferred_tool_results_after_submitted_form(monkeypatch: pytest.MonkeyPatch): +def test_agent_node_resumes_with_deferred_tool_results_after_submitted_form( + tool_providers, monkeypatch: pytest.MonkeyPatch +): # ENG-638: a submitted form re-enters _run; the human's answer is threaded # into the second Agent run as deferred_tool_results. snapshot = CompositorSessionSnapshot(layers=[]) @@ -902,10 +928,12 @@ def _fake_resolve(*, form_id: str, tenant_id: str, node_id: str) -> AskHumanResu assert form_id == "form-1" return AskHumanResumeOutcome(deferred_result=AskHumanToolResult(status="submitted", values={"note": "ok"})) - monkeypatch.setattr("core.workflow.nodes.agent_v2.agent_node.resolve_ask_human_form", _fake_resolve) + monkeypatch.setattr( + "services.workflow.execution.adapters.agent_v2.agent_node.resolve_ask_human_form", _fake_resolve + ) client = FakeAgentBackendRunClient() # SUCCESS scenario -> second run completes - node = _node(agent_backend_client=client, session_store=store) + node = _node(tool_providers=tool_providers, agent_backend_client=client, session_store=store) events = list(node._run()) @@ -915,7 +943,7 @@ def _fake_resolve(*, form_id: str, tenant_id: str, node_id: str) -> AskHumanResu assert any(isinstance(event, StreamCompletedEvent) for event in events) -def test_agent_node_repauses_when_resumed_form_still_waiting(monkeypatch: pytest.MonkeyPatch): +def test_agent_node_repauses_when_resumed_form_still_waiting(tool_providers, monkeypatch: pytest.MonkeyPatch): snapshot = CompositorSessionSnapshot(layers=[]) store = FakeSessionStore(snapshot=snapshot) store.loaded_session = _pending_session(snapshot) @@ -929,12 +957,12 @@ def test_agent_node_repauses_when_resumed_form_still_waiting(monkeypatch: pytest node_title="Budget review", ) monkeypatch.setattr( - "core.workflow.nodes.agent_v2.agent_node.resolve_ask_human_form", + "services.workflow.execution.adapters.agent_v2.agent_node.resolve_ask_human_form", lambda **_kwargs: AskHumanResumeOutcome(repause=repause), ) client = FakeAgentBackendRunClient() - node = _node(agent_backend_client=client, session_store=store) + node = _node(tool_providers=tool_providers, agent_backend_client=client, session_store=store) events = list(node._run()) @@ -945,7 +973,7 @@ def test_agent_node_repauses_when_resumed_form_still_waiting(monkeypatch: pytest assert client.request is None # no second Agent run was created -def test_agent_node_expired_ask_human_failure_keeps_binding_identity(monkeypatch: pytest.MonkeyPatch): +def test_agent_node_expired_ask_human_failure_keeps_binding_identity(tool_providers, monkeypatch: pytest.MonkeyPatch): snapshot = CompositorSessionSnapshot(layers=[]) store = FakeSessionStore(snapshot=snapshot) store.loaded_session = _pending_session(snapshot) @@ -954,11 +982,11 @@ def _raise_expired_form(**_kwargs): raise AssertionError("cannot resume globally expired ask_human form, form_id=form-1") monkeypatch.setattr( - "core.workflow.nodes.agent_v2.agent_node.resolve_ask_human_form", + "services.workflow.execution.adapters.agent_v2.agent_node.resolve_ask_human_form", _raise_expired_form, ) - events = list(_node(session_store=store)._run()) + events = list(_node(tool_providers=tool_providers, session_store=store)._run()) assert len(events) == 1 result = cast(StreamCompletedEvent, events[0]).node_run_result @@ -969,7 +997,7 @@ def _raise_expired_form(**_kwargs): assert "agent_workspace_binding_id" not in result.process_data -def test_agent_node_unexpected_post_resolution_failure_keeps_binding_identity(): +def test_agent_node_unexpected_post_resolution_failure_keeps_binding_identity(tool_providers): class _FailingSessionStore(FakeSessionStore): def load_or_create_node_execution_session( self, @@ -980,7 +1008,7 @@ def load_or_create_node_execution_session( del scope, home_snapshot_id raise RuntimeError("session store failed") - events = list(_node(session_store=_FailingSessionStore())._run()) + events = list(_node(tool_providers=tool_providers, session_store=_FailingSessionStore())._run()) assert len(events) == 1 result = cast(StreamCompletedEvent, events[0]).node_run_result @@ -994,9 +1022,9 @@ def load_or_create_node_execution_session( } -def test_agent_node_cancels_backend_run_when_stream_fails(): +def test_agent_node_cancels_backend_run_when_stream_fails(tool_providers): client = FailingStreamBackendClient() - node = _node(agent_backend_client=client) + node = _node(tool_providers=tool_providers, agent_backend_client=client) terminal, failure = node._consume_event_stream( "run-1", @@ -1013,9 +1041,9 @@ def test_agent_node_cancels_backend_run_when_stream_fails(): assert client.cancel_requests[0].reason == "event_stream_failed" -def test_agent_node_forwards_last_stream_cursor_when_cancelling_after_failure() -> None: +def test_agent_node_forwards_last_stream_cursor_when_cancelling_after_failure(tool_providers) -> None: client = FailingAfterStartedStreamBackendClient() - node = _node(agent_backend_client=client) + node = _node(tool_providers=tool_providers, agent_backend_client=client) terminal, failure = node._consume_event_stream( "run-1", @@ -1030,9 +1058,9 @@ def test_agent_node_forwards_last_stream_cursor_when_cancelling_after_failure() assert client.cancel_after == ["cursor-1"] -def test_agent_node_cancels_backend_run_when_stream_ends_without_terminal_event(): +def test_agent_node_cancels_backend_run_when_stream_ends_without_terminal_event(tool_providers): client = EmptyStreamBackendClient() - node = _node(agent_backend_client=client) + node = _node(tool_providers=tool_providers, agent_backend_client=client) terminal, failure = node._consume_event_stream( "run-1", @@ -1047,9 +1075,9 @@ def test_agent_node_cancels_backend_run_when_stream_ends_without_terminal_event( assert client.cancel_requests[0].reason == "stream_ended_without_terminal_event" -def test_agent_node_cancels_backend_run_when_stream_raises_unexpected_error(): +def test_agent_node_cancels_backend_run_when_stream_raises_unexpected_error(tool_providers): client = GenericFailingStreamBackendClient() - node = _node(agent_backend_client=client) + node = _node(tool_providers=tool_providers, agent_backend_client=client) terminal, failure = node._consume_event_stream( "run-1", @@ -1066,9 +1094,9 @@ def test_agent_node_cancels_backend_run_when_stream_raises_unexpected_error(): assert client.cancel_requests[0].reason == "event_stream_failed" -def test_agent_node_uses_graph_abort_reason_when_cancel_request_fails(caplog): +def test_agent_node_uses_graph_abort_reason_when_cancel_request_fails(tool_providers, caplog): client = CancelFailingStreamBackendClient() - node = _node(agent_backend_client=client) + node = _node(tool_providers=tool_providers, agent_backend_client=client) node.graph_runtime_state.graph_execution = SimpleNamespace(aborted=True) terminal, failure = node._consume_event_stream( @@ -1087,9 +1115,9 @@ def test_agent_node_uses_graph_abort_reason_when_cancel_request_fails(caplog): assert "Failed to finish cancelling Workflow Agent backend run" in caplog.text -def test_agent_node_cancels_backend_run_for_unexpected_internal_event(): +def test_agent_node_cancels_backend_run_for_unexpected_internal_event(tool_providers): client = FakeAgentBackendRunClient() - node = _node(agent_backend_client=client) + node = _node(tool_providers=tool_providers, agent_backend_client=client) node._agent_backend_client.cancel_run_and_wait = MagicMock( # type: ignore[method-assign] return_value=RunCancelledEvent(run_id="run-1") ) @@ -1139,7 +1167,7 @@ def test_agent_node_records_stream_usage_metadata(): (True, 2, {"text": "hello", "switch": "route-1"}, "route-1"), ], ) -def test_agent_node_selects_the_configured_success_exit(enabled, count, output, expected_handle): +def test_agent_node_selects_the_configured_success_exit(tool_providers, enabled, count, output, expected_handle): resolver = FakeBindingResolver() resolver.binding.node_job_config = WorkflowNodeJobConfig.model_validate( { @@ -1149,7 +1177,11 @@ def test_agent_node_selects_the_configured_success_exit(enabled, count, output, } } ) - node = _node(binding_resolver=resolver, agent_backend_client=FileOutputBackendClient(output_payload=output)) + node = _node( + tool_providers=tool_providers, + binding_resolver=resolver, + agent_backend_client=FileOutputBackendClient(output_payload=output), + ) result = list(node._run())[-1].node_run_result assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED assert result.edge_source_handle == expected_handle @@ -1167,13 +1199,20 @@ def test_agent_node_selects_the_configured_success_exit(enabled, count, output, ], ) def test_agent_routes_initialize_through_workflow_node_factory( - enabled, error_strategy, expected_execution_type, config_overrides + tool_providers, enabled, error_strategy, expected_execution_type, config_overrides, workflow_runtime ): - from core.workflow.node_factory import DifyNodeFactory + from services.workflow.execution.adapters.node_factory import DifyNodeFactory config_overrides(AGENT_BACKEND_USE_FAKE=True) - template = _node() - factory = DifyNodeFactory(template.graph_init_params, template.graph_runtime_state) + template = _node( + tool_providers=tool_providers, + ) + factory = DifyNodeFactory( + template.graph_init_params, + template.graph_runtime_state, + agent_binding_resolver=FakeBindingResolver(), + workflow_runtime=workflow_runtime, + ) node_data = template.node_data.model_dump(mode="python", by_alias=True) node_data.update( agent_output_routes={ @@ -1195,7 +1234,7 @@ def test_agent_routes_initialize_through_workflow_node_factory( assert node.node_data.agent_output_routes.enabled is enabled -def test_agent_routes_use_the_successful_retry_to_fan_out_and_skip_other_branches(): +def test_agent_routes_use_the_successful_retry_to_fan_out_and_skip_other_branches(tool_providers): from graphon.enums import NodeState from graphon.graph.edge import Edge from graphon.graph.graph import Graph @@ -1229,7 +1268,11 @@ def _events(self, run_id: str): }, } ) - node = _node(binding_resolver=resolver, agent_backend_client=RetryingBackendClient(output_payload={})) + node = _node( + tool_providers=tool_providers, + binding_resolver=resolver, + agent_backend_client=RetryingBackendClient(output_payload={}), + ) result = list(node._run())[-1].node_run_result assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED assert result.outputs["score"] == 42 @@ -1265,7 +1308,7 @@ def _events(self, run_id: str): @pytest.mark.parametrize("output", ["plain text", {}, {"switch": None}, {"switch": 1}, {"switch": "unknown"}]) -def test_agent_node_rejects_invalid_route_selection_without_custom_outputs(output): +def test_agent_node_rejects_invalid_route_selection_without_custom_outputs(tool_providers, output): resolver = FakeBindingResolver() resolver.binding.node_job_config = WorkflowNodeJobConfig.model_validate( { @@ -1275,7 +1318,11 @@ def test_agent_node_rejects_invalid_route_selection_without_custom_outputs(outpu } } ) - node = _node(binding_resolver=resolver, agent_backend_client=FileOutputBackendClient(output_payload=output)) + node = _node( + tool_providers=tool_providers, + binding_resolver=resolver, + agent_backend_client=FileOutputBackendClient(output_payload=output), + ) result = list(node._run())[-1].node_run_result assert result.status == WorkflowNodeExecutionStatus.FAILED assert result.error_type == "output_route_selection_failed" @@ -1309,7 +1356,7 @@ def test_agent_route_conditions_load_variables_only_when_routing_is_enabled(enab assert mapping == expected -def test_disabled_routes_continue_through_graphon_default_values(): +def test_disabled_routes_continue_through_graphon_default_values(tool_providers): from graphon.graph.edge import Edge from graphon.graph.graph import Graph from graphon.graph_engine.error_handler import ErrorHandler @@ -1317,7 +1364,7 @@ def test_disabled_routes_continue_through_graphon_default_values(): from graphon.graph_events.node import NodeRunExceptionEvent, NodeRunFailedEvent from graphon.node_events import NodeRunResult - node = _node(error_strategy=ErrorStrategy.DEFAULT_VALUE) + node = _node(tool_providers=tool_providers, error_strategy=ErrorStrategy.DEFAULT_VALUE) edges = {target: Edge(id=target, tail="agent-node", head=target) for target in ["next-a", "next-b"]} graph = Graph(root_node=node, nodes={"agent-node": node}, edges=edges, out_edges={"agent-node": list(edges)}) failure = NodeRunFailedEvent( @@ -1339,7 +1386,7 @@ def test_disabled_routes_continue_through_graphon_default_values(): assert [event.edge_id for event in traversed] == ["next-a", "next-b"] -def test_enabled_routes_reject_default_value_before_execution(): +def test_enabled_routes_reject_default_value_before_execution(tool_providers): resolver = FakeBindingResolver() resolver.binding.node_job_config = WorkflowNodeJobConfig.model_validate( { @@ -1350,4 +1397,4 @@ def test_enabled_routes_reject_default_value_before_execution(): } ) with pytest.raises(ValueError, match="default-value"): - _node(binding_resolver=resolver, error_strategy=ErrorStrategy.DEFAULT_VALUE) + _node(tool_providers=tool_providers, binding_resolver=resolver, error_strategy=ErrorStrategy.DEFAULT_VALUE) diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py index f4dd2b3b4c147a..70f99c94d42891 100644 --- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py +++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py @@ -1,15 +1,16 @@ from collections.abc import Iterator -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock from uuid import uuid4 import pytest from agenton.compositor import CompositorSessionSnapshot -from sqlalchemy import event, inspect +from flask import Flask +from sqlalchemy import create_engine, event, inspect from sqlalchemy.orm import ORMExecuteState, Session, sessionmaker from sqlalchemy.sql import Executable -from core.workflow.nodes.agent_v2.binding_resolver import WorkflowAgentBindingError, WorkflowAgentBindingResolver from core.workflow.nodes.agent_v2.session_store import WorkflowAgentSessionScope, WorkflowAgentWorkspaceStore +from enums.agent import WorkflowAgentBindingType from graphon.enums import WorkflowNodeExecutionStatus from models.agent import ( Agent, @@ -25,14 +26,143 @@ AgentWorkspace, AgentWorkspaceBinding, AgentWorkspaceOwnerType, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import AgentSoulConfig, AgentSoulModelConfig, WorkflowNodeJobConfig +from models.agent_runtime_contracts import WorkflowAgentBindingError from models.enums import CreatorUserRole from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom +from repositories.agent.runtime_repository import WorkflowAgentBindingResolver from services.agent.workspace_service import AgentWorkspaceService + +@pytest.fixture +def binding_resolver(sqlite_session_factory: sessionmaker[Session]) -> WorkflowAgentBindingResolver: + return WorkflowAgentBindingResolver(sqlite_session_factory) + + +@pytest.mark.parametrize("explicit_worker", [False, True]) +def test_node_binding_resolution_uses_injected_database_through_nested_factories( + app: Flask, monkeypatch: pytest.MonkeyPatch, binding_resolver: WorkflowAgentBindingResolver, explicit_worker: bool +) -> None: + from clients.agent_backend import AgentBackendRunClient + from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, build_dify_run_context + from enums import DeploymentEdition + from extensions.ext_application_services import build_application_services + from extensions.ext_redis import RedisClientWrapper + from graphon.runtime import GraphRuntimeState, VariablePool + from models.base import TypeBase + from services.workflow.execution.adapters.agent_v2.agent_node import DifyAgentNode + from services.workflow.execution.adapters.node_factory import DifyGraphInitContext, DifyNodeFactory + + # The normal unit-test factory remains bound to its empty database. The + # configured application/worker uses a second database with these bindings. + engine = create_engine("sqlite://") + sessions = sessionmaker(bind=engine, expire_on_commit=False) + monkeypatch.setattr( + "clients.agent_backend.factory.create_agent_backend_run_client", + Mock(return_value=Mock(spec=AgentBackendRunClient)), + ) + try: + TypeBase.metadata.create_all( + engine, tables=[TypeBase.metadata.tables[model.__tablename__] for model in RESOLVER_MODELS] + ) + ids = _resolve_ids() + with sessions.begin() as session: + agent = _agent(tenant_id=ids["tenant_id"]) + session.add(agent) + session.flush() + snapshot = _snapshot(tenant_id=ids["tenant_id"], agent_id=agent.id) + session.add(snapshot) + session.flush() + session.add( + _binding( + ids=ids, + agent_id=agent.id, + snapshot_id=snapshot.id, + binding_type=WorkflowAgentBindingType.INLINE_AGENT, + ) + ) + with pytest.raises(WorkflowAgentBindingError, match="not found"): + binding_resolver.resolve(**ids) + + services = build_application_services( + database_client=sessions, + deployment_edition=DeploymentEdition.COMMUNITY, + initialization_password="", + redis=Mock(spec=RedisClientWrapper), + ) + resolver = WorkflowAgentBindingResolver(sessions) if explicit_worker else services.workflow_agent_bindings + monkeypatch.delitem(app.extensions, "application_services", raising=False) + context = DifyGraphInitContext( + workflow_id=ids["workflow_id"], + graph_config={"nodes": [], "edges": []}, + call_depth=0, + run_context=build_dify_run_context( + tenant_id=ids["tenant_id"], + app_id=ids["app_id"], + user_id="account", + user_from=UserFrom.ACCOUNT, + invoke_from=InvokeFrom.DEBUGGER, + ), + ) + with app.app_context(): + factory = DifyNodeFactory.from_graph_init_context( + graph_init_context=context, + graph_runtime_state=GraphRuntimeState(variable_pool=VariablePool(), start_at=0), + agent_binding_resolver=resolver, + workflow_runtime=services.workflow_runtime, + ) + nested = factory.with_runtime_state(GraphRuntimeState(variable_pool=VariablePool(), start_at=0)) + for owner in (factory, nested): + node = owner.create_node( + {"id": ids["node_id"], "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}} + ) + assert isinstance(node, DifyAgentNode) + bundle = node._binding_resolver.resolve(**ids) + assert bundle.snapshot.id == snapshot.id + assert inspect(bundle.snapshot).detached + + # Single-node debugging must reach the same injected database, + # without consulting an installed Flask application container. + from services.workflow.execution.adapters.workflow_entry import WorkflowEntry + from tests.unit_tests.model_factories import make_workflow + + node_config = { + "id": ids["node_id"], + "data": {"type": "agent", "version": "2", "title": "Agent", "agent_node_kind": "dify_agent"}, + } + workflow = make_workflow( + tenant_id=ids["tenant_id"], + app_id=ids["app_id"], + workflow_id=ids["workflow_id"], + graph={"nodes": [node_config], "edges": []}, + ) + monkeypatch.setattr(WorkflowEntry, "_run_node_with_layers", Mock()) + node, _ = WorkflowEntry.single_step_run( + workflow=workflow, + node_id=ids["node_id"], + user_id="account", + user_inputs={}, + variable_pool=VariablePool(), + agent_binding_resolver=resolver, + workflow_runtime=services.workflow_runtime, + ) + assert isinstance(node, DifyAgentNode) + assert node._binding_resolver.resolve(**ids).snapshot.id == snapshot.id + with pytest.raises(ValueError, match="injected binding resolver"): + WorkflowEntry.single_step_run( + workflow=workflow, + node_id=ids["node_id"], + user_id="account", + user_inputs={}, + variable_pool=VariablePool(), + workflow_runtime=services.workflow_runtime, + ) + finally: + engine.dispose() + + RESOLVER_MODELS = (WorkflowAgentNodeBinding, Agent, AgentConfigSnapshot, AgentConfigRevision) CHATFLOW_MODELS = ( *RESOLVER_MODELS, @@ -82,6 +212,7 @@ def _conversation_participant( @pytest.mark.parametrize("sqlite_session", [CHATFLOW_MODELS], indirect=True) @pytest.mark.parametrize("binding_type", [WorkflowAgentBindingType.ROSTER_AGENT, WorkflowAgentBindingType.INLINE_AGENT]) def test_chatflow_keeps_participant_config_and_home_after_agent_update( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, monkeypatch: pytest.MonkeyPatch, binding_type: WorkflowAgentBindingType, @@ -105,7 +236,7 @@ def test_chatflow_keeps_participant_config_and_home_after_agent_update( sqlite_session.flush() participant = _conversation_participant(sqlite_session, ids=ids, binding=binding, snapshot=original) sqlite_session.commit() - resolver = WorkflowAgentBindingResolver() + resolver = binding_resolver assert resolver.resolve(**ids).snapshot.id == original.id published = _snapshot(tenant_id=ids["tenant_id"], agent_id=agent.id) @@ -177,6 +308,7 @@ def test_chatflow_keeps_participant_config_and_home_after_agent_update( @pytest.mark.parametrize("mismatch", ["tenant", "app", "conversation", "node", "agent", "retired"]) @pytest.mark.parametrize("sqlite_session", [CHATFLOW_MODELS], indirect=True) def test_chatflow_pin_is_scoped_to_active_conversation_node_participant( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, mismatch: str, ) -> None: @@ -212,12 +344,13 @@ def test_chatflow_pin_is_scoped_to_active_conversation_node_participant( participant.status = AgentWorkingResourceStatus.RETIRED sqlite_session.commit() - assert WorkflowAgentBindingResolver().resolve(**ids).snapshot.id == current.id + assert binding_resolver.resolve(**ids).snapshot.id == current.id @pytest.mark.parametrize("version_kind", [AgentConfigVersionKind.DRAFT, AgentConfigVersionKind.SNAPSHOT]) @pytest.mark.parametrize("sqlite_session", [CHATFLOW_MODELS], indirect=True) def test_chatflow_never_falls_back_from_invalid_pinned_generation( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, version_kind: AgentConfigVersionKind, ) -> None: @@ -240,7 +373,7 @@ def test_chatflow_never_falls_back_from_invalid_pinned_generation( sqlite_session.commit() with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**ids) + binding_resolver.resolve(**ids) expected = ( "agent_binding_generation_invalid" if version_kind == AgentConfigVersionKind.DRAFT @@ -346,6 +479,7 @@ def record_statement(execute_state: ORMExecuteState) -> None: @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_returns_detached_binding_bundle( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -363,7 +497,7 @@ def test_binding_resolver_returns_detached_binding_bundle( ) sqlite_session.add(binding) sqlite_session.commit() - bundle = WorkflowAgentBindingResolver().resolve(**ids) + bundle = binding_resolver.resolve(**ids) assert bundle.binding.id == binding.id assert bundle.agent.id == agent.id @@ -375,6 +509,7 @@ def test_binding_resolver_returns_detached_binding_bundle( @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_uses_active_snapshot_for_roster_agent( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -397,7 +532,7 @@ def test_binding_resolver_uses_active_snapshot_for_roster_agent( ) sqlite_session.add(binding) sqlite_session.commit() - bundle = WorkflowAgentBindingResolver().resolve(**ids) + bundle = binding_resolver.resolve(**ids) assert bundle.snapshot.id == active_snapshot.id @@ -411,6 +546,7 @@ def test_binding_resolver_uses_active_snapshot_for_roster_agent( ) @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_uses_pinned_snapshot_for_existing_node_execution( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, orm_statements: list[Executable], binding_type: WorkflowAgentBindingType, @@ -437,7 +573,7 @@ def test_binding_resolver_uses_pinned_snapshot_for_existing_node_execution( ) sqlite_session.add(binding) sqlite_session.commit() - bundle = WorkflowAgentBindingResolver().resolve( + bundle = binding_resolver.resolve( **ids, binding_id=binding.id, snapshot_id=pinned_snapshot.id, @@ -450,6 +586,7 @@ def test_binding_resolver_uses_pinned_snapshot_for_existing_node_execution( @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_does_not_fallback_from_an_explicit_empty_snapshot( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -473,7 +610,7 @@ def test_binding_resolver_does_not_fallback_from_an_explicit_empty_snapshot( sqlite_session.add(binding) sqlite_session.commit() with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**ids, binding_id=binding.id, snapshot_id="") + binding_resolver.resolve(**ids, binding_id=binding.id, snapshot_id="") assert exc_info.value.error_code == "agent_config_snapshot_not_found" @@ -483,11 +620,12 @@ def test_binding_resolver_does_not_fallback_from_an_explicit_empty_snapshot( [("binding-1", None), (None, "snapshot-1")], ) def test_binding_resolver_rejects_half_pinned_generation( + binding_resolver: WorkflowAgentBindingResolver, binding_id: str | None, snapshot_id: str | None, ) -> None: with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve( + binding_resolver.resolve( **_resolve_ids(), binding_id=binding_id, snapshot_id=snapshot_id, @@ -498,6 +636,7 @@ def test_binding_resolver_rejects_half_pinned_generation( @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_rejects_unpublished_roster_agent( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -521,7 +660,7 @@ def test_binding_resolver_rejects_unpublished_roster_agent( sqlite_session.add(binding) sqlite_session.commit() with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**ids) + binding_resolver.resolve(**ids) assert exc_info.value.error_code == "agent_not_available" assert "not been published" in str(exc_info.value) @@ -529,6 +668,7 @@ def test_binding_resolver_rejects_unpublished_roster_agent( @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_requires_publish_provenance_for_active_roster_snapshot( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -565,7 +705,7 @@ def test_binding_resolver_requires_publish_provenance_for_active_roster_snapshot ) sqlite_session.commit() with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**ids) + binding_resolver.resolve(**ids) assert exc_info.value.error_code == "agent_not_available" sqlite_session.add( @@ -579,21 +719,24 @@ def test_binding_resolver_requires_publish_provenance_for_active_roster_snapshot ) sqlite_session.commit() - bundle = WorkflowAgentBindingResolver().resolve(**ids) + bundle = binding_resolver.resolve(**ids) assert bundle.agent.id == agent.id assert bundle.snapshot.id == snapshot.id -def test_binding_resolver_raises_when_binding_missing() -> None: +def test_binding_resolver_raises_when_binding_missing( + binding_resolver: WorkflowAgentBindingResolver, +) -> None: with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**_resolve_ids()) + binding_resolver.resolve(**_resolve_ids()) assert exc_info.value.error_code == "agent_binding_not_found" @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_raises_when_agent_archived( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -609,13 +752,14 @@ def test_binding_resolver_raises_when_agent_archived( sqlite_session.add(binding) sqlite_session.commit() with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**ids) + binding_resolver.resolve(**ids) assert exc_info.value.error_code == "agent_not_available" @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_raises_when_snapshot_missing( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -631,6 +775,6 @@ def test_binding_resolver_raises_when_snapshot_missing( sqlite_session.add(binding) sqlite_session.commit() with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**ids) + binding_resolver.resolve(**ids) assert exc_info.value.error_code == "agent_config_snapshot_not_found" diff --git a/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_invocation_behavior.py b/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_invocation_behavior.py index 243dfccf83cbb3..aeeafdc3e302f5 100644 --- a/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_invocation_behavior.py +++ b/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_invocation_behavior.py @@ -6,8 +6,6 @@ from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom from core.tools.utils.configuration import ToolParameterConfigurationManager -from core.workflow.node_factory import DifyNodeFactory -from core.workflow.node_runtime import DifyToolNodeRuntime from core.workflow.system_variables import build_system_variables from graphon.enums import WorkflowNodeExecutionStatus from graphon.graph import Graph @@ -16,10 +14,12 @@ from graphon.nodes.tool.entities import ToolNodeData from graphon.nodes.tool.tool_node import ToolNode from graphon.runtime import GraphRuntimeState, VariablePool +from services.workflow.execution.adapters.node_factory import DifyNodeFactory +from services.workflow.execution.adapters.node_runtime import DifyToolNodeRuntime from tests.workflow_test_utils import build_test_graph_init_params -def init_tool_node(config: dict[str, object]) -> ToolNode: +def init_tool_node(config: dict[str, object], *, workflow_runtime) -> ToolNode: graph_config = { "edges": [ { @@ -68,14 +68,15 @@ def init_tool_node(config: dict[str, object]) -> ToolNode: graph_init_params=init_params, graph_runtime_state=graph_runtime_state, tool_file_manager=tool_file_manager, - runtime=DifyToolNodeRuntime(init_params.run_context), + runtime=DifyToolNodeRuntime(init_params.run_context, workflow_runtime=workflow_runtime), ) node.bind_execution_id(str(uuid.uuid4())) return node -def test_tool_variable_invoke() -> None: +def test_tool_variable_invoke(workflow_runtime) -> None: node = init_tool_node( + workflow_runtime=workflow_runtime, config={ "id": "1", "data": { @@ -90,7 +91,7 @@ def test_tool_variable_invoke() -> None: "tool_configurations": {}, "tool_parameters": {}, }, - } + }, ) with patch.object( @@ -109,8 +110,9 @@ def test_tool_variable_invoke() -> None: assert item.node_run_result.outputs.get("text") is not None -def test_tool_mixed_invoke() -> None: +def test_tool_mixed_invoke(workflow_runtime) -> None: node = init_tool_node( + workflow_runtime=workflow_runtime, config={ "id": "1", "data": { @@ -127,7 +129,7 @@ def test_tool_mixed_invoke() -> None: }, "tool_parameters": {}, }, - } + }, ) with patch.object( diff --git a/api/tests/unit_tests/core/workflow/test_node_mapping_bootstrap.py b/api/tests/unit_tests/core/workflow/test_node_mapping_bootstrap.py index c512eb04637dbf..6d7fdc05425729 100644 --- a/api/tests/unit_tests/core/workflow/test_node_mapping_bootstrap.py +++ b/api/tests/unit_tests/core/workflow/test_node_mapping_bootstrap.py @@ -20,16 +20,15 @@ def test_moved_core_nodes_resolve_after_importing_production_entrypoints(): env["PYTHONSAFEPATH"] = "1" script = textwrap.dedent( """ - from core.app.apps import workflow_app_runner - from core.workflow import workflow_entry + from services.workflow.execution.adapters import graph as workflow_app_runner + from services.workflow.execution.adapters import workflow_entry from core.workflow.nodes.knowledge_index import KNOWLEDGE_INDEX_NODE_TYPE - from core.workflow.node_factory import ( + from services.workflow.execution.adapters.node_factory import ( DifyNodeFactory, NODE_TYPE_CLASSES_MAPPING, resolve_workflow_node_class, ) - from core.workflow.nodes.agent import AgentNode - from core.workflow.nodes.agent_v2 import DifyAgentNode + from core.trigger.constants import TRIGGER_WEBHOOK_NODE_TYPE from graphon.enums import BuiltinNodeTypes from services import workflow_service from services.rag_pipeline import rag_pipeline @@ -37,7 +36,6 @@ def test_moved_core_nodes_resolve_after_importing_production_entrypoints(): _ = workflow_entry, workflow_app_runner, workflow_service, rag_pipeline expected = ( - BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL, KNOWLEDGE_INDEX_NODE_TYPE, BuiltinNodeTypes.DATASOURCE, ) @@ -47,6 +45,20 @@ def test_moved_core_nodes_resolve_after_importing_production_entrypoints(): resolved = DifyNodeFactory._resolve_node_class(node_type=node_type, node_version="1") assert resolved.__module__.startswith("core.workflow.nodes."), resolved.__module__ + # Resolve through the production bootstrap before importing the moved + # classes; preloading them here would hide missing registration. + agent = DifyNodeFactory._resolve_node_class(node_type=BuiltinNodeTypes.AGENT, node_version="1") + webhook = DifyNodeFactory._resolve_node_class(node_type=TRIGGER_WEBHOOK_NODE_TYPE, node_version="1") + from services.workflow.execution.adapters.agent_node import AgentNode + from services.workflow.execution.adapters.agent_v2.agent_node import DifyAgentNode + from services.workflow.execution.adapters.trigger_webhook import TriggerWebhookNode + + from services.workflow.execution.adapters.knowledge_retrieval import KnowledgeRetrievalNode + assert DifyNodeFactory._resolve_node_class( + node_type=BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL, node_version="1" + ) is KnowledgeRetrievalNode + assert agent is AgentNode + assert webhook is TriggerWebhookNode assert DifyNodeFactory._resolve_node_class( node_type=BuiltinNodeTypes.AGENT, node_version="2", diff --git a/api/tests/unit_tests/events/event_handlers/test_sync_plugin_trigger_when_app_created.py b/api/tests/unit_tests/events/event_handlers/test_sync_plugin_trigger_when_app_created.py deleted file mode 100644 index a9d631cf7dd55d..00000000000000 --- a/api/tests/unit_tests/events/event_handlers/test_sync_plugin_trigger_when_app_created.py +++ /dev/null @@ -1,31 +0,0 @@ -from types import SimpleNamespace -from typing import cast -from unittest.mock import patch - -from events.event_handlers.sync_plugin_trigger_when_app_created import handle -from models.model import AppMode -from models.workflow import Workflow - - -def test_syncs_plugin_trigger_relationships_from_published_workflow() -> None: - app = SimpleNamespace(mode=AppMode.WORKFLOW.value) - published_workflow = cast(Workflow, object()) - - with patch( - "events.event_handlers.sync_plugin_trigger_when_app_created.TriggerService.sync_plugin_trigger_relationships" - ) as sync_relationships: - handle(app, published_workflow=published_workflow) - - sync_relationships.assert_called_once_with(app, published_workflow) - - -def test_keeps_draft_workflow_relationship_sync() -> None: - app = SimpleNamespace(mode=AppMode.WORKFLOW.value) - draft_workflow = cast(Workflow, object()) - - with patch( - "events.event_handlers.sync_plugin_trigger_when_app_created.TriggerService.sync_plugin_trigger_relationships" - ) as sync_relationships: - handle(app, synced_draft_workflow=draft_workflow) - - sync_relationships.assert_called_once_with(app, draft_workflow) diff --git a/api/tests/unit_tests/events/event_handlers/test_sync_webhook_when_app_created.py b/api/tests/unit_tests/events/event_handlers/test_sync_webhook_when_app_created.py index 9f450474ec0aff..f262397319f5c1 100644 --- a/api/tests/unit_tests/events/event_handlers/test_sync_webhook_when_app_created.py +++ b/api/tests/unit_tests/events/event_handlers/test_sync_webhook_when_app_created.py @@ -1,32 +1,17 @@ -from types import SimpleNamespace -from typing import cast from unittest.mock import patch -from events.event_handlers import sync_webhook_when_app_created from events.event_handlers.sync_webhook_when_app_created import handle -from models.model import App -from models.workflow import Workflow +from models.model import AppMode +from tests.unit_tests.model_factories import make_app, make_workflow def test_draft_sync_preserves_stale_webhook_relationships() -> None: - app = cast(App, SimpleNamespace(mode="workflow")) - workflow = cast(Workflow, SimpleNamespace()) + app = make_app(mode=AppMode.WORKFLOW) + workflow = make_workflow() with patch( "events.event_handlers.sync_webhook_when_app_created.WebhookService.sync_webhook_relationships" ) as sync_webhook_relationships: handle(app, synced_draft_workflow=workflow) - sync_webhook_relationships.assert_called_once_with(app, workflow, remove_stale=False) - - -def test_published_sync_removes_stale_webhook_relationships() -> None: - app = cast(App, SimpleNamespace(mode="workflow")) - workflow = cast(Workflow, SimpleNamespace()) - - with patch( - "events.event_handlers.sync_webhook_when_app_created.WebhookService.sync_webhook_relationships" - ) as sync_webhook_relationships: - sync_webhook_when_app_created.handle_published(app, published_workflow=workflow) - - sync_webhook_relationships.assert_called_once_with(app, workflow, remove_stale=True) + sync_webhook_relationships.assert_called_once_with(app, workflow, remove_stale=False, session_factory=None) diff --git a/api/tests/unit_tests/extensions/test_ext_application_services.py b/api/tests/unit_tests/extensions/test_ext_application_services.py index 320b3eddca0c57..65dcb2e0383580 100644 --- a/api/tests/unit_tests/extensions/test_ext_application_services.py +++ b/api/tests/unit_tests/extensions/test_ext_application_services.py @@ -55,7 +55,7 @@ from repositories.plugin_file_upload_repository import SQLAlchemyPluginFileUploadOwnerRepository from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository from repositories.upload_file_delivery_repository import UploadFileDeliveryQueryRepository -from repositories.workflow_app_log_query_repository import WorkflowAppLogQueryRepository +from repositories.workflow.app_log_repository import WorkflowAppLogRepository from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepository from repositories.workspace.workspace_repository import WorkspaceRepository from services import audio_provider_gateway, recommended_app_catalog_gateway @@ -146,7 +146,7 @@ from services.web_authentication_service import WebAuthenticationService from services.webapp_access_adapters import EnterpriseWebAppAccessPolicyGateway from services.webapp_access_query_service import WebAppAccessQueryService, WebAppAccessUnavailableError -from services.workflow_app_log_query_service import WorkflowAppLogQueryService +from services.workflow.app_log_query_service import WorkflowAppLogQueryService from services.workflow_run_service import WorkflowRunService from services.workflow_statistic_query_service import WorkflowStatisticQueryService from tasks.delete_conversation_task import delete_conversation_related_data @@ -267,6 +267,10 @@ def test_build_application_services_preserves_composed_boundaries( assert isinstance(services.app_api_keys, AppApiKeyService) assert isinstance(services.dataset_api_keys, DatasetApiKeyService) assert services.dataset_api_keys._access is services.knowledge.datasets._dataset_access + assert services.snippet_generation._variables is services.workflow_variables + assert services.snippet_generation._runtime.contexts._sessions is sqlite_session_factory + assert services.knowledge.pipeline_generator._draft_variable_loader == services.workflow_variables.workflow_loader + assert services.knowledge.pipeline_generator._draft_variable_saver == services.workflow_variables.saver_factory assert isinstance(services.oauth_device, OAuthDeviceApplicationService) assert register_script.call_count == 3 @@ -558,7 +562,7 @@ def test_build_application_services_wires_workflow_app_log_boundary( ) assert isinstance(services.workflow_app_logs, WorkflowAppLogQueryService) - assert isinstance(services.workflow_app_logs._logs, WorkflowAppLogQueryRepository) + assert isinstance(services.workflow_app_logs._logs, WorkflowAppLogRepository) assert services.workflow_app_logs._logs._session_factory is sqlite_session_factory diff --git a/api/tests/unit_tests/extensions/test_workflow_warm_shutdown.py b/api/tests/unit_tests/extensions/test_workflow_warm_shutdown.py index ce774e4d16028e..e9a1a19eec97a5 100644 --- a/api/tests/unit_tests/extensions/test_workflow_warm_shutdown.py +++ b/api/tests/unit_tests/extensions/test_workflow_warm_shutdown.py @@ -3,10 +3,10 @@ import pytest -from core.app.apps.workflow.active_workflow_tasks import reset_active_workflow_tasks -from core.app.apps.workflow.command_channels import CelerySignalCommandChannel from extensions import workflow_warm_shutdown from graphon.graph_engine.entities.commands import AbortCommand +from services.workflow.execution.adapters.workflow.active_workflow_tasks import reset_active_workflow_tasks +from services.workflow.execution.adapters.workflow.command_channels import CelerySignalCommandChannel @pytest.fixture(autouse=True) diff --git a/api/tests/unit_tests/fields/test_workflow_fields.py b/api/tests/unit_tests/fields/test_workflow_fields.py new file mode 100644 index 00000000000000..cf5737a69322b1 --- /dev/null +++ b/api/tests/unit_tests/fields/test_workflow_fields.py @@ -0,0 +1,304 @@ +from dataclasses import asdict +from datetime import datetime + +import pytest +from pydantic import ValidationError +from sqlalchemy import event +from sqlalchemy.orm import Session + +from core.workflow.llm_environment_variable import LLMEnvironmentVariable +from extensions.application_services.snippets import build_snippet_service +from fields import workflow_fields +from graphon.variables import SecretVariable, StringVariable +from graphon.variables.variables import RAGPipelineVariable +from models.dataset import Pipeline +from models.snippet import CustomizedSnippet +from models.tools import WorkflowToolProvider +from models.workflow import WorkflowKind +from repositories.workflow.definition_repository import workflow_record +from services.rag_pipeline.rag_pipeline import RagPipelineService +from services.workflow_ref_service import WorkflowRef +from tests.unit_tests.model_factories import make_account, make_workflow + + +@pytest.mark.parametrize("snippet", [False, True]) +def test_version_lists_share_publication_order_and_owner_scope(sqlite_session: Session, snippet: bool) -> None: + kind = WorkflowKind.SNIPPET if snippet else WorkflowKind.STANDARD + older = make_workflow(workflow_id="older", app_id="owner", version="z-old", kind=kind) + newer = make_workflow(workflow_id="newer", app_id="owner", version="a-new", kind=kind, marked_name="Release") + draft = make_workflow(workflow_id="draft", app_id="owner", kind=kind) + foreign_tenant = make_workflow( + workflow_id="foreign", app_id="owner", tenant_id="other", version="foreign", kind=kind + ) + foreign_owner = make_workflow(workflow_id="other-owner", app_id="other-owner", version="other", kind=kind) + for year, workflow in [(2020, draft), (2021, older), (2022, newer), (2023, foreign_tenant), (2024, foreign_owner)]: + workflow.created_at = datetime(year, 1, 1) + sqlite_session.add(workflow) + if snippet: + sqlite_session.add(make_workflow(workflow_id="wrong-kind", app_id="owner", version="wrong-kind")) + sqlite_session.commit() + + if snippet: + owner = CustomizedSnippet(id="owner", tenant_id="tenant-1", workflow_id=newer.id) + service = build_snippet_service(session=sqlite_session) + page, has_more = service.get_all_published_workflows(session=sqlite_session, snippet=owner, page=1, limit=1) + assert [item.id for item in page] == ["newer"] + assert has_more + page, has_more = service.get_all_published_workflows(session=sqlite_session, snippet=owner, page=2, limit=1) + assert [item.id for item in page] == ["older"] + else: + pipeline = Pipeline(tenant_id="tenant-1", name="Pipeline", workflow_id=newer.id) + pipeline.id = "owner" + pipelines = RagPipelineService(sqlite_session) + page, has_more = pipelines.get_all_published_workflow( + session=sqlite_session, pipeline=pipeline, page=1, limit=2, user_id=None + ) + assert [item.id for item in page] == ["draft", "newer"] + assert has_more + page, has_more = pipelines.get_all_published_workflow( + session=sqlite_session, pipeline=pipeline, page=2, limit=2, user_id=None + ) + assert [item.id for item in page] == ["older"] + named, more = pipelines.get_all_published_workflow( + session=sqlite_session, pipeline=pipeline, page=1, limit=2, user_id="account-1", named_only=True + ) + assert [item.id for item in named] == ["newer"] + assert not more + assert not has_more + + +@pytest.fixture(autouse=True) +def _identity_workflow_encryption(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep variable serialization focused on ORM behavior, not the external key provider.""" + + monkeypatch.setattr(workflow_fields.encrypter, "encrypt_token", lambda *, token, **_kwargs: token) + monkeypatch.setattr(workflow_fields.encrypter, "decrypt_token", lambda *, token, **_kwargs: token) + + +def test_pipeline_variable_response_accepts_legacy_file_field_names() -> None: + response = workflow_fields.PipelineVariableResponse.model_validate( + { + "label": "Query", + "variable": "query", + "type": "single-file", + "belong_to_node_id": "shared", + "max_length": 0, + "required": False, + "unit": "", + "default_value": "", + "options": [], + "placeholder": "", + "tooltips": "", + "allowed_file_types": [], + "allow_file_extension": [".txt"], + "allow_file_upload_methods": ["remote_url"], + } + ).model_dump(mode="json") + + assert response["allowed_file_extensions"] == [".txt"] + assert response["allowed_file_upload_methods"] == ["remote_url"] + + +def test_pipeline_variable_response_accepts_explicit_null_optional_fields() -> None: + pipeline_variable = RAGPipelineVariable.model_validate( + { + "label": "Query", + "variable": "query", + "type": "text-input", + "belong_to_node_id": "shared", + "max_length": None, + "unit": None, + "default_value": None, + "options": None, + "placeholder": None, + "tooltips": None, + "allowed_file_types": None, + "allowed_file_extensions": None, + "allowed_file_upload_methods": None, + } + ).model_dump(mode="json") + + response = workflow_fields.PipelineVariableResponse.model_validate(pipeline_variable).model_dump(mode="json") + + assert response["max_length"] is None + assert response["allowed_file_types"] is None + assert response["allowed_file_extensions"] is None + assert response["allowed_file_upload_methods"] is None + + +def test_workflow_response_masks_secret_environment_variables(sqlite_session: Session) -> None: + workflow = make_workflow( + environment_variables=[ + SecretVariable(id="env-secret", name="API_KEY", value="plain-token", selector=["env", "API_KEY"]), + StringVariable(id="env-string", name="REGION", value="us-east-1", selector=["env", "REGION"]), + ] + ) + sqlite_session.add_all([make_account(), workflow]) + sqlite_session.commit() + + record = workflow_record(workflow, sqlite_session) + sqlite_session.close() + response = workflow_fields.WorkflowResponse.model_validate( + record, + from_attributes=True, + ).model_dump(mode="json") + + assert response["environment_variables"] == [ + { + "id": "env-secret", + "name": "API_KEY", + "value": workflow_fields.encrypter.full_mask_token(), + "value_type": "secret", + "description": "", + }, + { + "id": "env-string", + "name": "REGION", + "value": "us-east-1", + "value_type": "string", + "description": "", + }, + ] + + +def test_workflow_response_preserves_llm_environment_variable_type(sqlite_session: Session) -> None: + workflow = make_workflow( + environment_variables=[ + LLMEnvironmentVariable( + id="env-llm", + name="for_summarize", + value={"provider": "provider", "name": "model", "mode": "chat"}, + selector=["env", "for_summarize"], + ) + ] + ) + sqlite_session.add_all([make_account(), workflow]) + sqlite_session.commit() + + record = workflow_record(workflow, sqlite_session) + sqlite_session.close() + response = workflow_fields.WorkflowResponse.model_validate( + record, + from_attributes=True, + ).model_dump(mode="json") + + assert response["environment_variables"] == [ + { + "id": "env-llm", + "name": "for_summarize", + "value": {"provider": "provider", "name": "model", "mode": "chat"}, + "value_type": "llm", + "description": "", + } + ] + + +def test_workflow_response_rejects_invalid_environment_variable_dict(sqlite_session: Session) -> None: + data = asdict(workflow_record(make_workflow(), sqlite_session)) + data["environment_variables"] = [{"value_type": "not-a-segment-type"}] + sqlite_session.close() + with pytest.raises(ValidationError): + workflow_fields.WorkflowResponse.model_validate(data) + + +@pytest.mark.parametrize("owner", ["snippet", "pipeline"]) +@pytest.mark.parametrize("operation", ["draft", "published", "list", "update"]) +def test_workflow_records_serialize_after_session_closes( + sqlite_session: Session, + owner: str, + operation: str, +) -> None: + workflow = make_workflow(workflow_id="workflow-1", app_id="owner-1") + workflow.kind = WorkflowKind.SNIPPET if owner == "snippet" else WorkflowKind.STANDARD + workflow.version = "draft" if operation == "draft" else "2026-10-02 12:00:00" + workflow.updated_by = "editor-1" + author = make_account() + editor = make_account(account_id="editor-1", email="editor@example.com") + sqlite_session.add_all( + [ + workflow, + author, + editor, + WorkflowToolProvider( + name="workflow-tool", + label="Workflow", + icon="", + app_id="owner-1", + version=workflow.version, + user_id=author.id, + tenant_id=workflow.tenant_id, + description="", + ), + ] + ) + sqlite_session.commit() + if owner == "snippet": + snippet = CustomizedSnippet( + id="owner-1", + tenant_id="tenant-1", + name="Snippet", + description="", + type="node", + created_by=author.id, + workflow_id=workflow.id, + ) + snippets = build_snippet_service(session=sqlite_session) + match operation: + case "draft": + record = snippets.get_draft_workflow_record(snippet) + case "published": + record = snippets.get_published_workflow_record(snippet) + case "list": + records, _ = snippets.get_all_published_workflows( + session=sqlite_session, snippet=snippet, page=1, limit=20 + ) + record = records[0] + case _: + record = snippets.update_workflow( + session=sqlite_session, + snippet=snippet, + workflow_id=workflow.id, + account=editor, + data={"marked_name": "Updated"}, + ) + else: + pipeline = Pipeline(tenant_id="tenant-1", name="Pipeline", description="") + pipeline.id = "owner-1" + pipeline.workflow_id = workflow.id + pipelines = RagPipelineService(sqlite_session) + match operation: + case "draft": + record = pipelines.get_draft_workflow_record(pipeline) + case "published": + record = pipelines.get_published_workflow_record(pipeline) + case "list": + records, _ = pipelines.get_all_published_workflow( + session=sqlite_session, pipeline=pipeline, page=1, limit=20, user_id=None + ) + record = records[0] + case _: + record = pipelines.update_workflow( + session=sqlite_session, + account_id=editor.id, + data={"marked_name": "Updated"}, + workflow_ref=WorkflowRef(tenant_id="tenant-1", owner_id="owner-1", workflow_id=workflow.id), + ) + assert record is not None + engine = sqlite_session.get_bind() + sqlite_session.commit() + sqlite_session.close() + + def unexpected_query(*_args: object) -> None: + pytest.fail("workflow serialization must not execute SQL") + + event.listen(engine, "before_cursor_execute", unexpected_query) + try: + response = workflow_fields.WorkflowResponse.model_validate(record, from_attributes=True).model_dump(mode="json") + finally: + event.remove(engine, "before_cursor_execute", unexpected_query) + assert response["created_by"]["id"] == "account-1" + assert response["updated_by"]["id"] == "editor-1" + assert response["tool_published"] is True + assert response["graph"] == record.graph + if operation == "update": + assert response["marked_name"] == "Updated" diff --git a/api/tests/unit_tests/libs/test_cron_compatibility.py b/api/tests/unit_tests/libs/test_cron_compatibility.py index 6f3a94f6dc47cd..200efcb0b042a2 100644 --- a/api/tests/unit_tests/libs/test_cron_compatibility.py +++ b/api/tests/unit_tests/libs/test_cron_compatibility.py @@ -292,10 +292,10 @@ def test_execution_time_calculator_pattern(self): assert local_time.minute == 30 assert local_time.second == 0 - def test_schedule_service_integration(self): - """Test integration with ScheduleService patterns.""" + def test_schedule_policy_integration(self): + """Test visual scheduling policy output against the scheduler.""" from core.workflow.nodes.trigger_schedule.entities import VisualConfig - from services.trigger.schedule_service import ScheduleService + from services.trigger.schedule_policy import visual_to_cron # Test enhanced syntax through visual config conversion visual_configs = [ @@ -315,7 +315,7 @@ def test_schedule_service_integration(self): for test_case in visual_configs: with self.subTest(frequency=test_case["frequency"]): - cron_expr = ScheduleService.visual_to_cron(test_case["frequency"], test_case["config"]) + cron_expr = visual_to_cron(test_case["frequency"], test_case["config"]) assert cron_expr == test_case["expected_cron"] # Verify the generated cron expression is valid diff --git a/api/tests/unit_tests/models/test_agent.py b/api/tests/unit_tests/models/test_agent.py index fb2e834e8acfe0..2d31a9c42f4aa0 100644 --- a/api/tests/unit_tests/models/test_agent.py +++ b/api/tests/unit_tests/models/test_agent.py @@ -5,6 +5,7 @@ import sqlalchemy as sa from sqlalchemy.exc import IntegrityError +from enums.agent import WorkflowAgentBindingType from models.agent import ( Agent, AgentConfigDraft, @@ -17,7 +18,6 @@ AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import AgentSoulConfig diff --git a/api/tests/unit_tests/models/test_app_models.py b/api/tests/unit_tests/models/test_app_models.py index f1a1e8280df3e3..fd6a25c7bc759b 100644 --- a/api/tests/unit_tests/models/test_app_models.py +++ b/api/tests/unit_tests/models/test_app_models.py @@ -11,7 +11,6 @@ import json from datetime import UTC, datetime from decimal import Decimal -from unittest.mock import patch from uuid import UUID, uuid4 import pytest @@ -374,56 +373,6 @@ def test_workflow_with_session_returns_none_when_unpublished(self, sqlite_sessio with Session(expire_on_commit=False) as unbound_session: assert app.workflow_with_session(session=unbound_session) is None - @pytest.mark.parametrize("sqlite_session", [(App, AppModelConfig)], indirect=True) - def test_deleted_tools_checks_plugin_builtin_providers_through_core_plugin_service(self, sqlite_session: Session): - """Plugin-backed built-in tools are checked through core PluginService.""" - # Arrange - app = App( - tenant_id="tenant-1", - name="Test App", - mode=AppMode.CHAT, - enable_site=True, - enable_api=False, - created_by=str(uuid4()), - ) - sqlite_session.add(app) - sqlite_session.flush() - app_model_config = AppModelConfig( - app_id=app.id, - agent_mode=json.dumps( - { - "enabled": True, - "strategy": "function_call", - "tools": [ - { - "provider_type": "builtin", - "provider_id": "langgenius/openai/openai", - "tool_name": "chat", - "tool_parameters": {}, - } - ], - "prompt": None, - } - ), - ) - sqlite_session.add(app_model_config) - sqlite_session.flush() - app.app_model_config_id = app_model_config.id - sqlite_session.flush() - - # Act - with ( - patch("core.tools.tool_manager.ToolManager.get_hardcoded_provider", side_effect=Exception), - patch("core.plugin.plugin_service.PluginService.check_tools_existence", return_value=[False]) as exists, - ): - result = app.deleted_tools_with_session(session=sqlite_session) - - # Assert - assert result == [{"type": "builtin", "tool_name": "chat", "provider_id": "langgenius/openai/openai"}] - exists.assert_called_once() - assert exists.call_args.args[0] == "tenant-1" - assert [str(provider_id) for provider_id in exists.call_args.args[1]] == ["langgenius/openai/openai"] - class TestAppModelConfig: """Test suite for AppModelConfig model.""" diff --git a/api/tests/unit_tests/models/test_installed_app_session_accessors.py b/api/tests/unit_tests/models/test_installed_app_session_accessors.py index 0b1f5a040580c1..74366ece1a8935 100644 --- a/api/tests/unit_tests/models/test_installed_app_session_accessors.py +++ b/api/tests/unit_tests/models/test_installed_app_session_accessors.py @@ -1,51 +1,51 @@ -"""Regression coverage for the ``@property``→session-parameter refactor on -``InstalledApp.tenant``. - -The legacy ``@property`` reached for the global ``db.session`` internally and has been converted -to a plain method taking an explicit ``session: Session`` (per the pattern established in -#40370/#40797/#41394/#41830/#41885, tracked in #40372). -""" - -from uuid import uuid4 - -from sqlalchemy.orm import Session - -from models.account import Tenant -from models.model import InstalledApp - - -def _persist_tenant(session: Session) -> Tenant: - tenant = Tenant(name="Test Tenant") - session.add(tenant) - session.flush() - return tenant - - -def _persist_installed_app(session: Session, *, tenant_id: str) -> InstalledApp: - installed_app = InstalledApp( - tenant_id=tenant_id, - app_id=str(uuid4()), - app_owner_tenant_id=str(uuid4()), - position=0, - is_pinned=False, - last_used_at=None, - ) - session.add(installed_app) - session.flush() - return installed_app - - -class TestInstalledAppTenant: - def test_returns_the_matching_tenant(self, sqlite_session: Session) -> None: - tenant = _persist_tenant(sqlite_session) - installed_app = _persist_installed_app(sqlite_session, tenant_id=tenant.id) - - result = installed_app.tenant(session=sqlite_session) - - assert result is not None - assert result.id == tenant.id - - def test_returns_none_when_tenant_missing(self, sqlite_session: Session) -> None: - installed_app = _persist_installed_app(sqlite_session, tenant_id=str(uuid4())) - - assert installed_app.tenant(session=sqlite_session) is None +"""Regression coverage for the ``@property``→session-parameter refactor on +``InstalledApp.tenant``. + +The legacy ``@property`` reached for the global ``db.session`` internally and has been converted +to a plain method taking an explicit ``session: Session`` (per the pattern established in +#40370/#40797/#41394/#41830/#41885, tracked in #40372). +""" + +from uuid import uuid4 + +from sqlalchemy.orm import Session + +from models.account import Tenant +from models.model import InstalledApp + + +def _persist_tenant(session: Session) -> Tenant: + tenant = Tenant(name="Test Tenant") + session.add(tenant) + session.flush() + return tenant + + +def _persist_installed_app(session: Session, *, tenant_id: str) -> InstalledApp: + installed_app = InstalledApp( + tenant_id=tenant_id, + app_id=str(uuid4()), + app_owner_tenant_id=str(uuid4()), + position=0, + is_pinned=False, + last_used_at=None, + ) + session.add(installed_app) + session.flush() + return installed_app + + +class TestInstalledAppTenant: + def test_returns_the_matching_tenant(self, sqlite_session: Session) -> None: + tenant = _persist_tenant(sqlite_session) + installed_app = _persist_installed_app(sqlite_session, tenant_id=tenant.id) + + result = installed_app.tenant(session=sqlite_session) + + assert result is not None + assert result.id == tenant.id + + def test_returns_none_when_tenant_missing(self, sqlite_session: Session) -> None: + installed_app = _persist_installed_app(sqlite_session, tenant_id=str(uuid4())) + + assert installed_app.tenant(session=sqlite_session) is None diff --git a/api/tests/unit_tests/models/test_workflow_node_execution_accessors.py b/api/tests/unit_tests/models/test_workflow_node_execution_accessors.py index cdb07111dce97e..3295d4a47a8c60 100644 --- a/api/tests/unit_tests/models/test_workflow_node_execution_accessors.py +++ b/api/tests/unit_tests/models/test_workflow_node_execution_accessors.py @@ -7,15 +7,19 @@ with ``from_attributes=True``. """ +import json from uuid import uuid4 +import pytest from sqlalchemy.orm import Session -from fields.workflow_run_fields import node_execution_response_source +from fields.workflow_run_fields import WorkflowRunNodeExecutionResponse, node_execution_response_source +from graphon.enums import BuiltinNodeTypes from models.account import Account from models.enums import CreatorUserRole, EndUserType from models.model import EndUser from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom +from services.tools.tool_manager import ToolManager def _execution(role: CreatorUserRole, created_by: str) -> WorkflowNodeExecutionModel: @@ -95,13 +99,40 @@ def test_returns_none_when_role_is_account(self, sqlite_session: Session) -> Non class TestNodeExecutionResponseSource: - def test_accessors_resolve_via_wrapped_session_and_other_attributes_proxy(self, sqlite_session: Session) -> None: + @pytest.mark.parametrize("node_type", [BuiltinNodeTypes.TOOL, BuiltinNodeTypes.DATASOURCE]) + def test_provider_icons_survive_response_serialization( + self, sqlite_session, monkeypatch, node_type, *, tool_providers + ): + execution = _execution(CreatorUserRole.ACCOUNT, created_by="account") + execution.id = "execution" + execution.node_type = node_type + execution.execution_metadata = json.dumps( + { + "tool_info": {"provider_type": "builtin", "provider_id": "provider"}, + "datasource_info": {"icon": "source-icon"}, + } + ) + + def tool_icon(*, tenant_id, provider_type, provider_id, tool_providers): + assert tool_providers is not None + assert (tenant_id, provider_type, provider_id) == (execution.tenant_id, "builtin", "provider") + return "tool-icon" + + monkeypatch.setattr(ToolManager, "get_tool_icon", tool_icon) + response = WorkflowRunNodeExecutionResponse.model_validate( + node_execution_response_source(execution, session=sqlite_session, tool_providers=tool_providers) + ) + assert response.extras == {"icon": "tool-icon" if node_type == BuiltinNodeTypes.TOOL else "source-icon"} + + def test_accessors_resolve_via_wrapped_session_and_other_attributes_proxy( + self, sqlite_session: Session, *, tool_providers + ) -> None: account = Account(name="Test Account", email="test@example.com") sqlite_session.add(account) sqlite_session.flush() execution = _execution(CreatorUserRole.ACCOUNT, created_by=account.id) - source = node_execution_response_source(execution, session=sqlite_session) + source = node_execution_response_source(execution, session=sqlite_session, tool_providers=tool_providers) resolved = source.created_by_account assert resolved is not None diff --git a/api/tests/unit_tests/pyrefly.toml b/api/tests/unit_tests/pyrefly.toml index 8a8e78cf1f84af..ef7cecf5a3a35c 100644 --- a/api/tests/unit_tests/pyrefly.toml +++ b/api/tests/unit_tests/pyrefly.toml @@ -57,10 +57,8 @@ project-excludes = [ "controllers/console/app/test_model_config_api.py", "controllers/console/app/test_ops_trace_api.py", "controllers/console/app/test_statistic_api.py", - "controllers/console/app/test_workflow.py", "controllers/console/app/test_workflow_app_log_api.py", "controllers/console/app/test_workflow_comment_api.py", - "controllers/console/app/test_workflow_convert_api.py", "controllers/console/app/test_workflow_node_output_inspector.py", "controllers/console/app/test_workflow_pause_details_api.py", "controllers/console/app/test_workflow_run_api.py", @@ -104,7 +102,6 @@ project-excludes = [ "controllers/console/explore/test_trial.py", "controllers/console/explore/test_workflow.py", "controllers/console/explore/test_wraps.py", - "controllers/console/snippets/test_snippet_workflow.py", "controllers/console/snippets/test_snippet_workflow_draft_variable.py", "controllers/console/tag/test_tags.py", "controllers/console/test_document_detail_api_data_source_info.py", @@ -863,7 +860,6 @@ project-excludes = [ "services/workflow/test_workflow_draft_variable_service.py", "services/workflow/test_workflow_event_snapshot_service.py", "services/workflow/test_workflow_event_snapshot_service_additional.py", - "services/workflow/test_workflow_human_input_delivery.py", "services/workflow/test_workflow_restore.py", "tasks/test_agent_backend_session_cleanup_task.py", "tasks/test_async_workflow_tasks.py", diff --git a/api/tests/unit_tests/repositories/test_agent_execution_repository.py b/api/tests/unit_tests/repositories/test_agent_execution_repository.py new file mode 100644 index 00000000000000..33646b8d7c1993 --- /dev/null +++ b/api/tests/unit_tests/repositories/test_agent_execution_repository.py @@ -0,0 +1,148 @@ +"""Execution retention uses candidate-indexed references rather than graph scans.""" + +from datetime import timedelta +from functools import partial + +import pytest +from sqlalchemy import event, select +from sqlalchemy.orm import Session, sessionmaker + +from graphon.enums import WorkflowExecutionStatus +from libs.datetime_utils import naive_utc_now +from models.agent import WorkflowAgentNodeBinding +from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom +from models.workflow import WorkflowRun, WorkflowType +from repositories.agent.runtime_repository import WorkflowAgentExecutionRepository +from repositories.workflow.execution_write_repository import WorkflowExecutionWriteRepository +from tests.unit_tests.workflow_execution import debug_lease, execution_binding, set_debug_deadline + + +def _run(run_id: str, status: WorkflowExecutionStatus) -> WorkflowRun: + return WorkflowRun( + id=run_id, + tenant_id="tenant-1", + app_id="app-1", + workflow_id="workflow-1", + type=WorkflowType.WORKFLOW, + triggered_from=WorkflowRunTriggeredFrom.DEBUGGING, + version="draft", + status=status, + created_by_role=CreatorUserRole.ACCOUNT, + created_by="account-1", + graph='{"note": "Retention must not select or decode this graph"}', + ) + + +def _reference(run_id: str, agent_id: str) -> WorkflowAgentNodeBinding: + return execution_binding(workflow_run_id=run_id, tenant_id="tenant-1", app_id="app-1", agent_id=agent_id) + + +def test_retention_queries_candidates_through_agent_index(sqlite_session_factory: sessionmaker[Session]) -> None: + with sqlite_session_factory.begin() as session: + for i in range(100): + run_id = f"unrelated-{i}" + session.add(_run(run_id, WorkflowExecutionStatus.RUNNING)) + session.flush() + session.add(_reference(run_id, f"other-agent-{i}")) + for run_id, status in ( + ("running", WorkflowExecutionStatus.RUNNING), + ("paused", WorkflowExecutionStatus.PAUSED), + ("ended", WorkflowExecutionStatus.STOPPED), + ): + session.add(_run(run_id, status)) + session.flush() + session.add(_reference(run_id, f"agent-{run_id}")) + statements: list[tuple[str, tuple[object, ...]]] = [] + engine = sqlite_session_factory.kw["bind"] + + def record(_conn: object, _cursor: object, sql: str, parameters: tuple[object, ...], *_args: object) -> None: + statements.append((sql, parameters)) + + event.listen(engine, "before_cursor_execute", record) + try: + with sqlite_session_factory() as session: + assert WorkflowAgentExecutionRepository.retained_execution_agent_ids( + session, "tenant-1", ["agent-running", "agent-paused", "agent-ended"] + ) == {"agent-running", "agent-paused"} + assert not session.identity_map # No WorkflowRun graph or ORM rows were materialized. + finally: + event.remove(engine, "before_cursor_execute", record) + assert len(statements) == 1 + sql, parameters = statements[0] + assert "graph" not in sql + with engine.connect() as connection: + plan = connection.exec_driver_sql("EXPLAIN QUERY PLAN " + sql, parameters).all() + assert any("SEARCH" in row[3] and "workflow_agent_node_binding_agent_idx" in row[3] for row in plan) + + +@pytest.mark.parametrize("field", ["tenant_id", "app_id", "workflow_id"]) +def test_retention_requires_complete_owner_chain(sqlite_session_factory: sessionmaker[Session], field: str) -> None: + with sqlite_session_factory.begin() as session: + session.add(_run("run", WorkflowExecutionStatus.PAUSED)) + session.flush() + reference = _reference("run", "agent-1") + setattr(reference, field, "other-owner") + session.add(reference) + with sqlite_session_factory() as session: + assert WorkflowAgentExecutionRepository.retained_execution_agent_ids(session, "tenant-1", ["agent-1"]) == set() + + +@pytest.mark.parametrize("cancel", [False, True]) +def test_execution_finish_keeps_retry_candidates_including_cancelled_reservation( + sqlite_session_factory: sessionmaker[Session], cancel: bool +) -> None: + with sqlite_session_factory.begin() as session: + session.add(_run("run", WorkflowExecutionStatus.RUNNING if cancel else WorkflowExecutionStatus.PAUSED)) + session.flush() + if cancel: + set_debug_deadline(session, "run", naive_utc_now() + timedelta(minutes=1)) + session.add(_reference("run", "agent-1")) + repository = WorkflowAgentExecutionRepository(sqlite_session_factory) + finish = partial( + WorkflowExecutionWriteRepository(sqlite_session_factory).finish, + tenant_id="tenant-1", + app_id="app-1", + workflow_id="workflow-1", + execution_id="run", + ) + if not cancel: + assert finish() == WorkflowExecutionStatus.PAUSED + assert ( + repository.finished_agent_ids( + tenant_id="tenant-1", app_id="app-1", workflow_id="workflow-1", execution_id="run" + ) + == set() + ) + assert finish(status=WorkflowExecutionStatus.STOPPED, cancel=cancel) == WorkflowExecutionStatus.STOPPED + with sqlite_session_factory() as session: + assert WorkflowAgentExecutionRepository.retained_execution_agent_ids(session, "tenant-1", ["agent-1"]) == set() + assert session.scalar(select(WorkflowAgentNodeBinding)) is not None + run = session.get(WorkflowRun, "run") + assert run is not None + assert run.status == WorkflowExecutionStatus.STOPPED + if cancel: + reservation = debug_lease(session, "run") + assert reservation is not None + assert reservation.expires_at <= naive_utc_now() + assert finish(cancel=cancel) == WorkflowExecutionStatus.STOPPED + assert repository.finished_agent_ids( + tenant_id="tenant-1", app_id="app-1", workflow_id="workflow-1", execution_id="run" + ) == {"agent-1"} + + +def test_execution_pins_are_not_reported_as_editable_workflow_nodes(sqlite_session: Session) -> None: + from services.agent.composer_service import AgentComposerService + + pin = _reference("run", "agent-1") + pin.current_snapshot_id = "snapshot-1" + draft = _reference("workflow-1", "agent-1") + draft.workflow_version = "draft" + draft.current_snapshot_id = "snapshot-1" + sqlite_session.add_all([pin, draft]) + sqlite_session.commit() + + impact = AgentComposerService.calculate_impact( + session=sqlite_session, tenant_id="tenant-1", current_snapshot_id="snapshot-1" + ) + assert impact["workflow_node_count"] == 1 + assert impact["bindings"] == [{"app_id": "app-1", "workflow_id": "workflow-1", "node_id": "agent-1"}] diff --git a/api/tests/unit_tests/repositories/test_app_definition_query_repository_agent.py b/api/tests/unit_tests/repositories/test_app_definition_query_repository_agent.py index d483a43c59dbce..a2232fbe9e4247 100644 --- a/api/tests/unit_tests/repositories/test_app_definition_query_repository_agent.py +++ b/api/tests/unit_tests/repositories/test_app_definition_query_repository_agent.py @@ -6,7 +6,6 @@ from sqlalchemy.orm import Session, sessionmaker from core.app.app_config.common.parameters_mapping import get_parameters_from_feature_dict -from core.app.apps.agent_app.errors import AgentAppGeneratorError, AgentAppNotPublishedError from models.agent import ( Agent, AgentConfigRevision, @@ -21,6 +20,7 @@ AppDefinitionQueryRepository, _get_public_agent_parameter_config, ) +from services.app.generation.errors import AgentAppGeneratorError, AgentAppNotPublishedError def _stable_uuid(value: str) -> str: diff --git a/api/tests/unit_tests/repositories/test_app_preview_query_repository.py b/api/tests/unit_tests/repositories/test_app_preview_query_repository.py index 31ea2abadfc045..5206e104d6a705 100644 --- a/api/tests/unit_tests/repositories/test_app_preview_query_repository.py +++ b/api/tests/unit_tests/repositories/test_app_preview_query_repository.py @@ -444,7 +444,7 @@ def serialize_date(value: object) -> str: ) -def test_get_detail_resolves_only_configured_api_providers_owned_by_app_tenant( +def test_get_detail_resolves_only_configured_tool_providers_owned_by_app_tenant( sqlite_session_factory: sessionmaker[Session], preview_ref: AppPreviewRef, detail_config: AppModelConfig ) -> None: owner_provider, viewer_provider, unconfigured_provider = [ diff --git a/api/tests/unit_tests/repositories/test_sqlalchemy_execution_extra_content_repository.py b/api/tests/unit_tests/repositories/test_sqlalchemy_execution_extra_content_repository.py index 384992242b52a0..10b850bd79d17f 100644 --- a/api/tests/unit_tests/repositories/test_sqlalchemy_execution_extra_content_repository.py +++ b/api/tests/unit_tests/repositories/test_sqlalchemy_execution_extra_content_repository.py @@ -10,7 +10,11 @@ from libs.datetime_utils import naive_utc_now from models.execution_extra_content import HumanInputContent as HumanInputContentModel from models.human_input import HumanInputForm -from models.human_input_entities import FormDefinition, ParagraphInputConfig, UserActionConfig +from models.human_input_entities import ( + FormDefinition, + ParagraphInputConfig, + UserActionConfig, +) from repositories.sqlalchemy_execution_extra_content_repository import SQLAlchemyExecutionExtraContentRepository diff --git a/api/tests/unit_tests/repositories/test_workflow_app_log_query_repository.py b/api/tests/unit_tests/repositories/test_workflow_app_log_query_repository.py index 2de5e684a02cbc..a687876d471514 100644 --- a/api/tests/unit_tests/repositories/test_workflow_app_log_query_repository.py +++ b/api/tests/unit_tests/repositories/test_workflow_app_log_query_repository.py @@ -16,8 +16,8 @@ from models.model import EndUser from models.trigger import WorkflowTriggerLog from models.workflow import WorkflowAppLog, WorkflowAppLogCreatedFrom, WorkflowRun, WorkflowType -from repositories.workflow_app_log_query_repository import WorkflowAppLogQueryRepository -from services.workflow_app_log_query_service import WorkflowAppLogAccount, WorkflowAppLogEndUser +from repositories.workflow.app_log_repository import WorkflowAppLogRepository +from services.workflow.app_log_query_service import WorkflowAppLogAccount, WorkflowAppLogEndUser def _log( @@ -116,7 +116,7 @@ def test_get_paginated_returns_detached_actor_records_by_role_when_ids_overlap( sqlite_session.commit() sqlite_session.close() - result = WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory).get_paginated( + result = WorkflowAppLogRepository(session_factory=sqlite_session_factory).get_paginated( tenant_id=tenant.id, app_id="app-1", ) @@ -142,7 +142,7 @@ def test_get_paginated_returns_detached_actor_records_by_role_when_ids_overlap( def test_get_paginated_preserves_missing_account_filter_error( sqlite_session_factory: sessionmaker[Session], ) -> None: - repository = WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory) + repository = WorkflowAppLogRepository(session_factory=sqlite_session_factory) with pytest.raises(ValueError, match=r"^Account not found: missing@example\.com$"): repository.get_paginated( @@ -154,7 +154,7 @@ def test_get_paginated_preserves_missing_account_filter_error( def test_enum_value_rejects_missing_required_value() -> None: with pytest.raises(ValueError, match="Required enum value is missing"): - WorkflowAppLogQueryRepository._enum_value(None) + WorkflowAppLogRepository._enum_value(None) def test_get_paginated_projects_plugin_workflow_run_summary( @@ -171,7 +171,7 @@ def test_get_paginated_projects_plugin_workflow_run_summary( sqlite_session.add_all([run, log]) sqlite_session.commit() - result = WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory).get_paginated( + result = WorkflowAppLogRepository(session_factory=sqlite_session_factory).get_paginated( tenant_id="tenant-1", app_id="app-1", ) @@ -199,7 +199,7 @@ def test_get_paginated_keeps_log_when_workflow_run_is_missing( sqlite_session.add(log) sqlite_session.commit() - repository = WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory) + repository = WorkflowAppLogRepository(session_factory=sqlite_session_factory) result = repository.get_paginated(tenant_id="tenant-1", app_id="app-1") assert result.total == 1 @@ -252,7 +252,7 @@ def test_get_paginated_includes_trigger_metadata_only_with_detail( ) sqlite_session.commit() - repository = WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory) + repository = WorkflowAppLogRepository(session_factory=sqlite_session_factory) assert repository.get_paginated(tenant_id="tenant-1", app_id="app-1").data[0].details is None assert repository.get_paginated( @@ -289,7 +289,7 @@ def test_get_paginated_does_not_attach_workflow_run_from_another_scope( ) sqlite_session.commit() - repository = WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory) + repository = WorkflowAppLogRepository(session_factory=sqlite_session_factory) result = repository.get_paginated(tenant_id="tenant-1", app_id="app-1") assert result.total == 1 diff --git a/api/tests/unit_tests/repositories/workflow/test_definition_repository.py b/api/tests/unit_tests/repositories/workflow/test_definition_repository.py new file mode 100644 index 00000000000000..ac290f4f364f8d --- /dev/null +++ b/api/tests/unit_tests/repositories/workflow/test_definition_repository.py @@ -0,0 +1,1998 @@ +"""Persistence boundaries use real short SQLite sessions, including rollback.""" + +import json +from collections.abc import Callable +from dataclasses import replace +from typing import cast +from unittest.mock import Mock, patch +from uuid import uuid4 + +import pytest +from sqlalchemy import Select, event, func, select +from sqlalchemy.dialects import postgresql +from sqlalchemy.orm import ORMExecuteState, Session, sessionmaker + +from core.workflow.llm_environment_variable import LLMEnvironmentVariable +from enums import DeploymentEdition +from enums.agent import WorkflowAgentBindingType +from extensions.application_services.workflow import build_workflow_definition_gateway +from extensions.application_services.workflow_variables import ( + build_console_workflow_variables, + build_workflow_variable_service, +) +from fields.workflow_fields import WorkflowResponse +from graphon.model_runtime.entities.model_entities import ModelType +from graphon.variables import StringSegment, StringVariable +from machinery.context import RequestContext +from models import Account, App +from models.account import TenantAccountJoin, TenantAccountRole +from models.agent import WorkflowAgentNodeBinding +from models.agent_config_entities import WorkflowNodeJobConfig +from models.dataset import AppDatasetJoin +from models.enums import CredentialSourceType +from models.model import AppMode +from models.provider import LoadBalancingModelConfig +from models.tools import BuiltinToolProvider +from models.workflow import Workflow, WorkflowDraftVariable, WorkflowVersionCounter +from repositories.app.console_repository import ConsoleAppRepository +from repositories.credentials.query_repository import CredentialQueryRepository +from repositories.workflow.definition_repository import ( + WorkflowDefinitionRepository, + WorkflowDefinitionStore, + workflow_snapshot, +) +from repositories.workflow.draft_repository import WorkflowDraftRepository +from services import workflow_service as workflow_module +from services.agent.workflow_contracts import AgentSkillReader, WorkflowAgentBindingStore +from services.agent.workflow_publish_service import WorkflowAgentPublishService +from services.app.console_service import ConsoleAppNotFoundError +from services.data_migration.entities import ImportTarget +from services.data_migration.import_service import MigrationImportService +from services.errors.app import TriggerNodeLimitExceededError, WorkflowHashNotEqualError +from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError +from services.workflow import definition_gateway as gateway_module +from services.workflow.console_service import ( + ConsoleWorkflowService, + WorkflowAccess, + WorkflowAppLookup, + WorkflowConversion, + WorkflowPresence, + WorkflowRuntime, +) +from services.workflow.contracts import ( + DraftSyncCommand, + PreparedDraftSync, + ValidatedWorkflowPublication, + WorkflowOwner, + WorkflowPublication, + WorkflowSnapshot, +) +from services.workflow.definition_gateway import WorkflowDefinitionGateway +from services.workflow.draft_service import DraftAgentRetirement, WorkflowDraftService +from services.workflow.environment_variable_service import prepare_environment_variables +from services.workflow_service import WorkflowService +from tests.unit_tests.model_factories import make_account, make_app, make_tenant, make_workflow + +CONTEXT = RequestContext("request", None, "account-1", "tenant-1") + + +@pytest.mark.parametrize("kind", ["tool", "agent", "llm"]) +@pytest.mark.parametrize("denied", [False, True]) +def test_publication_credential_policy_runs_after_read_transactions_close( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, kind: str, denied: bool +) -> None: + provider = "langgenius/openai/openai" + tool = BuiltinToolProvider( + tenant_id="tenant-1", user_id="account-1", provider=provider, name="default", is_default=True + ) + data: dict[str, object] = {"type": kind, "provider_id": provider} + if kind == "agent": + data["agent_parameters"] = {"tools": {"value": [{"provider_name": provider}]}} + if kind == "llm": + data["model"] = {"provider": provider, "name": "model"} + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = json.dumps({"nodes": [{"id": "node", "data": data}], "edges": []}) + session.add(tool) + for tenant, credential, source in [ + ("tenant-1", "provider-credential", CredentialSourceType.PROVIDER), + ("tenant-1", "model-credential", CredentialSourceType.CUSTOM_MODEL), + ("foreign-tenant", "foreign-credential", CredentialSourceType.PROVIDER), + ]: + session.add( + LoadBalancingModelConfig( + tenant_id=tenant, + provider_name=provider, + model_name="model", + model_type=ModelType.LLM, + name=credential, + credential_id=credential, + credential_source_type=source, + ) + ) + sessions: list[Session] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + checked: list[str] = [] + + def check(credential_id: str, *_args: object, **_kwargs: object) -> None: + assert sessions + assert all(not session.in_transaction() for session in sessions) + checked.append(credential_id) + if denied: + raise ValueError("credential policy denied") + + workflows = WorkflowService(session_maker=seeded) + monkeypatch.setattr(workflow_module.SystemFeatureService, "is_plugin_manager_enabled", lambda: True) + monkeypatch.setattr(workflows, "validate_graph_structure", lambda **_kwargs: None) + monkeypatch.setattr(workflows, "_validate_llm_model_config", lambda *_args: None) + monkeypatch.setattr(workflows, "_is_load_balancing_enabled", lambda *_args: True) + monkeypatch.setattr("core.helper.credential_utils.check_credential_policy_compliance", check) + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=workflows, + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + event.listen(seeded, "after_begin", opened) + try: + if denied: + with pytest.raises(ValueError, match="credential policy denied"): + lifecycle.validate_publish(CONTEXT, "app-1") + else: + assert lifecycle.validate_publish(CONTEXT, "app-1").workflow.id == "draft" + expected = {"provider-credential", "model-credential"} if kind == "llm" else {tool.id} + assert set(checked) == expected + assert checked + assert all(not session.in_transaction() for session in sessions) + finally: + event.remove(seeded, "after_begin", opened) + + +@pytest.fixture +def seeded(sqlite_session_factory: sessionmaker[Session]) -> sessionmaker[Session]: + with sqlite_session_factory.begin() as session: + session.add_all( + [ + make_account(), + make_tenant(), + TenantAccountJoin(tenant_id="tenant-1", account_id="account-1", role=TenantAccountRole.OWNER), + make_app(workflow_id="published", mode=AppMode.WORKFLOW, created_by="account-1"), + make_workflow(workflow_id="draft"), + make_workflow(workflow_id="published", version="1", version_number=1), + WorkflowVersionCounter(app_id="app-1", last_version_number=1), + ] + ) + return sqlite_session_factory + + +@pytest.fixture +def repository(seeded: sessionmaker[Session]) -> WorkflowDefinitionRepository: + return WorkflowDefinitionRepository(session_factory=seeded) + + +def test_published_result_is_complete_after_session_closes(repository: WorkflowDefinitionRepository) -> None: + record = repository.published(CONTEXT, "app-1") + assert record is not None + response = WorkflowResponse.model_validate(record, from_attributes=True).model_dump(mode="json") + assert response["id"] == "published" + assert response["version_number"] == 1 + assert response["created_by"]["id"] == "account-1" + assert response["tool_published"] is False + + +def test_update_version_remarks_masks_secrets_without_key_io( + seeded: sessionmaker[Session], repository: WorkflowDefinitionRepository, monkeypatch: pytest.MonkeyPatch +) -> None: + from core.helper.encrypter import full_mask_token + + environment = json.dumps( + { + "API_KEY": {"id": "secret-id", "name": "API_KEY", "value_type": "secret", "value": "ciphertext"}, + "COUNT": {"id": "number-id", "name": "COUNT", "value_type": "number", "value": 3}, + } + ) + with seeded.begin() as session: + workflow = session.get(Workflow, "published") + assert workflow is not None + workflow._environment_variables = environment + sessions: list[Session] = [] + + def opened(session: Session, *_args: object) -> None: + sessions.append(session) + + def key_io(**_kwargs: object) -> str: + assert all(not session.in_transaction() for session in sessions) + pytest.fail("A display-only projection must not wrap or unwrap secret values") + + monkeypatch.setattr("core.helper.encrypter.decrypt_token", key_io) + monkeypatch.setattr("core.helper.encrypter.encrypt_token", key_io) + event.listen(seeded, "after_begin", opened) + try: + record = repository.update(CONTEXT, "app-1", "published", {"marked_comment": "New remarks"}) + assert record is not None + assert sessions + assert all(not session.in_transaction() for session in sessions) + assert {variable.name: variable.value for variable in record.environment_variables} == { + "API_KEY": full_mask_token(), + "COUNT": 3, + } + response = WorkflowResponse.model_validate(record, from_attributes=True) + assert response.marked_comment == "New remarks" + assert response.environment_variables[0].value == full_mask_token() + finally: + event.remove(seeded, "after_begin", opened) + with seeded() as session: + workflow = session.get(Workflow, "published") + assert workflow is not None + assert workflow.marked_comment == "New remarks" + assert workflow.updated_by == CONTEXT.account_id + assert workflow._environment_variables == environment + + +def test_version_counter_allocation_is_rolled_back_with_publication(seeded: sessionmaker[Session]) -> None: + from repositories.workflow.version_repository import allocate_version_number + + with seeded.begin() as session: + assert allocate_version_number(session=session, app_id="app-1") == 2 + with seeded() as session, session.begin(): + assert allocate_version_number(session=session, app_id="app-1") == 3 + session.rollback() + with seeded.begin() as session: + assert allocate_version_number(session=session, app_id="app-1") == 3 + + +@pytest.mark.parametrize("concurrent_edit", [False, True]) +def test_publication_copies_secret_ciphertext_without_key_io_under_locks( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, concurrent_edit: bool +) -> None: + ciphertext = json.dumps( + { + "secret-id": { + "id": "secret-id", + "name": "API_KEY", + "value_type": "secret", + "value": "wrapped-secret", + } + } + ) + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft._environment_variables = ciphertext + sessions: list[Session] = [] + unwrapped: list[str] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def unwrap(*, tenant_id: str, token: str) -> str: + assert tenant_id == "tenant-1" + assert all(not session.in_transaction() for session in sessions) + unwrapped.append(token) + if concurrent_edit: + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft._environment_variables = ciphertext.replace("wrapped-secret", "changed-secret") + return "plaintext" + + def wrap(**_kwargs: object) -> str: + pytest.fail("Publishing within the same tenant must preserve the encrypted value") + + monkeypatch.setattr("core.helper.encrypter.decrypt_token", unwrap) + monkeypatch.setattr("core.helper.encrypter.encrypt_token", wrap) + console_variables = build_console_workflow_variables( + database_client=seeded, variables=build_workflow_variable_service(database_client=seeded) + ) + with seeded.begin() as session: + session.get(App, "app-1").mode = AppMode.ADVANCED_CHAT + workflows = WorkflowService(seeded) + monkeypatch.setattr(workflows, "validate_graph_structure", lambda **_kwargs: None) + definitions = WorkflowDefinitionRepository(session_factory=seeded) + service = application( + build_workflow_definition_gateway(seeded, definitions, workflows, drafts=WorkflowDraftRepository(seeded)) + ) + event.listen(seeded, "after_begin", opened) + try: + if concurrent_edit: + with pytest.raises(WorkflowHashNotEqualError): + service.publish(CONTEXT, "app-1", marked_name="Secrets", marked_comment="") + else: + service.publish(CONTEXT, "app-1", marked_name="Secrets", marked_comment="") + finally: + event.remove(seeded, "after_begin", opened) + assert unwrapped == ["wrapped-secret"] + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + if concurrent_edit: + assert app.workflow_id == "published" + else: + assert app.workflow_id != "published" + published = session.get(Workflow, app.workflow_id) + assert published is not None + assert published._environment_variables == ciphertext + + +@pytest.mark.parametrize("entry", ["sync", "collaborative", "replace", "patch"]) +@pytest.mark.parametrize("concurrent_edit", [False, True]) +def test_draft_secret_preparation_releases_sessions_and_rechecks_snapshot( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, entry: str, concurrent_edit: bool +) -> None: + from constants import HIDDEN_VALUE + from graphon.variables import SecretVariable + + original = json.dumps( + {"API_KEY": {"id": "secret-id", "name": "API_KEY", "value_type": "secret", "value": "wrapped"}} + ) + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft._environment_variables = original + source = workflow_snapshot(draft) + + sessions: list[Session] = [] + operations: list[str] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def decrypt(*, tenant_id: str, token: str) -> str: + assert tenant_id == "tenant-1" + assert token == "wrapped" + assert all(not session.in_transaction() for session in sessions) + operations.append("decrypt") + return "old-secret" + + def encrypt(*, tenant_id: str, token: str) -> str: + assert tenant_id == "tenant-1" + assert all(not session.in_transaction() for session in sessions) + operations.append("encrypt") + if concurrent_edit and operations.count("encrypt") == 1: + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = '{"nodes": [], "edges": [], "concurrent": true}' + return "encrypted:" + token + + monkeypatch.setattr("core.helper.encrypter.decrypt_token", decrypt) + monkeypatch.setattr("core.helper.encrypter.encrypt_token", encrypt) + console_variables = build_console_workflow_variables( + database_client=seeded, variables=build_workflow_variable_service(database_client=seeded) + ) + with seeded.begin() as session: + session.get(App, "app-1").mode = AppMode.ADVANCED_CHAT + workflows = WorkflowService(seeded) + monkeypatch.setattr(workflows, "validate_graph_structure", lambda *_args, **_kwargs: None) + monkeypatch.setattr(workflows, "validate_features_structure", lambda *_args: None) + definitions = WorkflowDefinitionRepository(session_factory=seeded) + service = application( + build_workflow_definition_gateway(seeded, definitions, workflows, drafts=WorkflowDraftRepository(seeded)) + ) + variables = [ + SecretVariable(id="secret-id", name="RENAMED_KEY", value=HIDDEN_VALUE), + SecretVariable(id="new-secret", name="NEW_KEY", value="new-secret"), + ] + + def save() -> None: + if entry in ("replace", "patch"): + console_variables.update_environment( + CONTEXT, + "app-1", + [variable.model_dump(mode="json") for variable in variables], + deleted_ids=[] if entry == "patch" else None, + ) + else: + mappings = [variable.model_dump(mode="json") for variable in variables] + service.sync( + CONTEXT, + "app-1", + DraftSyncCommand( + graph={"nodes": [], "edges": [], "saved": True}, + features={}, + unique_hash=source.hash, + is_collaborative=entry == "collaborative", + conversation_variables=[], + environment_variables=mappings if entry == "sync" else None, + environment_upserts=mappings if entry == "collaborative" else None, + environment_deletions=[], + ), + ) + + event.listen(seeded, "after_begin", opened) + try: + if concurrent_edit and entry in ("sync", "collaborative"): + with pytest.raises(WorkflowHashNotEqualError): + save() + else: + save() + finally: + event.remove(seeded, "after_begin", opened) + assert "decrypt" in operations + assert "encrypt" in operations + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + if concurrent_edit and entry in ("sync", "collaborative"): + assert draft._environment_variables == original + else: + values = json.loads(draft._environment_variables) + assert values["RENAMED_KEY"]["value"] == "encrypted:old-secret" + assert values["NEW_KEY"]["value"] == "encrypted:new-secret" + if concurrent_edit: + assert draft.graph_dict["concurrent"] is True + + +@pytest.mark.parametrize("rollback", [False, True]) +def test_dsl_secret_preparation_precedes_the_atomic_import_transaction( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, rollback: bool +) -> None: + import yaml + + from constants.dsl_version import CURRENT_APP_DSL_VERSION + from extensions.application_services.workflow import build_app_dsl_service + from services.entities.dsl_entities import ImportStatus + + sessions: list[Session] = [] + encrypted: list[str] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def encrypt(*, tenant_id: str, token: str) -> str: + assert tenant_id == "tenant-1" + assert all(not session.in_transaction() for session in sessions) + encrypted.append(token) + return "encrypted:" + token + + monkeypatch.setattr("core.helper.encrypter.encrypt_token", encrypt) + monkeypatch.setattr(WorkflowService, "validate_graph_structure", lambda *_args, **_kwargs: None) + monkeypatch.setattr(WorkflowService, "validate_features_structure", lambda *_args, **_kwargs: None) + event.listen(Session, "after_begin", opened) + try: + with seeded() as session: + result = build_app_dsl_service(session).import_app( + account=make_account(tenant=make_tenant()), + import_mode="yaml-content", + app_id="app-1", + yaml_content=yaml.safe_dump( + { + "version": CURRENT_APP_DSL_VERSION, + "kind": "app", + "app": {"mode": "workflow", "name": "Imported"}, + "workflow": { + "graph": {"nodes": [], "edges": []}, + "features": {}, + "environment_variables": [ + {"id": "secret-id", "name": "KEY", "value_type": "secret", "value": "secret"} + ], + }, + } + ), + ) + assert result.status == ImportStatus.COMPLETED, result.error + if rollback: + session.rollback() + else: + session.commit() + finally: + event.remove(Session, "after_begin", opened) + assert encrypted == ["secret"] + with seeded() as session: + draft = session.get(Workflow, "draft") + app = session.get(App, "app-1") + assert draft is not None + assert app is not None + if rollback: + assert json.loads(draft._environment_variables) == {} + assert app.name != "Imported" + else: + assert json.loads(draft._environment_variables)["KEY"]["value"] == "encrypted:secret" + assert app.name == "Imported" + + +@pytest.mark.parametrize("rollback", [False, True]) +def test_workflow_and_bindings_delete_atomically_before_agent_retirement( + seeded: sessionmaker[Session], rollback: bool +) -> None: + with seeded.begin() as session: + session.add(make_workflow(workflow_id="obsolete", version="old-version")) + session.add_all( + WorkflowAgentNodeBinding( + id=f"binding-{node}", + tenant_id="tenant-1", + app_id="app-1", + workflow_id="obsolete", + workflow_version="old-version", + node_id=node, + binding_type=kind, + agent_id=agent, + node_job_config={}, + ) + for node, kind, agent in [ + ("one", WorkflowAgentBindingType.INLINE_AGENT, "inline-agent"), + ("two", WorkflowAgentBindingType.INLINE_AGENT, "inline-agent"), + ("roster", WorkflowAgentBindingType.ROSTER_AGENT, "roster-agent"), + ] + ) + definitions = WorkflowDefinitionRepository(session_factory=seeded) + service = application( + build_workflow_definition_gateway( + seeded, definitions, WorkflowService(seeded), drafts=WorkflowDraftRepository(seeded) + ) + ) + retired: list[list[str]] = [] + + def retire(context: RequestContext, agent_ids: list[str]) -> None: + assert context is CONTEXT + with seeded() as session: + assert session.get(Workflow, "obsolete") is None + assert session.scalar(select(func.count()).select_from(WorkflowAgentNodeBinding)) == 0 + retired.append(agent_ids) + + cast(Mock, service._runtime).retire_agents.side_effect = retire + + def fail_commit(_session: Session) -> None: + if rollback: + raise RuntimeError("delete commit failed") + + event.listen(seeded, "before_commit", fail_commit) + try: + if rollback: + with pytest.raises(RuntimeError, match="delete commit failed"): + service.delete(CONTEXT, WorkflowOwner("app-1"), "obsolete") + else: + service.delete(CONTEXT, WorkflowOwner("app-1"), "obsolete") + finally: + event.remove(seeded, "before_commit", fail_commit) + with seeded() as session: + assert (session.get(Workflow, "obsolete") is not None) is rollback + assert session.scalar(select(func.count()).select_from(WorkflowAgentNodeBinding)) == (3 if rollback else 0) + assert retired == ([] if rollback else [["inline-agent"]]) + + +@pytest.mark.parametrize("rollback", [False, True]) +def test_dsl_replacement_and_variable_invalidation_are_atomic(seeded: sessionmaker[Session], rollback: bool) -> None: + variables = [ + WorkflowDraftVariable.new_node_variable( + app_id=app_id, + user_id=user_id, + node_id="node", + name="value", + value=StringSegment(value="old"), + node_execution_id="execution", + ) + for app_id, user_id in [("app-1", "user-1"), ("app-1", "user-2"), ("other-app", "user-1")] + ] + with seeded.begin() as session: + session.add_all(variables) + + def import_draft() -> None: + with seeded.begin() as session: + app = session.get(App, "app-1") + draft = session.get(Workflow, "draft") + account = session.get(Account, "account-1") + assert app is not None + assert draft is not None + assert account is not None + WorkflowDefinitionStore.sync_draft_workflow( + app_model=app, + session=session, + graph={"nodes": [], "edges": [], "imported": True}, + features={}, + prepared=PreparedDraftSync(workflow_snapshot(draft), None), + account_id=account.id, + conversation_variables=[], + clear_debug_variables=True, + ) + if rollback: + raise RuntimeError("import failed") + + if rollback: + with pytest.raises(RuntimeError, match="import failed"): + import_draft() + else: + import_draft() + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + assert draft.graph_dict.get("imported", False) is not rollback + remaining = set(session.scalars(select(WorkflowDraftVariable.id))) + assert remaining == ({variable.id for variable in variables} if rollback else {variables[-1].id}) + + +def test_draft_projection_does_not_mutate_persisted_graph( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + WorkflowAgentPublishService, + "project_draft_bindings_to_graph", + lambda _self, **_kwargs: {"nodes": [{"id": "projected"}]}, + ) + repository = WorkflowDefinitionRepository(session_factory=seeded) + with repository.draft(CONTEXT, "app-1") as draft: + assert draft is not None + projected = WorkflowAgentPublishService(repository=draft.bindings).project_draft_bindings_to_graph( + draft_workflow=draft.workflow + ) + assert projected == {"nodes": [{"id": "projected"}]} + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + assert draft.graph_dict == {"nodes": [], "edges": []} + + +def test_queries_and_updates_keep_complete_owner_chain( + repository: WorkflowDefinitionRepository, seeded: sessionmaker[Session] +) -> None: + with pytest.raises(ConsoleAppNotFoundError): + repository.published(CONTEXT._replace(active_workspace_id="tenant-2"), "app-1") + with seeded.begin() as session: + session.add(make_workflow(workflow_id="foreign", tenant_id="tenant-2", app_id="other-app", version="1")) + assert repository.update(CONTEXT, "app-1", "foreign", {"marked_name": "modified"}) is None + with seeded() as session: + foreign = session.get(Workflow, "foreign") + assert foreign is not None + assert foreign.marked_name == "" + + +def application(lifecycle: WorkflowDefinitionGateway) -> ConsoleWorkflowService[WorkflowAgentBindingStore]: + return ConsoleWorkflowService( + conversion=Mock(spec=WorkflowConversion), + agent_services=WorkflowAgentPublishService, + definitions=cast(WorkflowDefinitionRepository, lifecycle._definitions), + drafts=WorkflowDraftService( + agent_services=WorkflowAgentPublishService, + definitions=cast(WorkflowDraftRepository, lifecycle._drafts), + lifecycle=lifecycle, + retirement=Mock(spec=DraftAgentRetirement), + ), + lifecycle=lifecycle, + runtime=Mock(spec=WorkflowRuntime), + apps=Mock(spec=WorkflowAppLookup), + presence=Mock(spec=WorkflowPresence), + access=Mock(spec=WorkflowAccess), + ) + + +def publish(lifecycle: WorkflowDefinitionGateway, snapshot: WorkflowSnapshot) -> WorkflowPublication: + # Supply the previously validated snapshot to exercise a concurrent edit or + # failed commit through the same orchestration as the console and migration. + with patch.object(lifecycle, "validate_publish", return_value=ValidatedWorkflowPublication(snapshot, ())): + result, _ = application(lifecycle).publish(CONTEXT, "app-1", marked_name="Release", marked_comment="") + return result + + +@pytest.mark.parametrize("fail_commit", [False, True]) +def test_publication_and_app_pointer_commit_atomically(seeded: sessionmaker[Session], fail_commit: bool) -> None: + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + snapshot = workflow_snapshot(draft) + + def fail(_session: object) -> None: + raise RuntimeError("commit failed") + + if fail_commit: + event.listen(seeded, "before_commit", fail) + try: + if fail_commit: + with pytest.raises(RuntimeError, match="commit failed"): + publish(lifecycle, snapshot) + else: + assert publish(lifecycle, snapshot).graph + finally: + if fail_commit: + event.remove(seeded, "before_commit", fail) + with seeded() as session: + app = session.get(App, "app-1") + version = session.scalar(select(Workflow).where(Workflow.marked_name == "Release")) + assert app is not None + assert (version is None) is fail_commit + if fail_commit: + assert app.workflow_id == "published" + else: + assert version is not None + assert app.workflow_id == version.id + + +@pytest.mark.parametrize( + ("workflow_id", "error"), [("draft", DraftWorkflowDeletionError), ("published", WorkflowInUseError)] +) +def test_delete_preserves_domain_rejections( + repository: WorkflowDefinitionRepository, workflow_id: str, error: type[Exception] +) -> None: + with pytest.raises(error): + repository.delete(CONTEXT, WorkflowOwner("app-1"), workflow_id) + + +def test_sync_prepares_patch_without_replacing_server_environment(seeded: sessionmaker[Session]) -> None: + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.features = '{"collaborator": true}' + draft.environment_variables = [StringVariable(id="untouched", name="OTHER", value="server")] + draft_hash = draft.unique_hash + command = DraftSyncCommand( + graph={"nodes": [], "edges": []}, + features={}, + unique_hash=draft_hash, + is_collaborative=True, + environment_upserts=[{"id": "id", "name": "KEY", "value_type": "string", "value": "value"}], + environment_deletions=[], + conversation_variables=[], + ) + application( + WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + ).sync(CONTEXT, "app-1", command) + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + assert draft.features_dict["collaborator"] is True + assert {value.name: value.value for value in draft.environment_variables} == {"OTHER": "server", "KEY": "value"} + application( + WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + ).sync(CONTEXT, "app-1", replace(command, environment_upserts=None)) + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + assert {value.name: value.value for value in draft.environment_variables} == {"OTHER": "server", "KEY": "value"} + + +def test_app_maintainers_reuses_tenant_scoped_app_repository(seeded: sessionmaker[Session]) -> None: + with seeded.begin() as session: + session.add_all( + [ + make_app(app_id="other", tenant_id="tenant-2"), + make_app(app_id="local", maintainer="owner"), + ] + ) + repository = ConsoleAppRepository(session_factory=seeded) + assert repository.maintainers(CONTEXT, ["app-1", "other", "local", "missing"]) == {"app-1": None, "local": "owner"} + + +@pytest.mark.parametrize("operation", ["sync", "restore", "publish", "features"]) +@pytest.mark.parametrize("fail_commit", [False, True]) +def test_use_case_notifies_only_after_real_transaction_commits( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, operation: str, fail_commit: bool +) -> None: + repository = WorkflowDefinitionRepository(session_factory=seeded) + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + runtime = Mock(spec=WorkflowRuntime) + service = ConsoleWorkflowService( + conversion=Mock(spec=WorkflowConversion), + agent_services=WorkflowAgentPublishService, + definitions=repository, + drafts=WorkflowDraftService( + agent_services=WorkflowAgentPublishService, + definitions=WorkflowDraftRepository(seeded), + lifecycle=lifecycle, + retirement=Mock(spec=DraftAgentRetirement), + ), + lifecycle=lifecycle, + runtime=runtime, + apps=Mock(spec=WorkflowAppLookup), + presence=Mock(spec=WorkflowPresence), + access=Mock(spec=WorkflowAccess), + ) + if operation in {"sync", "restore"}: + with seeded.begin() as session: + session.add( + WorkflowAgentNodeBinding( + id="removed-binding", + tenant_id="tenant-1", + app_id="app-1", + workflow_id="draft", + workflow_version="draft", + node_id="removed-node", + binding_type=WorkflowAgentBindingType.INLINE_AGENT, + agent_id="retired-agent", + node_job_config=WorkflowNodeJobConfig(), + ) + ) + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + before = workflow_snapshot(draft) + steps: list[str] = [] + sessions: list[Session] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def committed(_session: Session) -> None: + steps.append("commit") + + def fail(_session: Session) -> None: + raise RuntimeError("commit failed") + + def retire(context: RequestContext, agent_ids: list[str]) -> None: + assert steps == ["commit"] + assert all(not session.in_transaction() for session in sessions) + assert context == CONTEXT + assert agent_ids == ["retired-agent"] + with seeded() as session: + assert session.get(WorkflowAgentNodeBinding, "removed-binding") is None + steps.append("retire") + + cast(Mock, service._drafts._retirement).retire_unowned.side_effect = lambda **kwargs: retire( + CONTEXT, kwargs["agent_ids"] + ) + + def notify(_app: App, **kwargs: object) -> None: + assert "commit" in steps + assert all(not session.in_transaction() for session in sessions) + notified = kwargs["synced_draft_workflow"] + assert isinstance(notified, Workflow) + with seeded() as session: + persisted = session.get(Workflow, notified.id) + assert persisted is not None + assert persisted.graph == notified.graph + steps.append("notify") + + runtime.retire_agents.side_effect = retire + monkeypatch.setattr(gateway_module.app_draft_workflow_was_synced, "send", notify) + event.listen(seeded, "after_begin", opened) + event.listen(seeded, "after_commit", committed) + if fail_commit: + event.listen(seeded, "before_commit", fail) + + def execute() -> None: + match operation: + case "sync": + service.sync( + CONTEXT, + "app-1", + DraftSyncCommand( + graph={"nodes": [], "edges": [], "changed": True}, + features={}, + unique_hash=before.hash, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + conversation_variables=[], + ), + ) + case "restore": + service.restore(CONTEXT, "app-1", "published") + case "publish": + service.publish(CONTEXT, "app-1", marked_name="Release", marked_comment="") + case "features": + service.update_features(CONTEXT, "app-1", {"file_upload": {"enabled": False}}) + + try: + if fail_commit: + with pytest.raises(RuntimeError, match="commit failed"): + execute() + assert steps == [] + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + assert workflow_snapshot(draft) == before + if operation in {"sync", "restore"}: + assert session.get(WorkflowAgentNodeBinding, "removed-binding") is not None + else: + execute() + assert steps == (["commit", "retire", "notify"] if operation in {"sync", "restore"} else ["commit"]) + finally: + event.remove(seeded, "after_begin", opened) + event.remove(seeded, "after_commit", committed) + if fail_commit: + event.remove(seeded, "before_commit", fail) + + +@pytest.mark.parametrize("field", ["graph", "features", "environment_variables"]) +def test_publish_rejects_revision_modified_after_validation(seeded: sessionmaker[Session], field: str) -> None: + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + validated = workflow_snapshot(draft) + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + if field == "environment_variables": + draft.environment_variables = [StringVariable(id="changed", name="KEY", value="changed")] + else: + setattr(draft, field, '{"changed": true}') + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + with pytest.raises(WorkflowHashNotEqualError): + publish(lifecycle, validated) + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + assert app.workflow_id == "published" + + +@pytest.mark.parametrize("node_type", ["trigger-webhook", "trigger-plugin"]) +def test_trigger_limit_rejects_publish_before_version_or_pointer_changes( + seeded: sessionmaker[Session], node_type: str +) -> None: + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + { + "id": f"trigger-{index}", + "data": {"type": node_type, "title": "Trigger", "subscription_id": "sub"}, + } + for index in range(6) + ], + "edges": [], + } + ) + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + with pytest.raises(ValueError, match="maximum.*limit"): + application(lifecycle).publish(CONTEXT, "app-1", marked_name="Rejected", marked_comment="") + with seeded() as session: + app = session.get(App, "app-1") + counter = session.get(WorkflowVersionCounter, "app-1") + assert app is not None + assert app.workflow_id == "published" + assert counter is not None + assert counter.last_version_number == 1 + assert session.scalar(select(func.count()).select_from(Workflow)) == 2 + + +def test_publication_versions_increase_without_reusing_deleted_version(seeded: sessionmaker[Session]) -> None: + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + service = application(lifecycle) + for expected in [2, 3, 4]: + service.publish(CONTEXT, "app-1", marked_name="Release", marked_comment="") + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + workflow = session.get(Workflow, app.workflow_id) + assert workflow is not None + assert workflow.version_number == expected + if expected == 3: + with seeded.begin() as session: + app = session.get(App, "app-1") + assert app is not None + app.workflow_id = "published" + service.delete(CONTEXT, WorkflowOwner("app-1"), workflow.id) + + +@pytest.mark.parametrize( + "invalid_configuration", ["missing-draft", "missing-model", "model-mode", "schedule", "cloud-limit"] +) +def test_publication_retains_blocking_domain_validations( + seeded: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + config_overrides: Callable[..., None], + invalid_configuration: str, +) -> None: + expected_error: type[Exception] = ValueError + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + if invalid_configuration == "missing-draft": + session.delete(draft) + elif invalid_configuration in {"missing-model", "model-mode"}: + draft.graph = json.dumps( + { + "nodes": [ + { + "id": "llm", + "data": { + "type": "llm", + "model_selector": ["env", "shared_model"], + "model": { + "provider": "provider", + "name": "model", + "mode": "chat", + "completion_params": {}, + }, + }, + } + ], + "edges": [], + } + ) + if invalid_configuration == "model-mode": + draft.environment_variables = [ + LLMEnvironmentVariable( + name="shared_model", value={"provider": "provider", "name": "model", "mode": "completion"} + ) + ] + elif invalid_configuration == "schedule": + draft.graph = json.dumps( + { + "nodes": [ + { + "id": "schedule", + "data": { + "title": "Schedule", + "type": "trigger-schedule", + "mode": "cron", + "cron_expression": "invalid", + "timezone": "UTC", + }, + } + ], + "edges": [], + } + ) + else: + config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) + monkeypatch.setattr( + workflow_module.BillingService, "get_info", lambda *_args: {"subscription": {"plan": "sandbox"}} + ) + draft.graph = json.dumps( + {"nodes": [{"id": str(index), "data": {"type": "trigger-webhook"}} for index in range(3)], "edges": []} + ) + expected_error = TriggerNodeLimitExceededError + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + with pytest.raises(expected_error): + application(lifecycle).publish(CONTEXT, "app-1", marked_name="Rejected", marked_comment="") + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + assert app.workflow_id == "published" + assert session.scalar(select(Workflow.id).where(Workflow.marked_name == "Rejected")) is None + + +def test_restore_preserves_historical_features_without_normalizing_persisted_source( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch +) -> None: + features = json.dumps( + { + "file_upload": { + "image": {"enabled": True, "number_limits": 6, "transfer_methods": ["remote_url", "local_file"]} + } + } + ) + with seeded.begin() as session: + workflow = session.get(Workflow, "published") + assert workflow is not None + workflow.features = features + monkeypatch.setattr(gateway_module.app_draft_workflow_was_synced, "send", lambda *_args, **_kwargs: None) + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + application(lifecycle).restore(CONTEXT, "app-1", "published") + with seeded() as session: + for workflow_id in ["draft", "published"]: + workflow = session.get(Workflow, workflow_id) + assert workflow is not None + assert workflow.serialized_features == features + + +def test_publication_serializes_on_app_and_closes_the_transaction( + seeded: sessionmaker[Session], +) -> None: + locks: list[Session] = [] + + def capture(execution: ORMExecuteState) -> None: + if not isinstance(execution.statement, Select): + return + sql = str(execution.statement.compile(dialect=postgresql.dialect())) + if "FROM apps" in sql and "FOR UPDATE" in sql: + locks.append(execution.session) + + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + event.listen(seeded, "do_orm_execute", capture) + try: + application(lifecycle).publish(CONTEXT, "app-1", marked_name="Release", marked_comment="") + finally: + event.remove(seeded, "do_orm_execute", capture) + assert len(locks) == 1 + assert all(not session.in_transaction() for session in locks) + + +@pytest.mark.parametrize("over_limit", [False, True]) +def test_migration_uses_the_same_atomic_publication_boundary( + seeded: sessionmaker[Session], over_limit: bool, *, workflow_tools +) -> None: + app_id = str(uuid4()) + with seeded.begin() as session: + session.add(make_app(app_id=app_id, mode=AppMode.WORKFLOW, created_by="account-1")) + session.add( + make_workflow( + app_id=app_id, + graph={ + "nodes": [ + {"id": str(index), "data": {"type": "trigger-webhook", "title": "Webhook"}} + for index in range(6 if over_limit else 1) + ], + "edges": [], + }, + ) + ) + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + migration = MigrationImportService(workflows=application(lifecycle), app_dsl=Mock(), workflow_tools=workflow_tools) + with seeded() as session: + actor = session.get(Account, "account-1") + assert actor is not None + target = ImportTarget("tenant-1", "Test", actor.id, actor.email) + if over_limit: + with pytest.raises(ValueError, match="maximum webhook node limit"): + migration._ensure_workflow_app_is_published(target, actor, app_id, session=session) + else: + migration._ensure_workflow_app_is_published(target, actor, app_id, session=session) + app = session.get(App, app_id) + assert app is not None + if over_limit: + assert app.workflow_id is None + else: + assert app.workflow_id is not None + assert session.get(Workflow, app.workflow_id) is not None + from models.trigger import WorkflowWebhookTrigger + + assert ( + session.scalar(select(WorkflowWebhookTrigger).where(WorkflowWebhookTrigger.app_id == app_id)) + is not None + ) + + +def configure_runtime_draft(factory: sessionmaker[Session], name: str) -> None: + with factory.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + {"id": f"webhook-{name}", "data": {"type": "trigger-webhook", "title": name}}, + { + "id": f"plugin-{name}", + "data": { + "type": "trigger-plugin", + "title": name, + "provider_id": "test/provider", + "event_name": "created", + "subscription_id": f"subscription-{name}", + }, + }, + { + "id": f"schedule-{name}", + "data": { + "type": "trigger-schedule", + "title": name, + "mode": "cron", + "cron_expression": "0 * * * *", + "timezone": "UTC", + }, + }, + ], + "edges": [], + } + ) + + +def assert_runtime_configuration(factory: sessionmaker[Session], name: str) -> str: + from models.trigger import AppTrigger, WorkflowPluginTrigger, WorkflowSchedulePlan, WorkflowWebhookTrigger + + with factory() as session: + app = session.get(App, "app-1") + assert app is not None + version = session.get(Workflow, app.workflow_id) + assert version is not None + assert version.marked_name == name + assert set(session.scalars(select(AppTrigger.node_id).where(AppTrigger.app_id == app.id))) == { + f"webhook-{name}", + f"plugin-{name}", + f"schedule-{name}", + } + webhook = session.scalars(select(WorkflowWebhookTrigger).where(WorkflowWebhookTrigger.app_id == app.id)).one() + assert webhook.node_id == f"webhook-{name}" + assert len(webhook.webhook_id) == 24 + plugin = session.scalars(select(WorkflowPluginTrigger).where(WorkflowPluginTrigger.app_id == app.id)).one() + assert plugin.node_id == f"plugin-{name}" + assert plugin.subscription_id == f"subscription-{name}" + schedule = session.scalars(select(WorkflowSchedulePlan).where(WorkflowSchedulePlan.app_id == app.id)).one() + assert schedule.node_id == f"schedule-{name}" + assert schedule.next_run_at is not None + assert schedule.cron_expression == "0 * * * *" + return version.id + + +def test_redis_outage_cannot_leave_a_successful_publication_without_runtime_configuration( + seeded: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + from redis.exceptions import ConnectionError as RedisConnectionError + + from extensions.ext_redis import redis_client + + configure_runtime_draft(seeded, "A") + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + + def unavailable() -> None: + raise RedisConnectionError("Redis temporarily unavailable") + + monkeypatch.setattr(redis_client, "_require_client", unavailable) + application(lifecycle).publish(CONTEXT, "app-1", marked_name="A", marked_comment="") + assert_runtime_configuration(seeded, "A") + + +def test_trigger_storage_failure_rolls_back_version_pointer_and_every_projection( + seeded: sessionmaker[Session], +) -> None: + from models.trigger import WorkflowSchedulePlan + + configure_runtime_draft(seeded, "A") + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + service = application(lifecycle) + service.publish(CONTEXT, "app-1", marked_name="A", marked_comment="") + original_version = assert_runtime_configuration(seeded, "A") + configure_runtime_draft(seeded, "B") + + def reject_schedule_update(session: Session, _context: object, _instances: object) -> None: + # This fails after webhook/plugin writes have flushed, exercising real rollback. + for value in session.dirty: + if isinstance(value, WorkflowSchedulePlan) and value.node_id == "schedule-B": + raise RuntimeError("schedule storage unavailable") + + event.listen(seeded, "before_flush", reject_schedule_update) + try: + with pytest.raises(RuntimeError, match="schedule storage unavailable"): + service.publish(CONTEXT, "app-1", marked_name="B", marked_comment="") + finally: + event.remove(seeded, "before_flush", reject_schedule_update) + assert assert_runtime_configuration(seeded, "A") == original_version + with seeded() as session: + assert session.scalar(select(Workflow.id).where(Workflow.marked_name == "B")) is None + counter = session.get(WorkflowVersionCounter, "app-1") + assert counter is not None + assert counter.last_version_number == 2 + + +def test_republication_replaces_all_trigger_configuration( + seeded: sessionmaker[Session], +) -> None: + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + for name in ["A", "B"]: + configure_runtime_draft(seeded, name) + application(lifecycle).publish(CONTEXT, "app-1", marked_name=name, marked_comment="") + assert_runtime_configuration(seeded, name) + + +def test_draft_webhook_receiver_uses_injected_database( + seeded: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + from events.event_handlers.sync_webhook_when_app_created import handle + from models.trigger import WorkflowWebhookTrigger + from services.trigger import webhook_service + + configure_runtime_draft(seeded, "draft") + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + snapshot = workflow_snapshot(draft) + monkeypatch.setattr(webhook_service, "db", object()) + monkeypatch.setattr(gateway_module.app_draft_workflow_was_synced, "receivers_for", lambda _app: iter([handle])) + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + lifecycle.draft_synced(CONTEXT, WorkflowOwner("app-1"), snapshot) + with seeded() as session: + assert session.scalars(select(WorkflowWebhookTrigger.node_id)).all() == ["webhook-draft"] + + +@pytest.mark.parametrize("retained", [set(), {"kept"}]) +def test_dataset_sync_removes_every_duplicate_stale_relationship( + seeded: sessionmaker[Session], retained: set[str] +) -> None: + with seeded.begin() as session: + session.add_all( + [AppDatasetJoin(app_id="app-1", dataset_id=value) for value in ["removed", "removed", "kept", "kept"]] + ) + session.add(AppDatasetJoin(app_id="other-app", dataset_id="removed")) + with seeded.begin() as session: + app = session.get(App, "app-1") + assert app is not None + WorkflowDefinitionStore.sync_dataset_relationships(session, app, retained) + with seeded() as session: + assert ( + set(session.scalars(select(AppDatasetJoin.dataset_id).where(AppDatasetJoin.app_id == "app-1"))) == retained + ) + assert ( + session.scalars(select(AppDatasetJoin).where(AppDatasetJoin.app_id == "other-app")).one().dataset_id + == "removed" + ) + + +@pytest.mark.parametrize("field", ["environment", "conversation"]) +def test_variable_updates_lock_the_owned_draft_and_close_their_transaction( + seeded: sessionmaker[Session], field: str +) -> None: + repository = WorkflowDraftRepository(seeded) + snapshot = repository.snapshot(CONTEXT, WorkflowOwner("app-1")) + assert snapshot is not None + sessions: list[Session] = [] + locked_statements: list[str] = [] + + def capture(execution: ORMExecuteState) -> None: + if not isinstance(execution.statement, Select): + return + statement = str(execution.statement.compile(dialect=postgresql.dialect())) + if "FOR UPDATE" in statement: + sessions.append(execution.session) + locked_statements.append(statement) + + event.listen(seeded, "do_orm_execute", capture) + variables = [StringVariable(id="changed", name="changed", value="new")] + try: + repository.update_draft_variables( + CONTEXT, + "app-1", + expected=snapshot, + environment_variables=prepare_environment_variables( + tenant_id="tenant-1", source=snapshot, variables=variables + ).value + if field != "conversation" + else None, + conversation_variables=variables if field == "conversation" else None, + ) + finally: + event.remove(seeded, "do_orm_execute", capture) + assert len(locked_statements) == 2 + assert "FROM apps" in locked_statements[0] + assert all( + column in locked_statements[1] for column in ["workflows.app_id", "workflows.tenant_id", "workflows.version"] + ) + assert all(not session.in_transaction() for session in sessions) + with seeded() as session: + workflow = session.get(Workflow, "draft") + assert workflow is not None + actual = workflow.conversation_variables if field == "conversation" else workflow.environment_variables + assert [(v.id, v.value) for v in actual] == [("changed", "new")] + assert workflow.updated_by == "account-1" + assert workflow.updated_at is not None + + +def test_variable_update_does_not_commit_an_ambient_session(seeded: sessionmaker[Session]) -> None: + repository = WorkflowDraftRepository(seeded) + with seeded() as caller: + caller.add(make_account(account_id="uncommitted", email="uncommitted@example.com")) + repository.update_draft_variables( + CONTEXT, + "app-1", + expected=repository.snapshot(CONTEXT, WorkflowOwner("app-1")), + environment_variables="{}", + conversation_variables=None, + ) + with seeded() as reader: + assert reader.get(Account, "uncommitted") is None + assert caller.new + caller.rollback() + + +def test_variable_update_commit_failure_rolls_back_all_fields(seeded: sessionmaker[Session]) -> None: + repository = WorkflowDraftRepository(seeded) + with seeded() as session: + before = session.get(Workflow, "draft") + assert before is not None + snapshot = workflow_snapshot(before) + + def fail_commit(_session: Session) -> None: + raise RuntimeError("commit failed") + + event.listen(seeded, "before_commit", fail_commit) + try: + with pytest.raises(RuntimeError, match="commit failed"): + repository.update_draft_variables( + CONTEXT, + "app-1", + expected=snapshot, + environment_variables=prepare_environment_variables( + tenant_id="tenant-1", + source=snapshot, + variables=[StringVariable(id="new-env", name="key", value="value")], + ).value, + conversation_variables=[StringVariable(id="new-conv", name="topic", value="value")], + ) + finally: + event.remove(seeded, "before_commit", fail_commit) + with seeded() as session: + after = session.get(Workflow, "draft") + assert after is not None + assert workflow_snapshot(after) == snapshot + + +def test_variable_update_rejects_another_tenant(seeded: sessionmaker[Session]) -> None: + repository = WorkflowDraftRepository(seeded) + snapshot = repository.snapshot(CONTEXT, WorkflowOwner("app-1")) + with pytest.raises(ConsoleAppNotFoundError): + repository.update_draft_variables( + CONTEXT._replace(active_workspace_id="other-tenant"), + "app-1", + expected=snapshot, + environment_variables="{}", + conversation_variables=None, + ) + + +def test_republish_without_schedule_removes_the_runtime_plan(seeded: sessionmaker[Session]) -> None: + from models.trigger import WorkflowSchedulePlan + + configure_runtime_draft(seeded, "scheduled") + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=Mock(spec=AgentSkillReader), + ) + service = application(lifecycle) + service.publish(CONTEXT, "app-1", marked_name="Scheduled", marked_comment="") + with seeded.begin() as session: + assert session.scalar(select(WorkflowSchedulePlan).where(WorkflowSchedulePlan.app_id == "app-1")) is not None + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = json.dumps({"nodes": [], "edges": []}) + service.publish(CONTEXT, "app-1", marked_name="Unscheduled", marked_comment="") + with seeded() as session: + assert session.scalar(select(WorkflowSchedulePlan).where(WorkflowSchedulePlan.app_id == "app-1")) is None + + +@pytest.mark.parametrize("rollback", [False, True]) +def test_agent_bindings_and_active_version_share_the_publication_transaction( + seeded: sessionmaker[Session], rollback: bool +) -> None: + from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource + from models.agent_config_entities import AgentSoulConfig, AgentSoulModelConfig + from services.workflow.publication_service import prepare_publication, publish_workflow + + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + { + "id": "agent-node", + "data": { + "type": "agent", + "version": "2", + "agent_node_kind": "dify_agent", + }, + } + ], + "edges": [], + } + ) + session.add_all( + [ + Agent( + id="roster", + tenant_id="tenant-1", + name="Roster", + scope=AgentScope.ROSTER, + source=AgentSource.ROSTER, + active_config_snapshot_id="agent-config", + ), + AgentConfigSnapshot( + id="agent-config", + tenant_id="tenant-1", + agent_id="roster", + version=1, + config_snapshot=AgentSoulConfig( + model=AgentSoulModelConfig( + plugin_id="langgenius/openai", + model_provider="openai", + model="test-model", + ) + ), + ), + WorkflowAgentNodeBinding( + tenant_id="tenant-1", + app_id="app-1", + workflow_id="draft", + workflow_version="draft", + node_id="agent-node", + binding_type=WorkflowAgentBindingType.ROSTER_AGENT, + agent_id="roster", + current_snapshot_id="agent-config", + node_job_config={}, + created_by="account-1", + ), + ] + ) + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + snapshot = workflow_snapshot(draft) + repository = WorkflowDefinitionRepository(session_factory=seeded) + published_id = "" + + with repository.binding_reader() as bindings: + agents = WorkflowAgentPublishService(repository=bindings).publication_state(draft_workflow=snapshot) + + def publish() -> None: + nonlocal published_id + with repository.publication(CONTEXT, "app-1", snapshot) as transaction: + published = publish_workflow( + transaction, + prepare_publication(snapshot), + agent_service=WorkflowAgentPublishService(repository=transaction.bindings), + agents=agents, + marked_name="Agents", + marked_comment="", + ) + published_id = published.id + if rollback: + raise RuntimeError("rollback after binding copy and activation") + + if rollback: + with pytest.raises(RuntimeError, match="rollback after binding copy"): + publish() + else: + publish() + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + assert app.workflow_id == ("published" if rollback else published_id) + assert (session.get(Workflow, published_id) is None) is rollback + bindings = list( + session.scalars( + select(WorkflowAgentNodeBinding).where( + WorkflowAgentNodeBinding.workflow_id == published_id, + ) + ) + ) + assert len(bindings) == (0 if rollback else 1) + if bindings: + assert bindings[0].current_snapshot_id == "agent-config" + assert ( + session.scalar( + select(func.count()) + .select_from(WorkflowAgentNodeBinding) + .where( + WorkflowAgentNodeBinding.workflow_id == "draft", + ) + ) + == 1 + ) + + +@pytest.mark.parametrize("concurrent_edit", [None, "graph", "binding", "agent-snapshot"]) +def test_skill_archive_validation_releases_connections_and_rechecks_agent_state( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, concurrent_edit: str | None +) -> None: + import io + import zipfile + + from sqlalchemy.engine import Engine + from sqlalchemy.pool import QueuePool + + from extensions.application_services.workflow import build_workflow_definition_gateway + from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource + from models.agent_config_entities import AgentSoulConfig, AgentSoulModelConfig, AgentSoulPromptConfig + from models.skill import AgentSkillBindingSnapshot, Skill, SkillVersion, SkillVersionManifest + from models.tools import ToolFile + + soul = AgentSoulConfig( + model=AgentSoulModelConfig(plugin_id="langgenius/openai", model_provider="openai", model="test"), + prompt=AgentSoulPromptConfig(system_prompt="Use [§skill:archived-skill§]"), + ) + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [{"id": "agent", "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}], + "edges": [], + } + ) + archive = ToolFile( + tenant_id="tenant-1", + user_id="account-1", + conversation_id=None, + file_key="skills/published.zip", + mimetype="application/zip", + ) + session.add_all( + [ + archive, + Agent( + id="agent", + tenant_id="tenant-1", + name="Agent", + scope=AgentScope.ROSTER, + source=AgentSource.ROSTER, + active_config_snapshot_id="soul", + ), + AgentConfigSnapshot(id="soul", tenant_id="tenant-1", agent_id="agent", version=1, config_snapshot=soul), + AgentConfigSnapshot( + id="new-soul", tenant_id="tenant-1", agent_id="agent", version=2, config_snapshot=soul + ), + WorkflowAgentNodeBinding( + id="binding", + tenant_id="tenant-1", + app_id="app-1", + workflow_id="draft", + workflow_version="draft", + node_id="agent", + binding_type=WorkflowAgentBindingType.ROSTER_AGENT, + agent_id="agent", + current_snapshot_id="soul", + node_job_config={}, + created_by="account-1", + ), + Skill( + id="skill", + tenant_id="tenant-1", + name="draft-name", + display_name="Skill", + latest_published_version_id="version", + ), + SkillVersion( + id="version", + skill_id="skill", + version_number=1, + manifest=SkillVersionManifest(files=[]), + archive_tool_file_id=archive.id, + hash_code="hash", + archive_size=123, + ), + AgentSkillBindingSnapshot( + tenant_id="tenant-1", agent_id="agent", config_snapshot_id="soul", skill_id="skill", priority=0 + ), + ] + ) + archive_bytes = io.BytesIO() + with zipfile.ZipFile(archive_bytes, "w") as zip_file: + zip_file.writestr("SKILL.md", "---\nname: archived-skill\ndescription: Published skill\n---\nInstructions") + sessions: list[Session] = [] + statements: list[str] = [] + downloads: list[str] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def query(execution: ORMExecuteState) -> None: + if isinstance(execution.statement, Select): + statements.append(str(execution.statement.compile(dialect=postgresql.dialect()))) + + def load(key: str) -> bytes: + assert all(not session.in_transaction() for session in sessions) + engine = seeded.kw["bind"] + assert isinstance(engine, Engine) + assert isinstance(engine.pool, QueuePool) + assert engine.pool.checkedout() == 0 + assert not any("FOR UPDATE" in statement for statement in statements) + downloads.append(key) + if concurrent_edit: + with seeded.begin() as session: + if concurrent_edit == "graph": + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = '{"nodes": [], "edges": []}' + elif concurrent_edit == "binding": + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + else: + agent = session.get(Agent, "agent") + assert agent is not None + agent.active_config_snapshot_id = "new-soul" + return archive_bytes.getvalue() + + monkeypatch.setattr("services.skill_management_service.storage.load_once", load) + console_variables = build_console_workflow_variables( + database_client=seeded, variables=build_workflow_variable_service(database_client=seeded) + ) + with seeded.begin() as session: + session.get(App, "app-1").mode = AppMode.ADVANCED_CHAT + workflows = WorkflowService(seeded) + monkeypatch.setattr(workflows, "validate_graph_structure", lambda **_kwargs: None) + definitions = WorkflowDefinitionRepository(session_factory=seeded) + service = application( + build_workflow_definition_gateway(seeded, definitions, workflows, drafts=WorkflowDraftRepository(seeded)) + ) + event.listen(seeded, "after_begin", opened) + event.listen(seeded, "do_orm_execute", query) + try: + if concurrent_edit: + with pytest.raises(WorkflowHashNotEqualError): + service.publish(CONTEXT, "app-1", marked_name="Skill", marked_comment="") + else: + service.publish(CONTEXT, "app-1", marked_name="Skill", marked_comment="") + finally: + event.remove(seeded, "after_begin", opened) + event.remove(seeded, "do_orm_execute", query) + assert downloads == ["skills/published.zip"] + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + assert (app.workflow_id == "published") is bool(concurrent_edit) + versions = session.scalar(select(func.count()).select_from(Workflow).where(Workflow.version != "draft")) + assert versions == (1 if concurrent_edit else 2) + + +@pytest.mark.parametrize("rollback", [False, True]) +@pytest.mark.parametrize("packaged", [False, True]) +def test_dsl_draft_and_app_share_outer_commit_and_effects( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, rollback: bool, packaged: bool +) -> None: + from extensions.application_services.workflow import build_app_dsl_service + from services.agent.dsl_service import AgentDslService + from services.agent.retirement_service import WorkflowAgentRetirementService + from services.entities.dsl_entities import DslImportWarning + + effects: list[object] = [] + locks: list[str] = [] + + def query(execution: ORMExecuteState) -> None: + statement = execution.statement + if ( + isinstance(statement, Select) + and Workflow.__table__ in statement.get_final_froms() + and "FOR UPDATE" in str(statement.compile(dialect=postgresql.dialect())) + ): + locks.append(str(statement.compile(dialect=postgresql.dialect()))) + + def retire(**kwargs: object) -> None: + assert not outer.in_transaction() + effects.append(kwargs["agent_ids"]) + + def notify(_app: App, **kwargs: object) -> None: + assert not outer.in_transaction() + workflow = kwargs["synced_draft_workflow"] + assert isinstance(workflow, Workflow) + effects.append(workflow.graph_dict) + + def materialize( + _service: AgentDslService, **kwargs: object + ) -> tuple[dict[str, object], list[DslImportWarning], set[str]]: + assert outer.in_transaction() + workflow = kwargs["workflow"] + assert isinstance(workflow, WorkflowSnapshot) + assert workflow.app_id == "app-1" + # A dependent import write must be part of the same transaction. + outer.add(make_workflow(workflow_id="import-dependent", app_id="other-app")) + return {"nodes": [], "edges": [], "materialized": True}, [], {"removed-agent"} + + monkeypatch.setattr(WorkflowAgentRetirementService, "retire_unowned", staticmethod(retire)) + monkeypatch.setattr(AgentDslService, "import_workflow_packages", materialize) + event.listen(seeded, "do_orm_execute", query) + gateway_module.app_draft_workflow_was_synced.connect(notify, weak=False) + try: + with seeded() as outer: + app, account = outer.get(App, "app-1"), outer.get(Account, "account-1") + assert app is not None + assert account is not None + build_app_dsl_service(outer)._create_or_update_app( + app=app, + account=account, + data={ + "app": {"mode": "workflow", "name": "Imported"}, + "workflow": {"graph": {"nodes": [], "edges": [], "imported": True}}, + "agent_packages": {"package": {}} if packaged else {}, + }, + ) + assert effects == [] + assert len(locks) == 1 + if rollback: + outer.rollback() + # Reusing the session must not resurrect rolled-back effects. + outer.commit() + else: + outer.commit() + assert effects == ( + [] + if rollback + else [ + ["removed-agent"] if packaged else [], + {"nodes": [], "edges": [], "materialized" if packaged else "imported": True}, + ] + ) + finally: + event.remove(seeded, "do_orm_execute", query) + gateway_module.app_draft_workflow_was_synced.disconnect(notify) + with seeded() as session: + app, draft = session.get(App, "app-1"), session.get(Workflow, "draft") + assert app is not None + assert draft is not None + assert (app.name == "Imported") is not rollback + assert bool(draft.graph_dict.get("materialized" if packaged else "imported")) is not rollback + assert (session.get(Workflow, "import-dependent") is not None) is (packaged and not rollback) + + +@pytest.mark.parametrize("state", ["new", "replace", "stale", "collaborative"]) +def test_shared_draft_writer_preserves_hash_and_variable_rules(seeded: sessionmaker[Session], state: str) -> None: + from extensions.application_services.workflow import build_workflow_definition_gateway + + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.environment_variables = [StringVariable(id="server", name="SERVER", value="server")] + draft.features = '{"server": true}' + previous_hash = draft.unique_hash + if state == "new": + session.delete(draft) + definitions = WorkflowDefinitionRepository(session_factory=seeded) + lifecycle = build_workflow_definition_gateway( + seeded, definitions, WorkflowService(seeded), drafts=WorkflowDraftRepository(seeded) + ) + service = application(lifecycle) + command = DraftSyncCommand( + graph={"nodes": [], "edges": [], "edited": True}, + features={"client": True}, + unique_hash="stale" if state == "stale" else previous_hash, + is_collaborative=state == "collaborative", + environment_upserts=None, + environment_deletions=[], + conversation_variables=[], + environment_variables=[{"id": "client", "name": "CLIENT", "value_type": "string", "value": "client"}], + ) + if state == "stale": + with pytest.raises(WorkflowHashNotEqualError): + service.sync(CONTEXT, "app-1", command) + else: + result = service.sync(CONTEXT, "app-1", command) + assert result.hash != previous_hash + with seeded() as session: + draft = session.scalar(select(Workflow).where(Workflow.app_id == "app-1", Workflow.version == "draft")) + assert draft is not None + preserve = state in {"stale", "collaborative"} + assert draft.features_dict.get("server", False) is preserve + assert draft.features_dict.get("client", False) is not preserve + assert {variable.name: variable.value for variable in draft.environment_variables} == ( + {"SERVER": "server"} if preserve else {"CLIENT": "client"} + ) + assert bool(draft.graph_dict.get("edited")) is (state != "stale") + + +@pytest.mark.parametrize("snippet", [False, True]) +def test_publication_repository_is_independent_of_agent_service_and_commits_requested_operations( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, snippet: bool +) -> None: + from models.snippet import CustomizedSnippet, SnippetType + from models.workflow import WorkflowKind + from repositories.agent.workflow_binding_repository import WorkflowAgentBindingRepository + from repositories.workflow.snippet_publication_repository import SnippetPublicationRepository + + # A repository must not instantiate or invoke Agent business orchestration, + # even if it used to receive that service hidden behind a factory/Protocol. + monkeypatch.setattr( + WorkflowAgentPublishService, "__init__", Mock(side_effect=AssertionError("repository called service")) + ) + if snippet: + with seeded.begin() as session: + session.add( + CustomizedSnippet( + id="app-1", tenant_id="tenant-1", name="Snippet", type=SnippetType.NODE, created_by="account-1" + ) + ) + draft = session.get(Workflow, "draft") + assert draft is not None + draft.kind = WorkflowKind.SNIPPET + snippets = SnippetPublicationRepository(seeded) + with snippets.draft(tenant_id="tenant-1", snippet_id="app-1") as (source, bindings): + assert isinstance(bindings, WorkflowAgentBindingRepository) + with snippets.publication(source) as transaction: + workflow = transaction.create_version(account_id="account-1") + transaction.activate(workflow, account_id="account-1") + with seeded() as session: + row = session.get(CustomizedSnippet, "app-1") + assert row is not None + assert row.is_published is True + assert session.get(Workflow, workflow.id) is not None + else: + definitions = WorkflowDefinitionRepository(session_factory=seeded) + with definitions.draft(CONTEXT, "app-1") as read: + assert read is not None + assert isinstance(read.bindings, WorkflowAgentBindingRepository) + source = read.workflow + with definitions.publication(CONTEXT, "app-1", source) as publication: + created = publication.create_version(marked_name="Published", marked_comment="") + publication.activate(created) + assert isinstance(publication.bindings, WorkflowAgentBindingRepository) + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + assert app.workflow_id == created.id + assert session.get(Workflow, created.id) is not None diff --git a/api/tests/unit_tests/repositories/workflow/test_draft_repository.py b/api/tests/unit_tests/repositories/workflow/test_draft_repository.py new file mode 100644 index 00000000000000..4776a41255b242 --- /dev/null +++ b/api/tests/unit_tests/repositories/workflow/test_draft_repository.py @@ -0,0 +1,446 @@ +"""The shared draft use case commits each owner's state in one short transaction.""" + +import json +from collections.abc import Iterable +from dataclasses import replace +from typing import Literal +from uuid import uuid4 + +import pytest +from sqlalchemy import Select, event, select +from sqlalchemy.dialects import postgresql +from sqlalchemy.orm import ORMExecuteState, Session, sessionmaker + +from extensions.application_services.workflow import build_workflow_drafts +from extensions.ext_application_services import ApplicationServices +from graphon.variables import StringVariable +from machinery.context import RequestContext +from models.account import TenantAccountJoin, TenantAccountRole +from models.dataset import Pipeline +from models.snippet import CustomizedSnippet +from models.workflow import Workflow, WorkflowKind, WorkflowType +from repositories.workflow.definition_repository import workflow_snapshot +from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError +from services.workflow.contracts import DraftSyncCommand, WorkflowOwner +from tests.unit_tests.model_factories import make_account, make_tenant, make_workflow + +CONTEXT = RequestContext("draft", None, "account-1", "tenant-1") + + +@pytest.fixture(params=["snippet", "pipeline"]) +def owner(request: pytest.FixtureRequest, sqlite_session_factory: sessionmaker[Session]) -> WorkflowOwner: + kind: Literal["snippet", "pipeline"] = request.param + with sqlite_session_factory.begin() as session: + session.add_all( + [ + make_account(account_id=CONTEXT.account_id), + make_tenant(tenant_id=CONTEXT.active_workspace_id), + TenantAccountJoin( + tenant_id=CONTEXT.active_workspace_id, account_id=CONTEXT.account_id, role=TenantAccountRole.EDITOR + ), + ] + ) + if kind == "snippet": + session.add( + CustomizedSnippet( + id="owner", + tenant_id=CONTEXT.active_workspace_id, + name="Snippet", + type="node", + created_by=CONTEXT.account_id, + ) + ) + else: + pipeline = Pipeline(tenant_id=CONTEXT.active_workspace_id, name="Pipeline", created_by=CONTEXT.account_id) + pipeline.id = "owner" + session.add(pipeline) + return WorkflowOwner("owner", kind) + + +def command() -> DraftSyncCommand: + return DraftSyncCommand( + graph={"nodes": [{"id": "llm", "data": {"type": "llm"}}], "edges": []}, + features={}, + unique_hash=None, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=[], + conversation_variables=[], + input_fields=[{"variable": "query"}], + rag_pipeline_variables=[ + {"variable": "source", "belong_to_node_id": "llm", "type": "text-input", "label": "Source"} + ], + ) + + +def test_create_update_and_conflict_are_shared( + owner: WorkflowOwner, sqlite_session_factory: sessionmaker[Session] +) -> None: + service = build_workflow_drafts(sqlite_session_factory) + draft = service.sync(CONTEXT, owner, command()) + assert draft.kind == ("snippet" if owner.kind == "snippet" else "standard") + assert draft.type == ("workflow" if owner.kind == "snippet" else "rag-pipeline") + with sqlite_session_factory() as session: + if owner.kind == "snippet": + snippet = session.get(CustomizedSnippet, owner.id) + assert snippet is not None + assert snippet.input_fields_list == [{"variable": "query"}] + else: + pipeline = session.get(Pipeline, owner.id) + assert pipeline is not None + assert pipeline.workflow_id == draft.id + assert next(iter(json.loads(draft.rag_pipeline_variables).values()))["variable"] == "source" + updated = service.sync(CONTEXT, owner, replace(command(), unique_hash=draft.hash, graph={"nodes": [], "edges": []})) + assert updated.id == draft.id + assert updated.hash != draft.hash + assert updated.updated_by == CONTEXT.account_id + with pytest.raises(WorkflowHashNotEqualError): + service.sync(CONTEXT, owner, replace(command(), unique_hash=draft.hash, input_fields=[])) + with sqlite_session_factory() as session: + row = session.get(Workflow, draft.id) + assert row is not None + assert row.graph_dict == updated.graph_dict + if owner.kind == "snippet": + snippet = session.get(CustomizedSnippet, owner.id) + assert snippet is not None + assert snippet.input_fields_list == [{"variable": "query"}] + + +@pytest.mark.parametrize("existing", [False, True]) +def test_owner_and_draft_roll_back_together( + owner: WorkflowOwner, sqlite_session_factory: sessionmaker[Session], existing: bool +) -> None: + service = build_workflow_drafts(sqlite_session_factory) + before = service.sync(CONTEXT, owner, command()) if existing else None + + def fail(_session: Session) -> None: + raise RuntimeError("commit failed") + + event.listen(sqlite_session_factory, "before_commit", fail) + try: + with pytest.raises(RuntimeError, match="commit failed"): + service.sync( + CONTEXT, + owner, + replace(command(), unique_hash=before.hash if before else None, graph={"nodes": []}, input_fields=[]), + ) + finally: + event.remove(sqlite_session_factory, "before_commit", fail) + with sqlite_session_factory() as session: + draft = session.scalar(select(Workflow).where(Workflow.app_id == owner.id)) + assert (workflow_snapshot(draft) if draft else None) == before + if owner.kind == "snippet": + snippet = session.get(CustomizedSnippet, owner.id) + assert snippet is not None + assert snippet.input_fields_list == ([{"variable": "query"}] if existing else []) + else: + pipeline = session.get(Pipeline, owner.id) + assert pipeline is not None + assert pipeline.workflow_id == (before.id if before else None) + + +@pytest.mark.parametrize("existing", [False, True]) +def test_restore_copies_stored_values_and_sets_new_draft_pointer( + owner: WorkflowOwner, sqlite_session_factory: sessionmaker[Session], existing: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + service = build_workflow_drafts(sqlite_session_factory) + before = service.sync(CONTEXT, owner, command()) if existing else None + with sqlite_session_factory.begin() as session: + published = make_workflow( + workflow_id="published", + app_id=owner.id, + version="1", + graph={"nodes": [{"id": "restored", "data": {"type": "llm"}}], "edges": []}, + ) + published.kind = WorkflowKind.SNIPPET if owner.kind == "snippet" else WorkflowKind.STANDARD + published.type = WorkflowType.WORKFLOW if owner.kind == "snippet" else WorkflowType.RAG_PIPELINE + published.features = '{"some_preserved_feature": true}' + if owner.kind == "pipeline": + variable_id = str(uuid4()) + published._environment_variables = json.dumps( + {variable_id: {"id": variable_id, "name": "secret", "value_type": "secret", "value": "ciphertext"}} + ) + session.add(published) + + def no_key_io(*_args: object, **_kwargs: object) -> str: + pytest.fail("Restoring stored variables must not contact the key service") + + monkeypatch.setattr("core.helper.encrypter.decrypt_token", no_key_io) + monkeypatch.setattr("core.helper.encrypter.encrypt_token", no_key_io) + locked_sources: list[str] = [] + + def capture(query: ORMExecuteState) -> None: + if isinstance(query.statement, Select): + compiled = query.statement.compile(dialect=postgresql.dialect()) + if "FOR UPDATE" in str(compiled) and "published" in compiled.params.values(): + locked_sources.append(str(compiled)) + + event.listen(sqlite_session_factory, "do_orm_execute", capture) + try: + restored = service.restore(CONTEXT, owner, "published") + finally: + event.remove(sqlite_session_factory, "do_orm_execute", capture) + assert locked_sources, "The current restore use case must lock its published source" + + assert restored.version == "draft" + assert restored.graph == published.graph + assert restored.features == published.features + assert restored.environment_variables == published._environment_variables + assert restored.kind == published.kind + if before: + assert restored.id == before.id + with sqlite_session_factory() as session: + if owner.kind == "pipeline": + pipeline = session.get(Pipeline, owner.id) + assert pipeline is not None + assert pipeline.workflow_id == restored.id + source = session.get(Workflow, "published") + assert source is not None + assert source.graph == published.graph + + +def test_owner_and_version_are_scoped(owner: WorkflowOwner, sqlite_session_factory: sessionmaker[Session]) -> None: + service = build_workflow_drafts(sqlite_session_factory) + with pytest.raises(WorkflowNotFoundError): + service.sync(CONTEXT._replace(active_workspace_id="other"), owner, command()) + with pytest.raises(WorkflowNotFoundError): + service.restore(CONTEXT, owner, "missing") + draft = service.sync(CONTEXT, owner, command()) + with pytest.raises(IsDraftWorkflowError): + service.restore(CONTEXT, owner, draft.id) + with sqlite_session_factory.begin() as session: + session.add(make_workflow(workflow_id="foreign", app_id="other", version="1")) + with pytest.raises(WorkflowNotFoundError): + service.restore(CONTEXT, owner, "foreign") + + +def test_snippet_clears_variables_and_rejects_forbidden_graph(sqlite_session_factory: sessionmaker[Session]) -> None: + with sqlite_session_factory.begin() as session: + session.add( + CustomizedSnippet( + id="snippet", + tenant_id=CONTEXT.active_workspace_id, + name="Snippet", + type="node", + created_by=CONTEXT.account_id, + ) + ) + draft = make_workflow(workflow_id="draft", app_id="snippet") + draft.kind = WorkflowKind.SNIPPET + draft.environment_variables = [StringVariable(name="old", value="env")] + draft.conversation_variables = [StringVariable(name="old", value="conversation")] + session.add(draft) + service = build_workflow_drafts(sqlite_session_factory) + owner = WorkflowOwner("snippet", "snippet") + result = service.sync(CONTEXT, owner, replace(command(), unique_hash=draft.unique_hash)) + assert result.environment_variables == result.conversation_variables == "{}" + with pytest.raises(ValueError, match="start-1:start"): + service.sync( + CONTEXT, + owner, + replace( + command(), unique_hash=result.hash, graph={"nodes": [{"id": "start-1", "data": {"type": "start"}}]} + ), + ) + + +@pytest.mark.parametrize("commit_fails", [False, True]) +def test_version_delete_preserves_active_pointer_and_rolls_back( + owner: WorkflowOwner, sqlite_session_factory: sessionmaker[Session], commit_fails: bool +) -> None: + from repositories.workflow.definition_repository import WorkflowDefinitionRepository + from services.errors.workflow_service import WorkflowInUseError + + with sqlite_session_factory.begin() as session: + model = CustomizedSnippet if owner.kind == "snippet" else Pipeline + row = session.get(model, owner.id) + assert row is not None + row.workflow_id = "active" + for workflow_id in ["active", "old"]: + workflow = make_workflow(workflow_id=workflow_id, app_id=owner.id, version=workflow_id) + workflow.kind = WorkflowKind.SNIPPET if owner.kind == "snippet" else WorkflowKind.STANDARD + session.add(workflow) + repository = WorkflowDefinitionRepository(session_factory=sqlite_session_factory) + with pytest.raises(WorkflowInUseError): + repository.delete(CONTEXT, owner, "active") + + def fail(_session: Session) -> None: + raise RuntimeError("commit failed") + + if commit_fails: + event.listen(sqlite_session_factory, "before_commit", fail) + try: + if commit_fails: + with pytest.raises(RuntimeError, match="commit failed"): + repository.delete(CONTEXT, owner, "old") + else: + repository.delete(CONTEXT, owner, "old") + finally: + if commit_fails: + event.remove(sqlite_session_factory, "before_commit", fail) + with sqlite_session_factory() as session: + assert (session.get(Workflow, "old") is not None) == commit_fails + assert session.get(Workflow, "active") is not None + + +def test_pipeline_encryption_runs_after_read_session_closes( + sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch +) -> None: + pipeline = Pipeline(tenant_id=CONTEXT.active_workspace_id, name="Pipeline", created_by=CONTEXT.account_id) + with sqlite_session_factory.begin() as session: + session.add(pipeline) + sessions: list[Session] = [] + encrypted: list[str] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def encrypt(*, tenant_id: str, token: str) -> str: + assert tenant_id == CONTEXT.active_workspace_id + assert sessions + assert all(not session.in_transaction() for session in sessions) + encrypted.append(token) + return "ciphertext" + + monkeypatch.setattr("core.helper.encrypter.encrypt_token", encrypt) + event.listen(sqlite_session_factory, "after_begin", opened) + try: + result = build_workflow_drafts(sqlite_session_factory).sync( + CONTEXT, + WorkflowOwner(pipeline.id, "pipeline"), + replace( + command(), + environment_variables=[ + {"id": str(uuid4()), "name": "secret", "value_type": "secret", "value": "plaintext"} + ], + ), + ) + finally: + event.remove(sqlite_session_factory, "after_begin", opened) + assert encrypted == ["plaintext"] + assert next(iter(json.loads(result.environment_variables).values()))["value"] == "ciphertext" + + +@pytest.mark.parametrize("rollback", [False, True]) +def test_owner_deletion_cleans_bindings_and_retires_after_commit( + owner: WorkflowOwner, + sqlite_session_factory: sessionmaker[Session], + workflow_application: ApplicationServices, + monkeypatch: pytest.MonkeyPatch, + rollback: bool, +) -> None: + from enums.agent import WorkflowAgentBindingType + from models.agent import Agent, AgentScope, AgentSource, AgentStatus, WorkflowAgentNodeBinding + + with sqlite_session_factory.begin() as session: + workflow = make_workflow(workflow_id="published", app_id=owner.id, version="v1") + workflow.kind = WorkflowKind.SNIPPET if owner.kind == "snippet" else WorkflowKind.STANDARD + session.add_all( + [ + workflow, + Agent( + id="inline", + tenant_id="tenant-1", + name="Inline", + scope=AgentScope.WORKFLOW_ONLY, + source=AgentSource.WORKFLOW, + status=AgentStatus.ACTIVE, + ), + WorkflowAgentNodeBinding( + id="binding", + tenant_id="tenant-1", + app_id=owner.id, + workflow_id=workflow.id, + workflow_version=workflow.version, + node_id="agent", + binding_type=WorkflowAgentBindingType.INLINE_AGENT, + agent_id="inline", + current_snapshot_id="snapshot", + node_job_config={}, + ), + ] + ) + dispatched: list[set[str]] = [] + + def collect(*, purge_agent_ids: Iterable[str], **_kwargs: object) -> None: + with sqlite_session_factory() as session: + assert session.get(Workflow, "published") is None + assert session.get(WorkflowAgentNodeBinding, "binding") is None + agent = session.get(Agent, "inline") + assert agent is not None + assert agent.status == AgentStatus.ARCHIVED + dispatched.append(set(purge_agent_ids)) + + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", collect) + + def reject(_session: Session) -> None: + raise RuntimeError("commit failed") + + if rollback: + event.listen(sqlite_session_factory, "before_commit", reject) + try: + with pytest.raises(RuntimeError, match="commit failed"): + workflow_application.console_workflows.delete(CONTEXT, owner, "published") + finally: + event.remove(sqlite_session_factory, "before_commit", reject) + with sqlite_session_factory() as session: + assert session.get(Workflow, "published") is not None + assert session.get(WorkflowAgentNodeBinding, "binding") is not None + agent = session.get(Agent, "inline") + assert agent is not None + assert agent.status == AgentStatus.ACTIVE + assert dispatched == [] + else: + workflow_application.console_workflows.delete(CONTEXT, owner, "published") + assert dispatched == [{"inline"}] + + +@pytest.mark.parametrize("case", ["active", "draft", "tool", "missing", "foreign", "kind"]) +def test_owner_delete_preserves_version_constraints( + owner: WorkflowOwner, + sqlite_session_factory: sessionmaker[Session], + workflow_application: ApplicationServices, + case: str, +) -> None: + from models.tools import WorkflowToolProvider + from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError + + if case == "kind" and owner.kind != "snippet": + return + with sqlite_session_factory.begin() as session: + workflow = make_workflow(workflow_id="target", app_id=owner.id, version="draft" if case == "draft" else "v1") + workflow.kind = WorkflowKind.SNIPPET if owner.kind == "snippet" and case != "kind" else WorkflowKind.STANDARD + if case == "foreign": + workflow.tenant_id = "other" + session.add(workflow) + if case == "active": + row = session.get(CustomizedSnippet if owner.kind == "snippet" else Pipeline, owner.id) + assert row is not None + row.workflow_id = workflow.id + if case == "tool": + session.add( + WorkflowToolProvider( + tenant_id="tenant-1", + app_id=owner.id, + version="v1", + name="Tool", + label="Tool", + icon="", + description="", + parameter_configuration="[]", + privacy_policy="", + user_id="account-1", + ) + ) + expected = ( + WorkflowInUseError + if case in {"active", "tool"} + else DraftWorkflowDeletionError + if case == "draft" + else WorkflowNotFoundError + ) + with pytest.raises(expected): + workflow_application.console_workflows.delete(CONTEXT, owner, "missing" if case == "missing" else "target") + with sqlite_session_factory() as session: + assert session.get(Workflow, "target") is not None diff --git a/api/tests/unit_tests/services/agent/test_agent_dsl_service.py b/api/tests/unit_tests/services/agent/test_agent_dsl_service.py index 47b52d10af6291..4193e314feb9a9 100644 --- a/api/tests/unit_tests/services/agent/test_agent_dsl_service.py +++ b/api/tests/unit_tests/services/agent/test_agent_dsl_service.py @@ -1,4 +1,4 @@ -import json +from typing import TypedDict from unittest.mock import Mock from uuid import uuid4 @@ -7,6 +7,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session +from enums.agent import WorkflowAgentBindingType from graphon.enums import BuiltinNodeTypes from models.account import Account from models.agent import ( @@ -20,23 +21,23 @@ AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) -from models.agent_config_entities import AgentConfigFileRefConfig, AgentConfigSkillRefConfig, AgentSoulConfig +from models.agent_config_entities import AgentPackageMetadata, AgentSoulConfig from models.dataset import Dataset from models.enums import AppStatus from models.model import App, AppMode, IconType from models.tools import ToolFile from models.workflow import Workflow, WorkflowType +from repositories.agent.creation_repository import WorkflowAgentCreationRepository +from repositories.agent.workflow_binding_repository import workflow_binding_scope from services.agent.dsl_entities import ( AGENT_NODE_JOB_DSL_KEY, AGENT_PACKAGE_REF_KEY, AgentPackage, - AgentPackageMetadata, make_portable_agent_package, ) -from services.agent.dsl_service import AgentDslService, AgentPackageImportResult, is_agent_v2_graph +from services.agent.dsl_service import AgentDslService, AgentPackageImportResult from services.entities.dsl_entities import DslImportWarning from tests.unit_tests.model_factories import make_upload_file @@ -120,8 +121,18 @@ def _workflow(*, workflow_id: str = "workflow-1") -> Workflow: ) -def _agent_node(node_id: str, binding: object | None = None) -> dict: - data = {"type": BuiltinNodeTypes.AGENT, "version": "2", "agent_node_kind": "dify_agent"} +class _AgentNode(TypedDict): + id: str + data: dict[str, object] + + +class _Graph(TypedDict): + nodes: list[_AgentNode] + edges: list[object] + + +def _agent_node(node_id: str, binding: object | None = None) -> _AgentNode: + data: dict[str, object] = {"type": BuiltinNodeTypes.AGENT, "version": "2", "agent_node_kind": "dify_agent"} if binding is not None: data["agent_binding"] = binding return {"id": node_id, "data": data} @@ -260,7 +271,7 @@ def test_agent_package_normalizes_legacy_null_missing_asset_file_ids() -> None: {"name": "guide.md", "file_kind": "tool_file", "file_id": None, "is_missing": False}, ], ) -def test_agent_package_rejects_null_file_id_for_available_assets(asset: dict) -> None: +def test_agent_package_rejects_null_file_id_for_available_assets(asset: dict[str, object]) -> None: package = make_portable_agent_package(_agent(), AgentSoulConfig()).model_dump(mode="json") target = "config_files" if "file_kind" in asset else "config_skills" package["soul"][target] = [asset] @@ -344,7 +355,7 @@ def test_export_agent_app_uses_draft_or_active_snapshot(sqlite_session: Session, def test_export_workflow_packages_deduplicates_shared_agent(sqlite_session: Session) -> None: - graph = {"nodes": [_agent_node("node-1"), _agent_node("node-2")], "edges": []} + graph: _Graph = {"nodes": [_agent_node("node-1"), _agent_node("node-2")], "edges": []} bindings = [ WorkflowAgentNodeBinding( id=f"binding-{node_id}", @@ -387,7 +398,7 @@ def test_export_workflow_packages_rejects_incomplete_binding(sqlite_session: Ses def test_graph_without_package_bindings_removes_portable_fields() -> None: - graph = { + graph: _Graph = { "nodes": [ _agent_node( "portable", @@ -466,7 +477,7 @@ def create_backing_agent(**_kwargs: object) -> Agent: def test_import_workflow_packages_materializes_every_package_binding_as_inline(sqlite_session: Session) -> None: package = make_portable_agent_package(_agent(), AgentSoulConfig()) - graph = { + graph: _Graph = { "nodes": [ _agent_node( "roster-1", @@ -522,7 +533,7 @@ def test_import_workflow_packages_materializes_every_package_binding_as_inline(s workflow = _workflow() result, warnings, retirement_candidates = service.import_workflow_packages( - workflow=workflow, + workflow=workflow_binding_scope(workflow), portable_graph=graph, raw_packages={"agent_1": package.model_dump(mode="json")}, account=_account(), @@ -545,7 +556,6 @@ def test_import_workflow_packages_materializes_every_package_binding_as_inline(s ] assert all(binding["binding_type"] == WorkflowAgentBindingType.INLINE_AGENT.value for binding in bindings) assert AGENT_NODE_JOB_DSL_KEY not in result["nodes"][0]["data"] - assert json.loads(workflow.graph) == result added_bindings = sqlite_session.scalars( select(WorkflowAgentNodeBinding).where(WorkflowAgentNodeBinding.workflow_id == workflow.id) ).all() @@ -564,59 +574,19 @@ def test_import_workflow_packages_materializes_every_package_binding_as_inline(s ], ) def test_import_workflow_packages_rejects_invalid_package_binding( - sqlite_session: Session, binding: dict, error: str + sqlite_session: Session, binding: dict[str, object], error: str ) -> None: package = make_portable_agent_package(_agent(), AgentSoulConfig()) with pytest.raises(ValueError, match=error): AgentDslService(sqlite_session).import_workflow_packages( - workflow=_workflow(), + workflow=workflow_binding_scope(_workflow()), portable_graph={"nodes": [_agent_node("node-1", binding)], "edges": []}, raw_packages={"agent_1": package.model_dump(mode="json")}, account=_account(), ) -def test_clone_inline_binding_copies_soul(unbound_session: Session) -> None: - service = AgentDslService(unbound_session) - target_agent = _agent( - agent_id="target-agent", - scope=AgentScope.WORKFLOW_ONLY, - source=AgentSource.WORKFLOW, - ) - target_snapshot = _snapshot(snapshot_id="target-snapshot", agent_id=target_agent.id) - service._create_workflow_only_agent = Mock(return_value=(target_agent, target_snapshot)) - source_agent = _agent() - source_soul = AgentSoulConfig( - config_note="source", - config_skills=[AgentConfigSkillRefConfig(name="summarizer", file_id="skill-file-1")], - config_files=[AgentConfigFileRefConfig(name="brief.pdf", file_kind="upload_file", file_id="config-file-1")], - ) - source_snapshot = _snapshot(soul=source_soul) - workflow = _workflow() - - result = service.clone_inline_binding_for_node( - workflow=workflow, - node_id="target-node", - source_agent=source_agent, - source_snapshot=source_snapshot, - account_id="account-1", - ) - - assert result == (target_agent, target_snapshot) - create_kwargs = service._create_workflow_only_agent.call_args.kwargs - assert create_kwargs["metadata"].name == source_agent.name - cloned_soul = create_kwargs["soul"] - assert cloned_soul.config_note == "source" - assert [(item.name, item.file_kind, item.file_id) for item in cloned_soul.config_skills] == [ - ("summarizer", "tool_file", "skill-file-1") - ] - assert [(item.name, item.file_kind, item.file_id) for item in cloned_soul.config_files] == [ - ("brief.pdf", "upload_file", "config-file-1") - ] - assert create_kwargs["source"] == AgentSource.WORKFLOW - - def test_extract_package_dependencies_covers_models_features_tools_and_knowledge( monkeypatch: pytest.MonkeyPatch, unbound_session: Session ) -> None: @@ -793,7 +763,7 @@ def test_create_imported_inline_agent_uses_import_provenance(unbound_session: Se workflow = _workflow() result = service._create_imported_inline_agent( - workflow=workflow, + workflow=workflow_binding_scope(workflow), node_id="node-1", account=_account(), package=make_portable_agent_package(_agent(), soul), @@ -811,16 +781,11 @@ def test_create_workflow_only_agent_sets_backing_app_and_snapshot( monkeypatch: pytest.MonkeyPatch, sqlite_session: Session ) -> None: service = AgentDslService(sqlite_session) - roster_service = Mock() - backing_app = _app() - backing_app.id = "backing-app" - roster_service.create_hidden_backing_app_for_workflow_agent.return_value = backing_app - monkeypatch.setattr("services.agent.dsl_service.AgentRosterService", Mock(return_value=roster_service)) - monkeypatch.setattr("services.agent.dsl_service.agent_soul_has_model", Mock(return_value=True)) + monkeypatch.setattr("repositories.agent.creation_repository.agent_soul_has_model", Mock(return_value=True)) workflow = _workflow() agent, snapshot = service._create_workflow_only_agent( - workflow=workflow, + workflow=workflow_binding_scope(workflow), node_id="node-1", account_id="account-1", metadata=AgentPackageMetadata(name="Inline", icon_type=AgentIconType.EMOJI.value), @@ -832,7 +797,7 @@ def test_create_workflow_only_agent_sets_backing_app_and_snapshot( assert sqlite_session.get(Agent, agent.id) is agent assert sqlite_session.get(AgentConfigSnapshot, snapshot.id) is snapshot assert agent.scope == AgentScope.WORKFLOW_ONLY - assert agent.backing_app_id == "backing-app" + assert sqlite_session.get(App, agent.backing_app_id) is not None assert agent.active_config_snapshot_id == snapshot.id assert agent.active_config_has_model is True assert agent.active_config_is_published is True @@ -976,9 +941,8 @@ def test_create_snapshot_increments_version_and_records_revision(sqlite_session: ] ) sqlite_session.commit() - service = AgentDslService(sqlite_session) - snapshot = service._create_snapshot( + snapshot = WorkflowAgentCreationRepository(sqlite_session).create_snapshot( tenant_id="tenant-1", agent=agent, account_id="account-1", @@ -1011,7 +975,7 @@ def test_unique_roster_name_uses_first_available_suffix(sqlite_session: Session) assert result == "Agent import 2" -def test_require_helpers_and_graph_detection(sqlite_session: Session) -> None: +def test_require_helpers(sqlite_session: Session) -> None: agent = _agent() snapshot = _snapshot() sqlite_session.add_all([agent, snapshot]) @@ -1029,5 +993,3 @@ def test_require_helpers_and_graph_detection(sqlite_session: Session) -> None: assert AgentDslService._agent_icon_type(AgentIconType.EMOJI.value) == AgentIconType.EMOJI assert AgentDslService._agent_icon_type(None) is None - assert is_agent_v2_graph({"nodes": [_agent_node("agent")]}) is True - assert is_agent_v2_graph({"nodes": ["invalid", {"data": {"type": "start"}}]}) is False diff --git a/api/tests/unit_tests/services/agent/test_agent_observability_service.py b/api/tests/unit_tests/services/agent/test_agent_observability_service.py index db6a8c19cc0e27..51da8479ac25e7 100644 --- a/api/tests/unit_tests/services/agent/test_agent_observability_service.py +++ b/api/tests/unit_tests/services/agent/test_agent_observability_service.py @@ -7,9 +7,10 @@ from sqlalchemy.orm import Session from core.app.entities.app_invoke_entities import InvokeFrom +from enums.agent import WorkflowAgentBindingType from graphon.enums import WorkflowNodeExecutionStatus from libs.datetime_utils import naive_utc_now -from models.agent import WorkflowAgentBindingType, WorkflowAgentNodeBinding +from models.agent import WorkflowAgentNodeBinding from models.agent_config_entities import WorkflowNodeJobConfig from models.enums import ( ConversationFromSource, diff --git a/api/tests/unit_tests/services/agent/test_agent_services.py b/api/tests/unit_tests/services/agent/test_agent_services.py index 8dcc0f914efaf9..30ddba3034c1ae 100644 --- a/api/tests/unit_tests/services/agent/test_agent_services.py +++ b/api/tests/unit_tests/services/agent/test_agent_services.py @@ -7,9 +7,11 @@ import pytest from sqlalchemy import event, select from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker -from core.workflow.nodes.agent_v2.validators import WorkflowAgentNodeValidationError +from enums.agent import WorkflowAgentBindingType +from extensions.application_services.agent_bindings import build_workflow_agent_service +from extensions.application_services.workflow_variables import build_workflow_variable_service from models.account import Account from models.agent import ( Agent, @@ -27,7 +29,6 @@ AgentStatus, AgentWorkspaceBinding, AgentWorkspaceOwnerType, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import ( @@ -36,14 +37,17 @@ DeclaredOutputConfig, DeclaredOutputType, WorkflowNodeJobConfig, + agent_soul_has_model, ) from models.enums import ConversationFromSource, ConversationStatus from models.model import App, AppMode, AppModelConfig, Conversation, IconType, Message from models.skill import AgentSkillBindingSnapshot, Skill, SkillVersion, SkillVersionManifest from models.tools import ToolFile from models.workflow import Workflow +from repositories.agent.config_repository import AgentConfigRepository +from repositories.agent.workflow_binding_repository import workflow_binding_scope +from repositories.agent_workspace_repository import AgentWorkspaceRepository from services.agent import composer_service, roster_service -from services.agent.agent_soul_state import agent_soul_has_model from services.agent.composer_service import AgentComposerService from services.agent.composer_validator import ComposerConfigValidator from services.agent.errors import ( @@ -56,8 +60,10 @@ InvalidComposerConfigError, ) from services.agent.home_snapshot_service import AgentHomeSnapshotService -from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService from services.agent.roster_service import AgentRosterService +from services.agent.workflow_publish_service import WorkflowAgentPublishService +from services.agent.workflow_resources_gateway import WorkflowAgentSkillReader +from services.agent.workflow_validator import WorkflowAgentNodeValidationError from services.agent.workspace_service import AgentWorkspaceService from services.app_service import AppListParams, AppService from services.entities.agent_entities import ( @@ -66,6 +72,7 @@ ComposerSaveStrategy, ComposerVariant, ) +from services.skill_management_service import SkillManagementService from tests.unit_tests.model_factories import make_account, make_app, make_conversation, make_workflow @@ -264,13 +271,13 @@ def test_load_workflow_composer_serializes_existing_binding(monkeypatch: pytest. monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: _workflow()) monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **kwargs: binding) monkeypatch.setattr( - AgentComposerService, - "_get_agent_if_present", + AgentConfigRepository, + "get_agent", lambda **kwargs: _agent(), ) monkeypatch.setattr( - AgentComposerService, - "_get_version_if_present", + AgentConfigRepository, + "get_snapshot", lambda **kwargs: _snapshot(snapshot_id="version-1"), ) monkeypatch.setattr( @@ -298,10 +305,10 @@ def test_load_workflow_composer_uses_roster_preview_snapshot(monkeypatch: pytest monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: _workflow()) monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **kwargs: binding) - monkeypatch.setattr(AgentComposerService, "_get_agent_if_present", lambda **kwargs: agent) + monkeypatch.setattr(AgentConfigRepository, "get_agent", lambda **kwargs: agent) monkeypatch.setattr( - AgentComposerService, - "_require_version", + AgentConfigRepository, + "require_snapshot", lambda **kwargs: _snapshot(snapshot_id=kwargs["version_id"]), ) monkeypatch.setattr( @@ -345,10 +352,10 @@ def test_load_workflow_composer_uses_inline_preview_snapshot(monkeypatch: pytest monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: _workflow()) monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **kwargs: binding) - monkeypatch.setattr(AgentComposerService, "_get_agent_if_present", lambda **kwargs: agent) + monkeypatch.setattr(AgentConfigRepository, "get_agent", lambda **kwargs: agent) monkeypatch.setattr( - AgentComposerService, - "_require_version", + AgentConfigRepository, + "require_snapshot", lambda **kwargs: _snapshot(snapshot_id=kwargs["version_id"]), ) monkeypatch.setattr( @@ -474,13 +481,13 @@ def test_save_workflow_composer_dispatches_save_strategy(monkeypatch, strategy, monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: _workflow()) monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **kwargs: None) monkeypatch.setattr( - AgentComposerService, - "_get_agent_if_present", + AgentConfigRepository, + "get_agent", lambda **kwargs: _agent(), ) monkeypatch.setattr( - AgentComposerService, - "_get_version_if_present", + AgentConfigRepository, + "get_snapshot", lambda **kwargs: _snapshot(snapshot_id="version-1"), ) @@ -524,7 +531,13 @@ def test_save_workflow_composer_commits_before_retiring_replaced_inline_agent( session = sqlite_session events: list[str] = [] old_binding = WorkflowAgentNodeBinding( + tenant_id="tenant-1", + app_id="app-1", + workflow_id="workflow-1", + workflow_version="draft", + node_id="node-1", agent_id="old-inline-agent", + node_job_config=WorkflowNodeJobConfig(), binding_type=WorkflowAgentBindingType.INLINE_AGENT, ) new_binding = WorkflowAgentNodeBinding( @@ -533,16 +546,17 @@ def test_save_workflow_composer_commits_before_retiring_replaced_inline_agent( current_snapshot_id="version-1", ) monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **_kwargs: _workflow()) - monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **_kwargs: old_binding) + session.add(old_binding) + session.commit() monkeypatch.setattr(AgentComposerService, "_save_as_new_agent", lambda **_kwargs: new_binding) monkeypatch.setattr( - AgentComposerService, - "_get_agent_if_present", + AgentConfigRepository, + "get_agent", lambda **_kwargs: _agent(agent_id="new-agent"), ) monkeypatch.setattr( - AgentComposerService, - "_get_version_if_present", + AgentConfigRepository, + "get_snapshot", lambda **_kwargs: _snapshot(snapshot_id="version-1"), ) monkeypatch.setattr(AgentComposerService, "_serialize_workflow_state", lambda **_kwargs: {"state": "ok"}) @@ -554,7 +568,7 @@ def retire_unowned(**kwargs): assert kwargs["agent_ids"] == {"old-inline-agent"} events.append("retire") - monkeypatch.setattr(composer_service.WorkflowAgentRetirementService, "retire_unowned", retire_unowned) + monkeypatch.setattr(composer_service.WorkflowAgentRetirementService, "retire_unowned", staticmethod(retire_unowned)) payload = ComposerSavePayload.model_validate( { "variant": ComposerVariant.WORKFLOW, @@ -687,8 +701,8 @@ def test_load_agent_app_composer_exposes_draft_save_only(monkeypatch: pytest.Mon draft = AgentConfigDraft(config_snapshot=AgentSoulConfig.model_validate({"prompt": {"system_prompt": "x"}})) monkeypatch.setattr(AgentComposerService, "_require_agent_app_agent", lambda **kwargs: agent) - monkeypatch.setattr(AgentComposerService, "_get_agent_draft", lambda **kwargs: draft) - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **kwargs: None) + monkeypatch.setattr(AgentConfigRepository, "get_draft", lambda **kwargs: draft) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **kwargs: None) monkeypatch.setattr(AgentComposerService, "_serialize_agent", lambda _agent: {"id": _agent.id}) monkeypatch.setattr(AgentComposerService, "_serialize_version", lambda _version: None) monkeypatch.setattr(AgentComposerService, "_serialize_draft", lambda _draft: {"id": "draft-1"}) @@ -863,7 +877,7 @@ def test_save_agent_app_composer_updates_normal_draft(monkeypatch: pytest.Monkey "_save_agent_draft", lambda **kwargs: saved.update(kwargs) or AgentConfigDraft(id="draft-1", home_snapshot_id="home-initial"), ) - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **_kwargs: active_version) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **_kwargs: active_version) monkeypatch.setattr( AgentComposerService, "load_agent_composer", @@ -918,7 +932,7 @@ def test_save_agent_app_composer_keeps_published_when_draft_matches_active_snaps "_save_agent_draft", lambda **_kwargs: AgentConfigDraft(id="draft-1", home_snapshot_id="home-initial"), ) - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **_kwargs: active_version) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **_kwargs: active_version) monkeypatch.setattr( AgentComposerService, "load_agent_composer", @@ -1580,7 +1594,7 @@ def test_build_apply_retires_normal_preview_binding_before_replacing_draft_home( validate_generation = MagicMock() enqueue_collection = MagicMock(side_effect=lambda **_kwargs: lifecycle.append("enqueue")) monkeypatch.setattr(AgentWorkspaceService, "get_active_binding", get_active_binding) - monkeypatch.setattr(AgentWorkspaceService, "validate_binding_generation", validate_generation) + monkeypatch.setattr(AgentWorkspaceRepository, "validate_binding_generation", validate_generation) monkeypatch.setattr(AgentWorkspaceService, "retire_binding", retire_binding) monkeypatch.setattr(composer_service, "enqueue_agent_resource_collection", enqueue_collection) @@ -2279,7 +2293,9 @@ def test_agent_app_build_draft_apply_marks_unpublished_when_build_draft_differs( ) -def test_agent_app_composer_candidates_and_impact(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_agent_app_composer_candidates_and_impact( + monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, *, tool_providers +): session = sqlite_session bindings = [ WorkflowAgentNodeBinding( @@ -2308,14 +2324,16 @@ def _no_draft_workflow(**kwargs): monkeypatch.setattr(AgentComposerService, "_workspace_dify_tools", lambda **kwargs: []) workflow_candidates = AgentComposerService.get_workflow_candidates( + variables=build_workflow_variable_service(database_client=sessionmaker(bind=session.get_bind())), session=session, tenant_id="tenant-1", app_id="app-1", node_id="node-1", user_id="account-1", + tool_providers=tool_providers, ) agent_app_candidates = AgentComposerService.get_agent_app_candidates( - session=session, tenant_id="tenant-1", agent_id="agent-1", user_id="account-1" + session=session, tenant_id="tenant-1", agent_id="agent-1", user_id="account-1", tool_providers=tool_providers ) impact = AgentComposerService.calculate_impact( session=session, tenant_id="tenant-1", current_snapshot_id="version-1" @@ -2490,8 +2508,8 @@ def fake_create_roster_agent_for_composer(**kwargs): ) monkeypatch.setattr(AgentComposerService, "_require_agent", lambda **kwargs: roster_agent) monkeypatch.setattr( - AgentComposerService, - "_require_version", + AgentConfigRepository, + "require_snapshot", lambda **kwargs: AgentConfigSnapshot( id="source-version-1", tenant_id="tenant-1", @@ -2636,10 +2654,10 @@ def test_node_job_only_updates_inline_agent_soul(monkeypatch: pytest.MonkeyPatch config_snapshot=AgentSoulConfig.model_validate({"prompt": {"system_prompt": "old"}}), ) - monkeypatch.setattr(AgentComposerService, "_require_version", lambda **kwargs: current_snapshot) + monkeypatch.setattr(AgentConfigRepository, "require_snapshot", lambda **kwargs: current_snapshot) monkeypatch.setattr(AgentComposerService, "_update_current_version", lambda **kwargs: next_snapshot) monkeypatch.setattr(AgentComposerService, "_require_agent", lambda **kwargs: inline_agent) - monkeypatch.setattr(AgentComposerService, "_get_agent_draft", lambda **kwargs: normal_draft) + monkeypatch.setattr(AgentConfigRepository, "get_draft", lambda **kwargs: normal_draft) binding = WorkflowAgentNodeBinding( tenant_id="tenant-1", @@ -2738,7 +2756,7 @@ def test_get_or_create_normal_agent_draft_rebases_stale_workflow_only_draft(sqli session.add_all([agent, draft, active_snapshot]) session.commit() - resolved = AgentComposerService.get_or_create_normal_agent_draft( + resolved = AgentConfigRepository.normal_draft( session=session, tenant_id="tenant-1", agent=agent, @@ -2779,7 +2797,7 @@ def test_get_or_create_normal_agent_draft_keeps_roster_draft_edits(sqlite_sessio session.add_all([agent, draft]) session.commit() - resolved = AgentComposerService.get_or_create_normal_agent_draft( + resolved = AgentConfigRepository.normal_draft( session=session, tenant_id="tenant-1", agent=agent, @@ -2902,7 +2920,7 @@ def test_node_job_only_rejects_inline_binding_pointing_to_roster_agent( next_snapshot = AgentConfigSnapshot(id="inline-version-2", tenant_id="tenant-1", agent_id="agent-1", version=2) roster_agent = Agent(id="agent-1", scope=AgentScope.ROSTER) - monkeypatch.setattr(AgentComposerService, "_require_version", lambda **kwargs: current_snapshot) + monkeypatch.setattr(AgentConfigRepository, "require_snapshot", lambda **kwargs: current_snapshot) monkeypatch.setattr(AgentComposerService, "_update_current_version", lambda **kwargs: next_snapshot) monkeypatch.setattr(AgentComposerService, "_require_agent", lambda **kwargs: roster_agent) @@ -2999,7 +3017,7 @@ def test_copy_workflow_composer_from_roster_creates_inline_agent_and_preserves_n monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: workflow) monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **kwargs: binding) monkeypatch.setattr(AgentComposerService, "_require_agent", lambda **kwargs: roster_agent) - monkeypatch.setattr(AgentComposerService, "_require_version", lambda **kwargs: source_version) + monkeypatch.setattr(AgentConfigRepository, "require_snapshot", lambda **kwargs: source_version) def fake_create_workflow_only_agent(**kwargs): captured["create"] = kwargs @@ -3086,7 +3104,7 @@ def test_copy_workflow_composer_from_roster_rejects_stale_source_snapshot( ) session.commit() monkeypatch.setattr(AgentComposerService, "_require_agent", lambda **kwargs: roster_agent) - monkeypatch.setattr(AgentComposerService, "_require_version", lambda **kwargs: source_version) + monkeypatch.setattr(AgentConfigRepository, "require_snapshot", lambda **kwargs: source_version) with pytest.raises(AgentVersionConflictError): AgentComposerService.copy_workflow_composer_from_roster( @@ -3129,7 +3147,7 @@ def test_copy_workflow_composer_from_roster_rejects_unpublished_source( monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **kwargs: binding) monkeypatch.setattr(AgentComposerService, "_require_agent", lambda **kwargs: source_agent) require_version = MagicMock(side_effect=AssertionError("unpublished source must fail before loading snapshot")) - monkeypatch.setattr(AgentComposerService, "_require_version", require_version) + monkeypatch.setattr(AgentConfigRepository, "require_snapshot", require_version) with pytest.raises(InvalidComposerConfigError, match="published config snapshot"): AgentComposerService.copy_workflow_composer_from_roster( @@ -3179,8 +3197,8 @@ def test_copy_workflow_composer_from_roster_is_idempotent_when_already_inline( session = sqlite_session monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: _workflow()) monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **kwargs: inline_binding) - monkeypatch.setattr(AgentComposerService, "_get_agent_if_present", lambda **kwargs: inline_agent) - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **kwargs: inline_version) + monkeypatch.setattr(AgentConfigRepository, "get_agent", lambda **kwargs: inline_agent) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **kwargs: inline_version) def serialize_workflow_state(**kwargs): serialize_calls.append(kwargs) @@ -3319,8 +3337,8 @@ def get_app_backing_agent(self, *, tenant_id, app_id): monkeypatch.setattr(composer_service, "AgentRosterService", FakeAgentRosterService) monkeypatch.setattr(AgentComposerService, "_require_account", lambda **kwargs: _account()) monkeypatch.setattr( - AgentComposerService, - "_require_version", + AgentConfigRepository, + "require_snapshot", lambda **kwargs: AgentConfigSnapshot( id="empty-version-1", tenant_id="tenant-1", @@ -3453,7 +3471,7 @@ def test_agent_app_draft_match_does_not_mark_create_version_as_published( ) snapshot = AgentConfigSnapshot(config_snapshot=agent_soul, home_snapshot_id="home-1") session = sqlite_session - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **kwargs: snapshot) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **kwargs: snapshot) assert ( AgentComposerService._agent_soul_matches_active_config( @@ -3489,7 +3507,7 @@ def test_agent_app_draft_match_marks_publish_visible_revision_as_published( ) ) session.commit() - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **kwargs: snapshot) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **kwargs: snapshot) assert ( AgentComposerService._agent_soul_matches_active_config( @@ -3552,9 +3570,9 @@ def test_composer_version_helpers_and_lookup_errors(monkeypatch: pytest.MonkeyPa ) with pytest.raises(composer_service.AgentNotFoundError): AgentComposerService._require_agent(session=session, tenant_id="tenant-1", agent_id=None) - assert AgentComposerService._get_agent_if_present(session=session, tenant_id="tenant-1", agent_id="missing") is None + assert AgentConfigRepository.get_agent(session=session, tenant_id="tenant-1", agent_id="missing") is None assert ( - AgentComposerService._require_version( + AgentConfigRepository.require_snapshot( session=session, tenant_id="tenant-1", agent_id="agent-1", @@ -3563,7 +3581,7 @@ def test_composer_version_helpers_and_lookup_errors(monkeypatch: pytest.MonkeyPa == "version-1" ) with pytest.raises(composer_service.AgentVersionNotFoundError): - AgentComposerService._require_version( + AgentConfigRepository.require_snapshot( session=session, tenant_id="tenant-1", agent_id="agent-1", @@ -3596,7 +3614,7 @@ def test_composer_current_version_and_error_paths(monkeypatch: pytest.MonkeyPatc home_snapshot_id="home-1", config_snapshot='{"prompt":{"system_prompt":"old"}}', ) - monkeypatch.setattr(AgentComposerService, "_require_version", lambda **kwargs: version) + monkeypatch.setattr(AgentConfigRepository, "require_snapshot", lambda **kwargs: version) monkeypatch.setattr( AgentComposerService, "_require_agent", @@ -5636,9 +5654,8 @@ def _sync_roster_agent_task_refs( session.add(existing_binding) session.commit() - WorkflowAgentPublishService.sync_roster_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -5668,9 +5685,13 @@ def test_publish_validation_rejects_agent_soul_publish_only_errors(self, sqlite_ session.commit() with pytest.raises(InvalidComposerConfigError, match="human_involvement_not_referenced"): - WorkflowAgentPublishService.validate_agent_nodes_for_publish( - session=session, - draft_workflow=self._agent_workflow(), + WorkflowAgentPublishService.validate_publication_state( + build_workflow_agent_service(session).publication_state( + draft_workflow=workflow_binding_scope(self._agent_workflow()), + ), + skills=WorkflowAgentSkillReader( + SkillManagementService(session_maker=sessionmaker(bind=session.get_bind())) + ), ) def test_publish_validation_keeps_workspace_skill_refs_when_config_files_are_present(self, sqlite_session: Session): @@ -5737,9 +5758,13 @@ def test_publish_validation_keeps_workspace_skill_refs_when_config_files_are_pre session.add_all([binding, agent, snapshot, skill, skill_version, skill_binding, archive_file]) session.commit() - WorkflowAgentPublishService.validate_agent_nodes_for_publish( - session=session, - draft_workflow=self._agent_workflow(), + WorkflowAgentPublishService.validate_publication_state( + build_workflow_agent_service(session).publication_state( + draft_workflow=workflow_binding_scope(self._agent_workflow()), + ), + skills=WorkflowAgentSkillReader( + SkillManagementService(session_maker=sessionmaker(bind=session.get_bind())) + ), ) def test_publish_validation_rejects_dangling_agent_soul_config_refs(self, sqlite_session: Session): @@ -5761,9 +5786,13 @@ def test_publish_validation_rejects_dangling_agent_soul_config_refs(self, sqlite session.commit() with pytest.raises(WorkflowAgentNodeValidationError, match="skill_ref_dangling"): - WorkflowAgentPublishService.validate_agent_nodes_for_publish( - session=session, - draft_workflow=self._agent_workflow(), + WorkflowAgentPublishService.validate_publication_state( + build_workflow_agent_service(session).publication_state( + draft_workflow=workflow_binding_scope(self._agent_workflow()), + ), + skills=WorkflowAgentSkillReader( + SkillManagementService(session_maker=sessionmaker(bind=session.get_bind())) + ), ) def test_publish_validation_rejects_missing_config_assets(self): @@ -5850,9 +5879,8 @@ def test_projects_binding_declared_outputs_to_draft_graph_response(self, sqlite_ session.add(binding) session.commit() - graph = WorkflowAgentPublishService.project_draft_bindings_to_graph( - session=session, - draft_workflow=workflow, + graph = build_workflow_agent_service(session).project_draft_bindings_to_graph( + draft_workflow=workflow_binding_scope(workflow), ) node_data = graph["nodes"][0]["data"] @@ -5911,9 +5939,8 @@ def test_projects_inline_binding_over_pending_inline_graph_response(self, sqlite session.add(binding) session.commit() - graph = WorkflowAgentPublishService.project_draft_bindings_to_graph( - session=session, - draft_workflow=workflow, + graph = build_workflow_agent_service(session).project_draft_bindings_to_graph( + draft_workflow=workflow_binding_scope(workflow), ) assert graph["nodes"][0]["data"]["agent_binding"] == { @@ -5966,9 +5993,8 @@ def test_keeps_pending_inline_graph_response_over_existing_roster_binding(self, session.add(binding) session.commit() - graph = WorkflowAgentPublishService.project_draft_bindings_to_graph( - session=session, - draft_workflow=workflow, + graph = build_workflow_agent_service(session).project_draft_bindings_to_graph( + draft_workflow=workflow_binding_scope(workflow), ) assert graph["nodes"][0]["data"]["agent_binding"] == { @@ -6022,9 +6048,8 @@ def test_creates_roster_binding_from_agent_node_graph(self, sqlite_session: Sess session.add_all([agent, self._publish_revision()]) session.commit() - WorkflowAgentPublishService.sync_roster_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6147,9 +6172,8 @@ def test_creates_inline_binding_from_agent_node_graph(self, sqlite_session: Sess session.add_all([agent, snapshot]) session.commit() - WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6204,9 +6228,8 @@ def test_keeps_pending_inline_binding_in_draft_graph_without_db_binding(self, sq session.add(existing_binding) session.commit() - WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6258,9 +6281,8 @@ def test_clones_inline_binding_for_agent_owned_by_another_node(self, monkeypatch clone = MagicMock(return_value=(Agent(id="cloned-agent"), "cloned-snapshot")) monkeypatch.setattr(WorkflowAgentPublishService, "_clone_inline_graph_binding_for_node", clone) - WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6300,9 +6322,8 @@ def test_rejects_agent_node_graph_binding_with_unsupported_type(self, sqlite_ses ) with pytest.raises(ValueError, match="unsupported agent_binding type"): - WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6333,9 +6354,8 @@ def test_treats_partial_inline_binding_as_pending_draft_state(self, sqlite_sessi ), ) - WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6385,9 +6405,8 @@ def test_rejects_inline_binding_with_missing_snapshot(self, sqlite_session: Sess session.commit() with pytest.raises(ValueError, match="missing inline agent config snapshot"): - WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6448,9 +6467,8 @@ def test_updates_existing_roster_binding_prompt_from_agent_node_graph(self, sqli session.add_all([agent, self._publish_revision(), existing_binding]) session.commit() - WorkflowAgentPublishService.sync_roster_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6521,9 +6539,8 @@ def test_updates_existing_roster_binding_declared_outputs_from_agent_node_graph( session.add_all([agent, self._publish_revision(), existing_binding]) session.commit() - WorkflowAgentPublishService.sync_roster_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6722,9 +6739,8 @@ def test_deletes_draft_binding_and_returns_only_replaced_inline_agent( ] ) sqlite_session.commit() - retirement_candidates = WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=sqlite_session, - draft_workflow=workflow, + retirement_candidates = build_workflow_agent_service(sqlite_session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6889,7 +6905,7 @@ def count_flush(_session: Session, _flush_context: object) -> None: "_save_agent_draft", lambda **kwargs: saved.update(kwargs) or AgentConfigDraft(id="draft-1", home_snapshot_id="home-initial"), ) - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **_kwargs: active_version) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **_kwargs: active_version) monkeypatch.setattr(AgentComposerService, "load_agent_composer", lambda **_kwargs: {"loaded": True}) payload = ComposerSavePayload.model_validate( @@ -6931,7 +6947,9 @@ def count_flush(_session: Session, _flush_context: object) -> None: assert not session.dirty -def test_workspace_dify_tools_returns_provider_and_tool_granularities(monkeypatch: pytest.MonkeyPatch): +def test_workspace_dify_tools_returns_provider_and_tool_granularities( + monkeypatch: pytest.MonkeyPatch, *, tool_providers +): """The slash-menu Tools tab needs both selection granularities: a provider hosts many tools (like an MCP server), so candidates return one provider-level entry (id = /*, = all tools) plus one per tool.""" @@ -6948,15 +6966,17 @@ def test_workspace_dify_tools_returns_provider_and_tool_granularities(monkeypatc ], ) - import services.tools.legacy_builtin_tools_manage_service as builtin_tools_module + import services.tools.builtin_tools_manage_service as builtin_tools_module monkeypatch.setattr( builtin_tools_module.BuiltinToolManageService, "list_builtin_tools", - staticmethod(lambda user_id, tenant_id: [provider]), + staticmethod(lambda user_id, tenant_id, **_kwargs: [provider]), ) - entries = AgentComposerService._workspace_dify_tools(tenant_id="tenant-1", user_id="user-1") + entries = AgentComposerService._workspace_dify_tools( + tenant_id="tenant-1", user_id="user-1", tool_providers=tool_providers + ) assert entries[0] == { "id": "duckduckgo/*", diff --git a/api/tests/unit_tests/services/agent/test_deletion_service.py b/api/tests/unit_tests/services/agent/test_deletion_service.py index 26e9c472a6b3b9..44dcb65d65a641 100644 --- a/api/tests/unit_tests/services/agent/test_deletion_service.py +++ b/api/tests/unit_tests/services/agent/test_deletion_service.py @@ -9,6 +9,7 @@ from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.sql.dml import Delete +from enums.agent import WorkflowAgentBindingType from models.agent import ( Agent, AgentConfigDraft, @@ -27,7 +28,6 @@ AgentWorkspace, AgentWorkspaceBinding, AgentWorkspaceOwnerType, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import AgentSoulConfig diff --git a/api/tests/unit_tests/services/agent/test_home_snapshot_service.py b/api/tests/unit_tests/services/agent/test_home_snapshot_service.py index 444959aec092f1..7fbad20cdba185 100644 --- a/api/tests/unit_tests/services/agent/test_home_snapshot_service.py +++ b/api/tests/unit_tests/services/agent/test_home_snapshot_service.py @@ -23,6 +23,7 @@ AgentWorkspaceOwnerType, ) from models.agent_config_entities import AgentSoulConfig +from repositories.agent_workspace_repository import AgentWorkspaceRepository from services.agent.errors import ( AgentBuildSandboxNotFoundError, AgentHomeSnapshotCreateFailedError, @@ -404,7 +405,7 @@ def _apply_with_client_error(monkeypatch: pytest.MonkeyPatch, error: Exception) client.create_home_snapshot_from_binding_sync.side_effect = error monkeypatch.setattr(AgentHomeSnapshotService, "_client", lambda: nullcontext(client)) monkeypatch.setattr(AgentWorkspaceService, "get_active_binding", MagicMock(return_value=binding)) - monkeypatch.setattr(AgentWorkspaceService, "validate_binding_generation", MagicMock()) + monkeypatch.setattr(AgentWorkspaceRepository, "validate_binding_generation", MagicMock()) AgentHomeSnapshotService.create_for_build_apply(session=session, build_draft=_build_draft()) diff --git a/api/tests/unit_tests/services/agent/test_log_service.py b/api/tests/unit_tests/services/agent/test_log_service.py index 993310df381239..cd88a3f7452b29 100644 --- a/api/tests/unit_tests/services/agent/test_log_service.py +++ b/api/tests/unit_tests/services/agent/test_log_service.py @@ -43,7 +43,7 @@ def _provide_app_context(): @pytest.fixture -def store(monkeypatch): +def store(monkeypatch, workflow_variables): engine = create_engine("sqlite://", poolclass=QueuePool) TypeBase.metadata.create_all( engine, @@ -151,7 +151,7 @@ def reject_global(*_args, **_kwargs): message_files='["generated-file"]', ) ) - service = build_agent_app_services(database_client=sessions).logs + service = build_agent_app_services(database_client=sessions, variables=workflow_variables).logs try: yield sessions, service finally: diff --git a/api/tests/unit_tests/services/agent/test_retirement_service.py b/api/tests/unit_tests/services/agent/test_retirement_service.py index 47244d7b8c4c2a..0eb38ffba9647f 100644 --- a/api/tests/unit_tests/services/agent/test_retirement_service.py +++ b/api/tests/unit_tests/services/agent/test_retirement_service.py @@ -1,10 +1,9 @@ -from contextlib import nullcontext -from types import SimpleNamespace from unittest.mock import MagicMock import pytest -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker +from enums.agent import WorkflowAgentBindingType from models.agent import ( Agent, AgentConfigVersionKind, @@ -17,37 +16,25 @@ AgentWorkspace, AgentWorkspaceBinding, AgentWorkspaceOwnerType, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.enums import AppStatus from models.model import App, AppMode from models.workflow import Workflow, WorkflowType -from services.agent.home_snapshot_service import AgentHomeSnapshotService -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService -from services.agent.workspace_service import AgentWorkspaceService +from repositories.agent.retirement_repository import AgentRetirement, WorkflowAgentRetirementRepository +from services.agent.retirement_service import WorkflowAgentRetirementService -def test_retire_unowned_failure_propagates(monkeypatch: pytest.MonkeyPatch) -> None: - context = MagicMock() +def test_retire_unowned_failure_propagates() -> None: + repository = MagicMock(spec=WorkflowAgentRetirementRepository) error = RuntimeError("retirement failed") - monkeypatch.setattr( - "services.agent.legacy_retirement_service.session_factory.create_session", - lambda: context, - ) - monkeypatch.setattr( - WorkflowAgentRetirementService, - "archive_unowned", - MagicMock(side_effect=error), - ) - + repository.retire.side_effect = error with pytest.raises(RuntimeError) as exc_info: - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService(repository).retire_unowned( tenant_id="tenant-1", agent_ids=["agent-1"], account_id="account-1", ) - assert exc_info.value is error @@ -81,6 +68,7 @@ def _workflow_only_agent(*, backing_app_id: str | None = None) -> Agent: def test_retire_unowned_requires_an_exact_persisted_workflow_owner_key( monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, + sqlite_session_factory: sessionmaker[Session], workflow_version: str, pointer_to_owner: bool, mismatched_key: str | None, @@ -132,18 +120,15 @@ def test_retire_unowned_requires_an_exact_persisted_workflow_owner_key( ) sqlite_session.add_all([agent, app, workflow, binding]) sqlite_session.commit() - monkeypatch.setattr( - "services.agent.legacy_retirement_service.session_factory.create_session", - lambda: nullcontext(sqlite_session), - ) celery_delay = MagicMock() monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", celery_delay) - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService(WorkflowAgentRetirementRepository(sqlite_session_factory)).retire_unowned( tenant_id="tenant-1", agent_ids=[agent.id], account_id="account-1", ) + sqlite_session.expire_all() stored_agent = sqlite_session.get(Agent, agent.id) assert stored_agent is not None assert stored_agent.status is expected_status @@ -161,6 +146,7 @@ def test_retire_unowned_requires_an_exact_persisted_workflow_owner_key( def test_retire_unowned_archives_orphan_and_retires_resources( monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, + sqlite_session_factory: sessionmaker[Session], ) -> None: agent = _workflow_only_agent(backing_app_id="hidden-app-1") hidden_app = App( @@ -204,24 +190,23 @@ def test_retire_unowned_archives_orphan_and_retires_resources( ) sqlite_session.add_all([agent, hidden_app, home, workspace, binding]) sqlite_session.commit() - monkeypatch.setattr( - "services.agent.legacy_retirement_service.session_factory.create_session", - lambda: nullcontext(sqlite_session), - ) + hidden_app_id = hidden_app.id + monkeypatch.setattr("core.db.session_factory._session_maker", sessionmaker()) cleanup_app = MagicMock() enqueue_collection = MagicMock() - monkeypatch.setattr("services.agent.legacy_retirement_service.remove_app_and_related_data_task.delay", cleanup_app) + monkeypatch.setattr("services.agent.retirement_service.remove_app_and_related_data_task.delay", cleanup_app) monkeypatch.setattr( - "services.agent.legacy_retirement_service.enqueue_agent_resource_collection", + "services.agent.retirement_service.enqueue_agent_resource_collection", enqueue_collection, ) - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService(WorkflowAgentRetirementRepository(sqlite_session_factory)).retire_unowned( tenant_id="tenant-1", agent_ids=[agent.id], account_id="account-1", ) + sqlite_session.expire_all() stored_agent = sqlite_session.get(Agent, agent.id) stored_binding = sqlite_session.get(AgentWorkspaceBinding, binding.id) stored_workspace = sqlite_session.get(AgentWorkspace, workspace.id) @@ -230,12 +215,12 @@ def test_retire_unowned_archives_orphan_and_retires_resources( assert stored_binding is not None assert stored_workspace is not None assert stored_home is not None - assert sqlite_session.get(App, hidden_app.id) is None + assert sqlite_session.get(App, hidden_app_id) is None assert stored_agent.status is AgentStatus.ARCHIVED assert stored_binding.status is AgentWorkingResourceStatus.RETIRED assert stored_workspace.status is AgentWorkingResourceStatus.RETIRED assert stored_home.status is AgentWorkingResourceStatus.RETIRED - cleanup_app.assert_called_once_with(tenant_id="tenant-1", app_id=hidden_app.id) + cleanup_app.assert_called_once_with(tenant_id="tenant-1", app_id=hidden_app_id) enqueue_collection.assert_called_once_with( tenant_id="tenant-1", workspace_ids=[workspace.id], @@ -246,44 +231,21 @@ def test_retire_unowned_archives_orphan_and_retires_resources( def test_hidden_app_enqueue_failure_prevents_agent_purge_enqueue(monkeypatch: pytest.MonkeyPatch) -> None: - context = MagicMock() - session = context.__enter__.return_value - session.scalars.side_effect = [ - SimpleNamespace( - all=MagicMock( - return_value=[ - SimpleNamespace(backing_app_id="hidden-app-1"), - SimpleNamespace(backing_app_id="hidden-app-2"), - ] - ) - ), - SimpleNamespace(all=MagicMock(return_value=[])), - SimpleNamespace(all=MagicMock(return_value=[])), - SimpleNamespace(all=MagicMock(return_value=[])), - SimpleNamespace(all=MagicMock(return_value=[])), - ] - monkeypatch.setattr( - "services.agent.legacy_retirement_service.session_factory.create_session", - lambda: context, + repository = MagicMock(spec=WorkflowAgentRetirementRepository) + repository.retire.return_value = AgentRetirement( + ["agent-1", "agent-2"], ["hidden-app-1", "hidden-app-2"], [], [], [] ) - monkeypatch.setattr( - WorkflowAgentRetirementService, - "archive_unowned", - MagicMock(return_value=["agent-1", "agent-2"]), - ) - monkeypatch.setattr(AgentWorkspaceService, "retire_all_for_app", MagicMock(return_value=[])) - monkeypatch.setattr(AgentHomeSnapshotService, "retire_all_for_agent", MagicMock(return_value=[])) error = RuntimeError("broker unavailable") cleanup_app = MagicMock(side_effect=[None, error]) - monkeypatch.setattr("services.agent.legacy_retirement_service.remove_app_and_related_data_task.delay", cleanup_app) + monkeypatch.setattr("services.agent.retirement_service.remove_app_and_related_data_task.delay", cleanup_app) enqueue_collection = MagicMock() monkeypatch.setattr( - "services.agent.legacy_retirement_service.enqueue_agent_resource_collection", + "services.agent.retirement_service.enqueue_agent_resource_collection", enqueue_collection, ) with pytest.raises(RuntimeError) as exc_info: - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService(repository).retire_unowned( tenant_id="tenant-1", agent_ids=["agent-1", "agent-2"], account_id="account-1", @@ -292,103 +254,3 @@ def test_hidden_app_enqueue_failure_prevents_agent_purge_enqueue(monkeypatch: py assert exc_info.value is error assert [call.kwargs["app_id"] for call in cleanup_app.call_args_list] == ["hidden-app-1", "hidden-app-2"] enqueue_collection.assert_not_called() - - -def test_retire_unowned_retry_after_hidden_app_enqueue_failure_preserves_full_collector_payload( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -) -> None: - agent = _workflow_only_agent(backing_app_id="hidden-app-1") - hidden_app = App( - id="hidden-app-1", - tenant_id="tenant-1", - name="Inline Agent runtime", - mode=AppMode.AGENT, - status=AppStatus.NORMAL, - enable_site=False, - enable_api=False, - ) - home = AgentHomeSnapshot( - id="home-1", - tenant_id="tenant-1", - agent_id=agent.id, - snapshot_ref="home-ref", - status=AgentWorkingResourceStatus.ACTIVE, - ) - workspace = AgentWorkspace( - id="workspace-1", - tenant_id="tenant-1", - app_id=hidden_app.id, - owner_type=AgentWorkspaceOwnerType.CONVERSATION, - owner_id="conversation-1", - owner_scope_key="root", - backend_workspace_ref="workspace-ref", - status=AgentWorkingResourceStatus.ACTIVE, - active_guard=1, - ) - binding = AgentWorkspaceBinding( - id="binding-1", - tenant_id="tenant-1", - app_id=hidden_app.id, - workspace_id=workspace.id, - agent_id=agent.id, - base_home_snapshot_id=home.id, - agent_config_version_id="config-1", - agent_config_version_kind=AgentConfigVersionKind.SNAPSHOT, - backend_binding_ref="binding-ref", - status=AgentWorkingResourceStatus.ACTIVE, - ) - sqlite_session.add_all([agent, hidden_app, home, workspace, binding]) - sqlite_session.commit() - agent_id = agent.id - hidden_app_id = hidden_app.id - home_id = home.id - workspace_id = workspace.id - binding_id = binding.id - error = RuntimeError("broker unavailable") - cleanup_app = MagicMock(side_effect=[error, None]) - enqueue_collection = MagicMock() - monkeypatch.setattr("services.agent.legacy_retirement_service.remove_app_and_related_data_task.delay", cleanup_app) - monkeypatch.setattr( - "services.agent.legacy_retirement_service.enqueue_agent_resource_collection", - enqueue_collection, - ) - - with pytest.raises(RuntimeError) as exc_info: - WorkflowAgentRetirementService.retire_unowned( - tenant_id="tenant-1", - agent_ids=[agent_id], - account_id="account-1", - ) - - assert exc_info.value is error - sqlite_session.expire_all() - stored_agent = sqlite_session.get(Agent, agent_id) - stored_workspace = sqlite_session.get(AgentWorkspace, workspace_id) - stored_binding = sqlite_session.get(AgentWorkspaceBinding, binding_id) - stored_home = sqlite_session.get(AgentHomeSnapshot, home_id) - assert stored_agent is not None - assert stored_workspace is not None - assert stored_binding is not None - assert stored_home is not None - assert stored_agent.status is AgentStatus.ARCHIVED - assert sqlite_session.get(App, hidden_app_id) is None - assert stored_workspace.status is AgentWorkingResourceStatus.RETIRED - assert stored_binding.status is AgentWorkingResourceStatus.RETIRED - assert stored_home.status is AgentWorkingResourceStatus.RETIRED - enqueue_collection.assert_not_called() - - WorkflowAgentRetirementService.retire_unowned( - tenant_id="tenant-1", - agent_ids=[agent_id], - account_id="account-1", - ) - - assert cleanup_app.call_count == 2 - enqueue_collection.assert_called_once_with( - tenant_id="tenant-1", - workspace_ids=[workspace_id], - binding_ids=[binding_id], - home_snapshot_ids=[home_id], - purge_agent_ids=[agent_id], - ) diff --git a/api/tests/unit_tests/services/agent/test_roster_package_importer.py b/api/tests/unit_tests/services/agent/test_roster_package_importer.py index eff7b37fc29a7a..3b0991ebd6c8ef 100644 --- a/api/tests/unit_tests/services/agent/test_roster_package_importer.py +++ b/api/tests/unit_tests/services/agent/test_roster_package_importer.py @@ -31,17 +31,12 @@ AgentSource, AgentStatus, ) -from models.agent_config_entities import AgentSoulConfig +from models.agent_config_entities import AgentPackageMetadata, AgentSoulConfig from models.base import Base, TypeBase from models.model import App, AppModelConfig, InstalledApp, Site, UploadFile from models.tools import ToolFile from services.agent import roster_package_importer as importer_module -from services.agent.dsl_entities import ( - AgentPackage, - AgentPackageMetadata, - AgentPackageWorkspaceSkill, - make_agent_app_dsl, -) +from services.agent.dsl_entities import AgentPackage, AgentPackageWorkspaceSkill, make_agent_app_dsl from services.agent.errors import ( AgentNameConflictError, InvalidRosterAgentPackageError, diff --git a/api/tests/unit_tests/services/agent/test_roster_package_service.py b/api/tests/unit_tests/services/agent/test_roster_package_service.py index c2189cb0b06b04..e5d1afd4b2c648 100644 --- a/api/tests/unit_tests/services/agent/test_roster_package_service.py +++ b/api/tests/unit_tests/services/agent/test_roster_package_service.py @@ -30,13 +30,13 @@ AgentSource, AgentStatus, ) -from models.agent_config_entities import AgentSoulConfig +from models.agent_config_entities import AgentPackageMetadata, AgentSoulConfig from models.enums import AppStatus, CreatorUserRole, CustomizeTokenStrategy from models.model import App, AppMode, IconType, Site from models.skill import AgentSkillBindingSnapshot, Skill, SkillVersion, SkillVersionManifest from models.tools import ToolFile from services.agent import roster_package_exporter as roster_package_exporter_module -from services.agent.dsl_entities import AgentAppDsl, AgentPackage, AgentPackageMetadata, make_agent_app_dsl +from services.agent.dsl_entities import AgentAppDsl, AgentPackage, make_agent_app_dsl from services.agent.errors import ( AgentVersionNotFoundError, InvalidRosterAgentPackageError, @@ -1063,15 +1063,14 @@ def test_export_uses_current_workspace_skill_bindings( pool = sqlite_engine.pool assert isinstance(pool, QueuePool) - def load_legacy_skill(*, tenant_id: str, file_id: str) -> bytes: - assert tenant_id == "tenant-1" - assert file_id + def load_published_skill(storage_key: str) -> bytes: + assert storage_key == "tools/workspace.zip" assert pool.checkedout() == 0 return workspace_payload monkeypatch.setattr( - "services.skill_management_service.SkillManagementService._load_tool_file_bytes", - staticmethod(load_legacy_skill), + "services.skill_management_service.storage.load_once", + load_published_skill, ) archive_file = ToolFile( user_id="account-1", diff --git a/api/tests/unit_tests/services/agent/test_tool_invocation_service.py b/api/tests/unit_tests/services/agent/test_tool_invocation_service.py index c3f58ddaa17210..f363343c3d9f54 100644 --- a/api/tests/unit_tests/services/agent/test_tool_invocation_service.py +++ b/api/tests/unit_tests/services/agent/test_tool_invocation_service.py @@ -4,16 +4,21 @@ from unittest.mock import Mock, patch import pytest +from sqlalchemy import Connection, Engine, event from sqlalchemy.orm import Session, sessionmaker +from core.app.entities.app_invoke_entities import InvokeFrom from core.tools.__base.tool import Tool -from core.tools.entities.tool_entities import ToolInvokeMessage, ToolProviderType +from core.tools.__base.tool_runtime import ToolRuntime +from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_entities import ToolEntity, ToolIdentity, ToolInvokeMessage, ToolProviderType from core.tools.errors import ( ToolInvokeError, ToolParameterValidationError, ToolProviderCredentialValidationError, ToolProviderNotFoundError, ) +from core.tools.plugin_tool.tool import PluginTool from models.enums import AppStatus from models.model import App, AppMode from repositories.app.agent_app_repository import AgentAppRepository @@ -21,6 +26,7 @@ from services.entities.agent_tool_inner import AgentToolInvokeRequest from services.errors.agent_tool_inner import AgentToolInnerServiceError from services.tools.agent_invocation_gateway import AgentToolInvocationGateway +from services.tools.tool_manager import ToolManager from services.workflow.execution.ports import WorkflowRuntime from services.workflow.variable_contracts import WorkflowExecutionVariables @@ -93,6 +99,69 @@ def _service(session: Session, runtime: WorkflowRuntime, variables: WorkflowExec ) +@pytest.mark.parametrize("sqlite_session", [(App,)], indirect=True) +@pytest.mark.parametrize("fail_during_stream", [False, True]) +def test_app_lookup_releases_connection_before_tool_setup_and_stream_consumption( + sqlite_session: Session, + sqlite_engine: Engine, + workflow_runtime: WorkflowRuntime, + monkeypatch: pytest.MonkeyPatch, + fail_during_stream: bool, +) -> None: + _persist_app(sqlite_session) + active: set[Connection] = set() + begin, finish = active.add, active.discard + event.listen(sqlite_engine, "begin", begin) + event.listen(sqlite_engine, "commit", finish) + event.listen(sqlite_engine, "rollback", finish) + tool = PluginTool( + entity=ToolEntity( + identity=ToolIdentity( + author="author", name="search", provider="provider", label=I18nObject(en_US="Search") + ), + parameters=[], + ), + runtime=ToolRuntime(tenant_id=TENANT_ID, invoke_from=InvokeFrom.SERVICE_API, runtime_parameters={}), + tenant_id=TENANT_ID, + icon="icon.svg", + plugin_unique_identifier="plugin-id", + ) + calls = [] + + def resolve_runtime(**_kwargs): + assert not active + calls.append("setup") + return tool + + class PluginTransport: + def invoke(self, **kwargs): + assert not active + assert kwargs["tenant_id"] == TENANT_ID + assert kwargs["app_id"] == APP_ID + calls.append("stream") + yield tool.create_text_message("ok") + assert not active + if fail_during_stream: + raise ToolInvokeError("stream interrupted") + + monkeypatch.setattr(ToolManager, "get_agent_tool_runtime", resolve_runtime) + monkeypatch.setattr("core.tools.plugin_tool.tool.PluginToolManager", PluginTransport) + try: + service = _service(sqlite_session, workflow_runtime) + if fail_during_stream: + with pytest.raises(AgentToolInnerServiceError) as raised: + service.invoke(_request()) + assert raised.value.error_code == "agent_tool_invoke_failed" + else: + assert service.invoke(_request()).observation == "ok" + assert calls == ["setup", "stream"] + assert not active + finally: + event.remove(sqlite_engine, "begin", begin) + event.remove(sqlite_engine, "commit", finish) + event.remove(sqlite_engine, "rollback", finish) + + @pytest.mark.parametrize("sqlite_session", [(App,)], indirect=True) def test_invoke_uses_agent_tool_runtime_and_returns_observation( sqlite_session: Session, *, workflow_runtime: WorkflowRuntime diff --git a/api/tests/unit_tests/services/agent/test_workflow_package_service.py b/api/tests/unit_tests/services/agent/test_workflow_package_service.py index 79cd06f1fe1370..1539d0b3c33afb 100644 --- a/api/tests/unit_tests/services/agent/test_workflow_package_service.py +++ b/api/tests/unit_tests/services/agent/test_workflow_package_service.py @@ -16,16 +16,10 @@ from sqlalchemy.orm import Session from sqlalchemy.pool import QueuePool +from enums.agent import WorkflowAgentBindingType +from extensions.application_services.workflow import build_app_dsl_service from models import Account, App, AppMode -from models.agent import ( - Agent, - AgentConfigSnapshot, - AgentScope, - AgentSource, - AgentStatus, - WorkflowAgentBindingType, - WorkflowAgentNodeBinding, -) +from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource, AgentStatus, WorkflowAgentNodeBinding from models.agent_config_entities import AgentSoulConfig from models.model import UploadFile from models.skill import AgentSkillBinding, AgentSkillBindingSnapshot, Skill, SkillVersion, SkillVersionManifest @@ -228,7 +222,7 @@ def _import(session: Session, content: bytes, *, app_id: str | None = None, acco prepared = AppPackageService().read_package(io.BytesIO(content)) assert prepared is not None with prepared: - result = AppDslService(session).import_app( + result = build_app_dsl_service(session).import_app( account=account or _account(), import_mode="yaml-content", yaml_content=prepared.dsl, @@ -409,7 +403,7 @@ def mutate(manifest: AppPackageManifest, data: dict[str, object], members: dict[ result = _import(sqlite_session, _rewrite(content, mutate)) assert result.status == ImportStatus.PENDING saved_count = len(storage.saved) - confirmed = AppDslService(sqlite_session).confirm_import(import_id=result.id, account=_account()) + confirmed = build_app_dsl_service(sqlite_session).confirm_import(import_id=result.id, account=_account()) assert confirmed.status == ImportStatus.COMPLETED_WITH_WARNINGS, confirmed.error assert any( warning.code == "agent_skill_missing" and warning.path.startswith("agent_packages.agent_1.") @@ -427,9 +421,14 @@ def test_overwrite_permission_is_checked_before_resource_upload( app = _source(sqlite_session, storage) with AppPackageService().export_app(app_model=app) as exported: content = exported.archive.read() - monkeypatch.setattr(AppDslService, "_load_app_for_overwrite", Mock(side_effect=NoPermissionError("Denied"))) - with pytest.raises(NoPermissionError): + target = make_app(app_id="protected-app", tenant_id="tenant-2", mode=AppMode.WORKFLOW, maintainer="another-user") + sqlite_session.add(target) + sqlite_session.commit() + denied = Mock(return_value=False) + monkeypatch.setattr(AppDslService, "_check_overwrite_access", denied) + with pytest.raises(NoPermissionError, match="permission to overwrite"): _import(sqlite_session, content, app_id="protected-app") + denied.assert_called_once() assert not storage.saved @@ -450,7 +449,7 @@ def test_overwrite_replaces_inline_agents_and_reports_previous_owners( ) sqlite_session.commit() retirement = Mock() - monkeypatch.setattr("services.app_dsl_service.WorkflowAgentRetirementService.retire_unowned", retirement) + monkeypatch.setattr("services.agent.retirement_service.WorkflowAgentRetirementService.retire_unowned", retirement) updated = _import(sqlite_session, content, app_id=original.app_id) assert updated.status == ImportStatus.COMPLETED, updated.error assert updated.app_id == original.app_id @@ -460,8 +459,13 @@ def test_overwrite_replaces_inline_agents_and_reports_previous_owners( ) ) assert len(new_agents) == len(old_agents) == 2 + assert None not in old_agents assert not (new_agents & old_agents) - retirement.assert_called_once_with(tenant_id="tenant-2", agent_ids=old_agents, account_id="account-1") + retirement.assert_called_once_with( + tenant_id="tenant-2", + agent_ids=sorted(agent_id for agent_id in old_agents if agent_id is not None), + account_id="account-1", + ) @pytest.mark.parametrize("legacy_skill", [False, True]) @@ -493,8 +497,8 @@ def load_legacy_skill(*, tenant_id: str, file_id: str) -> bytes: return storage.files["tools/workspace.zip"] monkeypatch.setattr( - "services.skill_management_service.SkillManagementService._load_tool_file_bytes", - staticmethod(load_legacy_skill), + "services.skill_management_service.storage.load_once", + lambda key: load_legacy_skill(tenant_id="tenant-1", file_id=key), ) load_stream = storage.load_stream save = storage.save diff --git a/api/tests/unit_tests/services/app/test_agent_app_access_service.py b/api/tests/unit_tests/services/app/test_agent_app_access_service.py index 49167ca07cdd2a..fe70e80fe05eba 100644 --- a/api/tests/unit_tests/services/app/test_agent_app_access_service.py +++ b/api/tests/unit_tests/services/app/test_agent_app_access_service.py @@ -1,22 +1,18 @@ """Published Agent App references use the canonical roster query.""" +from unittest.mock import Mock + import pytest from sqlalchemy.orm import Session, sessionmaker +from enums.agent import WorkflowAgentBindingType from extensions.application_services.agent import build_agent_app_services from machinery.context import RequestContext -from models.agent import ( - Agent, - AgentKind, - AgentScope, - AgentSource, - AgentStatus, - WorkflowAgentBindingType, - WorkflowAgentNodeBinding, -) +from models.agent import Agent, AgentKind, AgentScope, AgentSource, AgentStatus, WorkflowAgentNodeBinding from models.agent_config_entities import WorkflowNodeJobConfig from models.model import AppMode from services.app.agent_app_contracts import AgentAppNotFoundError +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.unit_tests.model_factories import make_app @@ -62,7 +58,9 @@ def test_access_only_lists_current_published_workflow_references( ) ) sqlite_session.commit() - service = build_agent_app_services(database_client=sqlite_session_factory).access + service = build_agent_app_services( + database_client=sqlite_session_factory, variables=Mock(spec=WorkflowExecutionVariables) + ).access context = RequestContext("request", None, "account", "tenant-1") [reference] = service.list_referencing_workflows(context, "agent") assert reference["app_id"] == "workflow-app" @@ -97,6 +95,8 @@ def test_access_requires_public_agent_app( agent.app_id = "missing" sqlite_session.add_all([make_app(app_id="agent-app", mode=AppMode.AGENT), agent]) sqlite_session.commit() - service = build_agent_app_services(database_client=sqlite_session_factory).access + service = build_agent_app_services( + database_client=sqlite_session_factory, variables=Mock(spec=WorkflowExecutionVariables) + ).access with pytest.raises(AgentAppNotFoundError): service.list_referencing_workflows(RequestContext("request", None, "account", "tenant-1"), "agent") diff --git a/api/tests/unit_tests/services/app/test_agent_app_feature_service.py b/api/tests/unit_tests/services/app/test_agent_app_feature_service.py index 76d66264042280..5ee43a0e8ec017 100644 --- a/api/tests/unit_tests/services/app/test_agent_app_feature_service.py +++ b/api/tests/unit_tests/services/app/test_agent_app_feature_service.py @@ -7,6 +7,8 @@ touching model / prompt / agent_mode. """ +from unittest.mock import Mock + import pytest from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker @@ -18,6 +20,7 @@ from models.model import App, AppMode, AppModelConfig from services.app.agent_app_contracts import AgentAppNotFoundError from services.app.agent_app_feature_gateway import AgentAppFeatureValidator +from services.workflow.variable_contracts import WorkflowExecutionVariables TENANT_ID = "11111111-1111-1111-1111-111111111111" APP_ID = "22222222-2222-2222-2222-222222222222" @@ -109,7 +112,9 @@ def agent_app(sqlite_session: Session) -> App: def test_feature_save_is_atomic( agent_app: App, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session] ) -> None: - service = build_agent_app_services(database_client=sqlite_session_factory).features + service = build_agent_app_services( + database_client=sqlite_session_factory, variables=Mock(spec=WorkflowExecutionVariables) + ).features context = RequestContext("request", None, ACCOUNT_ID, TENANT_ID) calls: list[str] = [] @@ -143,7 +148,9 @@ def fail(_sender: App, **_kwargs: object) -> None: def test_feature_validation_failure_does_not_write( agent_app: App, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session] ) -> None: - service = build_agent_app_services(database_client=sqlite_session_factory).features + service = build_agent_app_services( + database_client=sqlite_session_factory, variables=Mock(spec=WorkflowExecutionVariables) + ).features with pytest.raises(ValueError, match="opening_statement"): service.update_features( RequestContext("request", None, ACCOUNT_ID, TENANT_ID), "agent", {"opening_statement": 123} @@ -170,7 +177,9 @@ def test_features_require_owned_active_agent_app( else: agent.status = AgentStatus.ARCHIVED sqlite_session.commit() - service = build_agent_app_services(database_client=sqlite_session_factory).features + service = build_agent_app_services( + database_client=sqlite_session_factory, variables=Mock(spec=WorkflowExecutionVariables) + ).features with pytest.raises(AgentAppNotFoundError): service.update_features(RequestContext("request", None, ACCOUNT_ID, TENANT_ID), "agent", {}) assert sqlite_session.scalar(select(AppModelConfig)) is None diff --git a/api/tests/unit_tests/services/app/test_agent_chat_attachments.py b/api/tests/unit_tests/services/app/test_agent_chat_attachments.py new file mode 100644 index 00000000000000..42d7133f92a648 --- /dev/null +++ b/api/tests/unit_tests/services/app/test_agent_chat_attachments.py @@ -0,0 +1,201 @@ +"""Agent Chat reads image metadata through its injected database before accessing storage.""" + +import contextvars +from dataclasses import dataclass +from uuid import uuid4 + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import QueuePool + +from core.app.app_config.entities import DatasetRetrieveConfigEntity +from core.app.entities.app_invoke_entities import ( + AgentChatAppGenerateEntity, + InvokeFrom, + ModelConfigWithCredentialsEntity, +) +from core.db import session_factory +from core.tools.__base.tool_runtime import ToolRuntime +from core.tools.entities.tool_entities import ToolInvokeMeta +from extensions.ext_database import db +from graphon.file import file_manager +from graphon.model_runtime.entities.model_entities import AIModelEntity, ModelFeature +from models.base import TypeBase +from models.model import App, Conversation, Message, MessageAgentThought, MessageFile, UploadFile +from repositories.app.generation_repository import AppGenerationRepository +from services.agent.chat.function_call_runner import FunctionCallAgentRunner +from services.app.generation.adapters import agent_chat as worker_module +from services.app.generation.adapters.agent_chat import AgentChatAppGenerator +from services.tools.dataset.tool import DatasetRetrieverTool +from services.tools.tool_engine import ToolEngine +from tests.unit_tests.core.agent.test_base_agent_runner import ( + _app_config, + _app_generate, + _conversation, + _message, + _tool_entity, +) +from tests.unit_tests.model_factories import make_app, make_upload_file + + +@dataclass +class LLM: + def get_model_schema(self, _name, _credentials): + return AIModelEntity.model_construct(features=[ModelFeature.VISION]) + + +@dataclass +class Model: + model_type_instance: LLM + model_name: str = "model" + credentials: dict | None = None + + +@dataclass +class Queue: + def publish_error(self, error, _source): + raise AssertionError("worker failed") from error + + +@pytest.fixture(params=[False, True], ids=["retain-after-commit", "expire-after-commit"]) +def agent_database(request, monkeypatch): + engine = create_engine("sqlite://", poolclass=QueuePool) + TypeBase.metadata.create_all( + engine, + tables=[ + model.__table__ for model in (App, Conversation, Message, MessageAgentThought, MessageFile, UploadFile) + ], + ) + sessions = sessionmaker(engine, expire_on_commit=request.param) + with sessions.begin() as session: + session.add_all([make_app(app_id="app1", tenant_id="tenant"), _conversation(), _message()]) + + def forbidden(*_args, **_kwargs): + pytest.fail("Agent Chat accessed a global database") + + monkeypatch.setattr(session_factory, "create_session", forbidden) + monkeypatch.setattr(session_factory, "get_session_maker", forbidden) + monkeypatch.setattr(db, "session", forbidden) + monkeypatch.setattr(type(db), "engine", property(forbidden)) + try: + yield sessions, AppGenerationRepository(sessions), engine + finally: + engine.dispose() + + +def test_dataset_images_are_owned_ordered_and_materialized_before_storage(agent_database, monkeypatch): + sessions, records, engine = agent_database + ids = [str(uuid4()) for _ in range(5)] + with sessions.begin() as session: + session.add_all( + [ + make_upload_file( + file_id=ids[0], + tenant_id="tenant", + name="first.png", + extension="png", + mime_type="image/png", + key="first", + ), + make_upload_file( + file_id=ids[1], + tenant_id="tenant", + name="second.png", + extension="png", + mime_type="image/png", + key="second", + ), + make_upload_file( + file_id=ids[2], tenant_id="foreign", extension="png", mime_type="image/png", key="foreign" + ), + make_upload_file(file_id=ids[3], tenant_id="tenant", mime_type="text/plain"), + ] + ) + config = _app_config() + runner = FunctionCallAgentRunner( + records=records, + dataset_tools=lambda **_kwargs: [], + tenant_id="tenant", + application_generate_entity=_app_generate(app_config=config), + conversation=_conversation(), + app_config=config, + model_config=ModelConfigWithCredentialsEntity.model_construct(), + config=None, + queue_manager=Queue(), + message=_message(), + user_id="user", + model_instance=Model(LLM()), + ) + reads = [] + + class FileRuntime: + multimodal_send_format = "base64" + + def load_file_bytes(self, *, file): + assert engine.pool.checkedout() == 0 + reads.append(file.filename) + return b"image bytes" + + monkeypatch.setattr(file_manager, "get_workflow_file_runtime", FileRuntime) + + class Retrieval: + def retrieve_dataset(self, *, query, **_kwargs): + assert engine.pool.checkedout() == 0 + return query + + tool = DatasetRetrieverTool( + entity=_tool_entity("dataset"), + runtime=ToolRuntime(tenant_id="tenant"), + retrieval=Retrieval(), + dataset_id="dataset", + config=DatasetRetrieveConfigEntity(retrieve_strategy="single"), + top_k=2, + inputs={}, + invoke_from=InvokeFrom.DEBUGGER, + return_resource=False, + hit_callback=None, + app_id="app1", + ) + response = "\n".join(f"![image](/files/{id}/file-preview)" for id in [ids[1], ids[0], *ids]) + text, files, meta = ToolEngine.agent_invoke( + records=records, + tool=tool, + tool_parameters={"query": response}, + user_id="user", + tenant_id="tenant", + message=_message(), + invoke_from=InvokeFrom.DEBUGGER, + agent_tool_callback=runner.agent_callback, + ) + assert files == [] + assert isinstance(meta, ToolInvokeMeta) + assert meta.error is None + contents = runner._build_dataset_tool_image_contents(text, tool) + assert reads == ["second.png", "first.png"] + assert all(content.base64_data == "aW1hZ2UgYnl0ZXM=" for content in contents) + assert engine.pool.checkedout() == 0 + + +def test_worker_passes_records_and_dataset_factory_without_opening_session( + agent_database, monkeypatch, app, annotation_replies +): + _, records, engine = agent_database + calls = [] + + def datasets(**_kwargs): + return [] + + class Runner(worker_module.AgentChatAppRunner): + def run(self, *, application_generate_entity, **kwargs): + assert self._records is records + assert self._dataset_tools is datasets + assert "session" not in kwargs + assert engine.pool.checkedout() == 0 + calls.append(application_generate_entity.task_id) + + monkeypatch.setattr(worker_module, "AgentChatAppRunner", Runner) + generator = AgentChatAppGenerator(dataset_tools=datasets, records=records, annotations=annotation_replies) + entity = AgentChatAppGenerateEntity.model_construct(task_id="task", app_config=_app_config()) + generator._generate_worker(app, contextvars.copy_context(), entity, Queue(), "conv1", "msg_current") + assert calls == ["task"] diff --git a/api/tests/unit_tests/services/app/test_console_gateway.py b/api/tests/unit_tests/services/app/test_console_gateway.py index a06055246e5fc3..adcf908bc761e4 100644 --- a/api/tests/unit_tests/services/app/test_console_gateway.py +++ b/api/tests/unit_tests/services/app/test_console_gateway.py @@ -1,3 +1,5 @@ +from extensions.application_services.workflow import build_app_dsl_service + """Policy and existing DSL adapters retain their domain-specific behavior.""" from collections.abc import Callable, Generator, Iterator @@ -366,7 +368,7 @@ def fetch_plugins(_self: PluginInstaller, tenant_id: str, plugin_ids: list[str]) monkeypatch.setattr(PluginInstaller, "fetch_plugin_installation_by_ids", fetch_plugins) gateway = AppTransferGateway( session_factory=factory, - dsl_factory=AppDslService, + dsl_factory=build_app_dsl_service, packages=AppPackageService(), agent_packages=RosterAgentPackageExporter(), agent_importer=RosterAgentPackageImporter(), diff --git a/api/tests/unit_tests/services/app/test_console_service.py b/api/tests/unit_tests/services/app/test_console_service.py index 75eb694c9e85ef..b1d2224ea35aa5 100644 --- a/api/tests/unit_tests/services/app/test_console_service.py +++ b/api/tests/unit_tests/services/app/test_console_service.py @@ -15,7 +15,6 @@ from core.plugin.impl.plugin import PluginInstaller from core.tools.entities.tool_entities import ApiProviderSchemaType -from core.tools.tool_manager import ToolManager from events.app_event import app_was_updated from graphon.model_runtime.entities.model_entities import ModelType from machinery.context import RequestContext @@ -63,6 +62,7 @@ ImportStatus, ) from services.model_provider.service import ModelProviderService +from services.tools.tool_manager import ToolManager CONTEXT = RequestContext("request", "trace", "actor", "workspace") RECORD = AppRecord(id="app", name="Example", mode_compatible_with_agent="chat") @@ -473,6 +473,9 @@ def test_lifecycle_agent_creation_seeds_workspace_default_model( monkeypatch: pytest.MonkeyPatch, provider_name: str | None, expected_model: tuple[str, str, str] | None, + *, + tool_providers, + workflow_queries, ) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) with factory.begin() as session: @@ -508,7 +511,9 @@ def test_lifecycle_agent_creation_seeds_workspace_default_model( ) context = RequestContext("request", None, account.id, tenant.id) - app = AppLifecycleGateway(session_factory=factory).create( + app = AppLifecycleGateway( + session_factory=factory, tool_providers=tool_providers, workflow_queries=workflow_queries + ).create( context, CreateAppParams(name="Agent", mode=AppMode.AGENT.value), AppCreationSettings({"enable_site": False, "enable_api": False}, None), @@ -532,7 +537,7 @@ def test_lifecycle_agent_creation_seeds_workspace_default_model( @pytest.mark.parametrize("failure", [None, "persist", "external"]) @pytest.mark.parametrize("mode", [AppMode.WORKFLOW, AppMode.AGENT]) def test_lifecycle_runs_after_atomic_creation_and_session_close( - sqlite_engine: Engine, failure: str | None, mode: AppMode + sqlite_engine: Engine, failure: str | None, mode: AppMode, *, tool_providers, workflow_queries ) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) with factory.begin() as session: @@ -620,7 +625,9 @@ def deleted(self, context: RequestContext, deleted: AppDeletion) -> None: transfers=Transfers(), creators=Creators(), tracing=Tracing(), - lifecycle=LifecycleProbe(session_factory=factory), + lifecycle=LifecycleProbe( + tool_providers=tool_providers, session_factory=factory, workflow_queries=workflow_queries + ), ) event.listen(sqlite_engine, "checkout", checkout) event.listen(sqlite_engine, "checkin", checkin) @@ -653,7 +660,9 @@ def deleted(self, context: RequestContext, deleted: AppDeletion) -> None: event.remove(sqlite_engine, "checkin", checkin) -def test_app_deletion_failure_rolls_back_backing_agent_changes(sqlite_engine: Engine) -> None: +def test_app_deletion_failure_rolls_back_backing_agent_changes( + sqlite_engine: Engine, *, tool_providers, workflow_queries +) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) with factory.begin() as session: account = Account(name="Creator", email=f"{uuid4()}@example.com") @@ -668,7 +677,9 @@ def test_app_deletion_failure_rolls_back_backing_agent_changes(sqlite_engine: En ] ) context = RequestContext("request", None, account.id, tenant.id) - gateway = AppLifecycleGateway(session_factory=factory) + gateway = AppLifecycleGateway( + session_factory=factory, tool_providers=tool_providers, workflow_queries=workflow_queries + ) app = gateway.create( context, CreateAppParams(name="Agent", mode="agent"), @@ -696,7 +707,13 @@ def reject_commit(_session: Session) -> None: @pytest.mark.parametrize("operation", ["get", "update", "create", "copy"]) @pytest.mark.parametrize("remote_fails", [False, True]) def test_detail_tool_enrichment_releases_database_before_plugin_io( - sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch, operation: str, remote_fails: bool + sqlite_engine: Engine, + monkeypatch: pytest.MonkeyPatch, + operation: str, + remote_fails: bool, + *, + tool_providers, + workflow_queries, ) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) missing_api = str(uuid4()) @@ -813,7 +830,9 @@ def import_dsl( transfers=CopyTransfer(), creators=Creators(), tracing=Tracing(), - lifecycle=LifecycleGateway(session_factory=factory), + lifecycle=LifecycleGateway( + tool_providers=tool_providers, session_factory=factory, workflow_queries=workflow_queries + ), ) monkeypatch.setattr(ToolManager, "get_hardcoded_provider", hardcoded) monkeypatch.setattr(PluginInstaller, "check_tools_existence", check_plugins) @@ -860,7 +879,9 @@ def read_result() -> AppRecord: @pytest.mark.parametrize("operation", ["rename", "icon", "site"]) @pytest.mark.parametrize("commit_fails", [False, True]) -def test_app_mutations_publish_only_after_commit(sqlite_engine: Engine, operation: str, commit_fails: bool) -> None: +def test_app_mutations_publish_only_after_commit( + sqlite_engine: Engine, operation: str, commit_fails: bool, *, tool_providers, workflow_queries +) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) context = RequestContext("request", None, str(uuid4()), str(uuid4())) with factory.begin() as session: @@ -881,7 +902,9 @@ def test_app_mutations_publish_only_after_commit(sqlite_engine: Engine, operatio transfers=Transfers(), creators=Creators(), tracing=Tracing(), - lifecycle=AppLifecycleGateway(session_factory=factory), + lifecycle=AppLifecycleGateway( + session_factory=factory, tool_providers=tool_providers, workflow_queries=workflow_queries + ), ) signals: list[str] = [] checked_out: set[object] = set() @@ -946,7 +969,9 @@ def update() -> AppRecord: @pytest.mark.parametrize("surface", ["site", "api"]) -def test_unpublished_agent_cannot_enable_access_through_console_service(sqlite_engine: Engine, surface: str) -> None: +def test_unpublished_agent_cannot_enable_access_through_console_service( + sqlite_engine: Engine, surface: str, *, tool_providers, workflow_queries +) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False) context = RequestContext("request", None, str(uuid4()), str(uuid4())) with factory.begin() as session: @@ -972,7 +997,9 @@ def test_unpublished_agent_cannot_enable_access_through_console_service(sqlite_e transfers=Transfers(), creators=Creators(), tracing=Tracing(), - lifecycle=AppLifecycleGateway(session_factory=factory), + lifecycle=AppLifecycleGateway( + session_factory=factory, tool_providers=tool_providers, workflow_queries=workflow_queries + ), ) signals: list[object] = [] diff --git a/api/tests/unit_tests/services/app/test_response_gateway.py b/api/tests/unit_tests/services/app/test_response_gateway.py index 95512bced132fa..b7ebc0638df2c9 100644 --- a/api/tests/unit_tests/services/app/test_response_gateway.py +++ b/api/tests/unit_tests/services/app/test_response_gateway.py @@ -6,7 +6,7 @@ from services.entities.app_entities import AppRecord -def test_mask_record_omits_unverified_tool_parameters_without_mutating_record() -> None: +def test_mask_record_omits_unverified_tool_parameters_without_mutating_record(tool_providers, workflow_queries) -> None: agent_mode = { "enabled": True, "tools": [ @@ -26,7 +26,9 @@ def test_mask_record_omits_unverified_tool_parameters_without_mutating_record() ) context = RequestContext("request", None, "account", "tenant") - masked = AppResponseGateway.mask_record(context, app) + masked = AppResponseGateway(tool_providers=tool_providers, workflow_queries=workflow_queries).mask_record( + context, app + ) assert masked is not app assert masked.app_model_config is not None diff --git a/api/tests/unit_tests/services/data_migration/test_export_service.py b/api/tests/unit_tests/services/data_migration/test_export_service.py index faad5d2052114c..a29bccc24ffcbf 100644 --- a/api/tests/unit_tests/services/data_migration/test_export_service.py +++ b/api/tests/unit_tests/services/data_migration/test_export_service.py @@ -98,10 +98,13 @@ def test_export_config_parser_rejects_unsupported_app_modes(): ) -def test_secret_free_api_tool_export_uses_masking_and_omits_credentials(monkeypatch: pytest.MonkeyPatch): +def test_secret_free_api_tool_export_uses_masking_and_omits_credentials( + monkeypatch: pytest.MonkeyPatch, *, workflow_tools, tool_providers +): calls = [] - def fake_get_api_provider(provider: str, tenant_id: str, mask: bool = True): + def fake_get_api_provider(provider: str, tenant_id: str, mask: bool = True, *, tool_providers): + assert tool_providers is service._tool_providers calls.append((provider, tenant_id, mask)) return {"credentials": {"api_key": "masked"}, "schema": {"openapi": "3.0.0"}, "tools": ["unused"]} @@ -109,7 +112,7 @@ def fake_get_api_provider(provider: str, tenant_id: str, mask: bool = True): "services.data_migration.export_service.ToolManager.user_get_api_provider", fake_get_api_provider, ) - service = MigrationExportService() + service = MigrationExportService(workflow_tools=workflow_tools, tool_providers=tool_providers) tools: list[dict] = [] report_items = [] @@ -127,8 +130,8 @@ def fake_get_api_provider(provider: str, tenant_id: str, mask: bool = True): @pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_secret_free_mcp_dependencies_are_dependency_only(sqlite_session: Session): - service = MigrationExportService() +def test_secret_free_mcp_dependencies_are_dependency_only(sqlite_session: Session, *, workflow_tools, tool_providers): + service = MigrationExportService(workflow_tools=workflow_tools, tool_providers=tool_providers) dependencies: list[dict] = [] mcp_tools: list[dict] = [] report_items = [] @@ -158,7 +161,9 @@ def test_secret_free_mcp_dependencies_are_dependency_only(sqlite_session: Sessio @pytest.mark.parametrize("sqlite_session", [(MCPToolProvider,)], indirect=True) -def test_get_mcp_provider_does_not_compare_non_uuid_identifier_to_uuid_id(sqlite_session: Session): +def test_get_mcp_provider_does_not_compare_non_uuid_identifier_to_uuid_id( + sqlite_session: Session, *, workflow_tools, tool_providers +): sqlite_session.add( MCPToolProvider( name="Other tenant provider", @@ -184,7 +189,9 @@ def capture_statement(_conn, _cursor, statement, _parameters, _context, _execute try: with pytest.raises(MigrationDataError, match="MCP provider not found"): - MigrationExportService()._get_mcp_provider(_TENANT_ID, "my-test-mcp", session=sqlite_session) + MigrationExportService(workflow_tools=workflow_tools, tool_providers=tool_providers)._get_mcp_provider( + _TENANT_ID, "my-test-mcp", session=sqlite_session + ) finally: event.remove(bind, "before_cursor_execute", capture_statement) @@ -193,8 +200,8 @@ def capture_statement(_conn, _cursor, statement, _parameters, _context, _execute assert "tool_mcp_providers.server_identifier =" in statements[0] -def test_dependency_ids_are_deduplicated_with_manual_selection_first(): - service = MigrationExportService() +def test_dependency_ids_are_deduplicated_with_manual_selection_first(*, workflow_tools, tool_providers): + service = MigrationExportService(workflow_tools=workflow_tools, tool_providers=tool_providers) provider_ids = service._provider_ids( manual_provider_ids=["weather", "weather", "manual"], discovered_dependencies=[ @@ -208,8 +215,8 @@ def test_dependency_ids_are_deduplicated_with_manual_selection_first(): assert provider_ids == ["weather", "manual", "forecast"] -def test_api_provider_ids_use_provider_name_from_discovered_dependencies(): - service = MigrationExportService() +def test_api_provider_ids_use_provider_name_from_discovered_dependencies(*, workflow_tools, tool_providers): + service = MigrationExportService(workflow_tools=workflow_tools, tool_providers=tool_providers) provider_ids = service._provider_ids( manual_provider_ids=[], discovered_dependencies=[ @@ -221,8 +228,8 @@ def test_api_provider_ids_use_provider_name_from_discovered_dependencies(): assert provider_ids == ["weather"] -def test_mcp_authentication_export_omits_runtime_header_shape(): - service = MigrationExportService() +def test_mcp_authentication_export_omits_runtime_header_shape(*, workflow_tools, tool_providers): + service = MigrationExportService(workflow_tools=workflow_tools, tool_providers=tool_providers) assert service._serialize_mcp_authentication({"Authorization": "Bearer token"}) is None assert service._serialize_mcp_authentication({"client_id": "id", "client_secret": "secret"}) == { diff --git a/api/tests/unit_tests/services/data_migration/test_import_service.py b/api/tests/unit_tests/services/data_migration/test_import_service.py index 512d2f10e666aa..550ebd780f9619 100644 --- a/api/tests/unit_tests/services/data_migration/test_import_service.py +++ b/api/tests/unit_tests/services/data_migration/test_import_service.py @@ -1,6 +1,4 @@ -from contextlib import nullcontext from dataclasses import dataclass -from types import SimpleNamespace from typing import cast from unittest.mock import Mock @@ -26,23 +24,27 @@ ResourceReportItem, ResourceType, ) -from services.data_migration.import_service import ImportRequest, ImportTargetResolver, MigrationImportService +from services.data_migration.import_service import ( + ImportRequest, + ImportTargetResolver, + MigrationImportService, + WorkflowPublisher, +) from services.entities.dsl_entities import ImportStatus from tests.unit_tests.config_override import apply_config_overrides @dataclass(frozen=True) class Database: - """Typed database binding used by import code that still reads ``db.engine``.""" + """Explicit SQLite binding shared by migration tests.""" engine: Engine session: Session @pytest.fixture -def database(sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> Database: +def database(sqlite_session: Session) -> Database: database = Database(engine=cast(Engine, sqlite_session.get_bind()), session=sqlite_session) - monkeypatch.setattr(import_service, "db", database) return database @@ -236,7 +238,7 @@ def test_package_target_tenant_id_ignores_invalid_uuid(database: Database): assert database.session.query(Tenant).count() == 0 -def test_options_override_replaces_package_defaults(database: Database): +def test_options_override_replaces_package_defaults(database: Database, *, workflow_tools): package = MigrationPackage.from_mapping( { "metadata": { @@ -275,7 +277,12 @@ def _import_workflows( override = ImportOptions(create_app_api_token_on_import=False, conflict_strategy=ConflictStrategy.SKIP) - CapturingImportService(target_resolver=StubResolver()).import_package( + CapturingImportService( + app_dsl=Mock(), + workflows=Mock(spec=WorkflowPublisher), + target_resolver=StubResolver(), + workflow_tools=workflow_tools, + ).import_package( ImportRequest(package=package, options_override=override), session=database.session, ) @@ -283,19 +290,26 @@ def _import_workflows( assert captured_options == [override] -def test_only_preserve_id_strategy_reuses_source_app_id(): - service = MigrationImportService() +def test_only_preserve_id_strategy_reuses_source_app_id(*, workflow_tools): + service = MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + ) assert service._should_preserve_source_app_id(ImportOptions(id_strategy=IdStrategy.PRESERVE_ID)) is True assert service._should_preserve_source_app_id(ImportOptions(id_strategy=IdStrategy.GENERATE_NEW_ID)) is False -def test_find_existing_app_ignores_invalid_uuid(database: Database): +def test_find_existing_app_ignores_invalid_uuid(database: Database, *, workflow_tools): _persist_app(database.session, app_id="app-other", tenant_id="tenant-2") - assert MigrationImportService()._find_existing_app("not-a-uuid", "tenant-1", session=database.session) is None + assert ( + MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._find_existing_app("not-a-uuid", "tenant-1", session=database.session) + is None + ) -def test_find_existing_workflow_tool_does_not_compare_invalid_uuid(database: Database): +def test_find_existing_workflow_tool_does_not_compare_invalid_uuid(database: Database, *, workflow_tools): provider = _persist_workflow_provider(database.session, provider_id="provider-1", app_id="app-id", name="tool-name") statements = [] @@ -304,9 +318,9 @@ def capture_statement(orm_execute_state) -> None: event.listen(database.session, "do_orm_execute", capture_statement) try: - result = MigrationImportService()._find_existing_workflow_tool( - "tenant-1", "not-a-uuid", "tool-name", "app-id", session=database.session - ) + result = MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._find_existing_workflow_tool("tenant-1", "not-a-uuid", "tool-name", "app-id", session=database.session) finally: event.remove(database.session, "do_orm_execute", capture_statement) @@ -315,7 +329,7 @@ def capture_statement(orm_execute_state) -> None: assert f"{WorkflowToolProvider.__tablename__}.id" not in where_clause -def test_find_existing_mcp_tool_does_not_compare_invalid_uuid(database: Database): +def test_find_existing_mcp_tool_does_not_compare_invalid_uuid(database: Database, *, workflow_tools): provider = _persist_mcp_provider(database.session, provider_id="provider-1") statements = [] @@ -324,9 +338,9 @@ def capture_statement(orm_execute_state) -> None: event.listen(database.session, "do_orm_execute", capture_statement) try: - result = MigrationImportService()._find_existing_mcp_tool( - "tenant-1", "my-test-mcp", "my-test-mcp", session=database.session - ) + result = MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._find_existing_mcp_tool("tenant-1", "my-test-mcp", "my-test-mcp", session=database.session) finally: event.remove(database.session, "do_orm_execute", capture_statement) @@ -337,22 +351,24 @@ def capture_statement(orm_execute_state) -> None: def test_workflow_app_import_closes_read_transaction_before_dsl_overwrite( - monkeypatch: pytest.MonkeyPatch, database: Database + monkeypatch: pytest.MonkeyPatch, database: Database, *, workflow_tools ): class StubAppDslService: def __init__(self, session): + assert session is database.session self.session = session def import_app(self, **kwargs): assert not self.session.in_transaction() return Import(id="import-id", status=ImportStatus.COMPLETED, app_id="imported-app-id") - monkeypatch.setattr(import_service, "AppDslService", StubAppDslService) apply_config_overrides(monkeypatch, RBAC_ENABLED=True) existing_app = _persist_app(database.session, app_id="11111111-1111-4111-8111-111111111111") database.session.begin() - imported_app_id = MigrationImportService()._import_workflow_app( + imported_app_id = MigrationImportService( + app_dsl=StubAppDslService, workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._import_workflow_app( account=object(), workflow_data={"name": "main_chatflow"}, dsl_content="app:\n mode: workflow\n", @@ -365,7 +381,7 @@ def import_app(self, **kwargs): assert imported_app_id == "imported-app-id" -def test_rewrite_workflow_dsl_replaces_tool_provider_ids(): +def test_rewrite_workflow_dsl_replaces_tool_provider_ids(*, workflow_tools): dsl_content = yaml.safe_dump( { "app": {"mode": "workflow"}, @@ -400,7 +416,9 @@ def test_rewrite_workflow_dsl_replaces_tool_provider_ids(): } ) - rewritten = MigrationImportService()._rewrite_workflow_dsl_provider_ids( + rewritten = MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._rewrite_workflow_dsl_provider_ids( dsl_content, { "source-api-provider-id": "target-api-provider-id", @@ -415,7 +433,7 @@ def test_rewrite_workflow_dsl_replaces_tool_provider_ids(): ) -def test_source_api_provider_ids_are_discovered_from_workflow_dsl(): +def test_source_api_provider_ids_are_discovered_from_workflow_dsl(*, workflow_tools): package = MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -444,11 +462,13 @@ def test_source_api_provider_ids_are_discovered_from_workflow_dsl(): } ) - assert MigrationImportService()._source_api_provider_ids_by_name(package) == {"weather": {"source-api-provider-id"}} + assert MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._source_api_provider_ids_by_name(package) == {"weather": {"source-api-provider-id"}} def test_workflow_tool_import_publishes_referenced_app_before_create( - monkeypatch: pytest.MonkeyPatch, database: Database + monkeypatch: pytest.MonkeyPatch, database: Database, *, workflow_tools ): events = [] _persist_tenant_account(database.session) @@ -467,8 +487,10 @@ def create_workflow_tool(**kwargs) -> None: app_id=kwargs["workflow_app_id"], ) - monkeypatch.setattr(import_service.WorkflowToolManageService, "create_workflow_tool", create_workflow_tool) - PublishingImportService()._import_workflow_tools( + monkeypatch.setattr(workflow_tools, "create_workflow_tool", create_workflow_tool) + PublishingImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._import_workflow_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -486,33 +508,29 @@ def create_workflow_tool(**kwargs) -> None: assert events == [("published", app_id), ("created", app_id)] -def test_ensure_workflow_app_is_published_updates_current_workflow( - monkeypatch: pytest.MonkeyPatch, - database: Database, -) -> None: +def test_ensure_workflow_app_is_published_updates_current_workflow(database: Database, *, workflow_tools) -> None: _, account = _persist_tenant_account(database.session) app_id = "00000000-0000-0000-0000-000000000001" _persist_app(database.session, app_id=app_id) - publish = Mock(return_value=SimpleNamespace(id="published-workflow")) - monkeypatch.setattr(import_service, "WorkflowService", Mock(return_value=SimpleNamespace(publish_workflow=publish))) - monkeypatch.setattr( - import_service, - "sessionmaker", - lambda _engine: SimpleNamespace(begin=lambda: nullcontext(database.session)), - ) - MigrationImportService()._ensure_workflow_app_is_published( + publisher = Mock(spec=WorkflowPublisher) + MigrationImportService( + app_dsl=Mock(), workflows=publisher, workflow_tools=workflow_tools + )._ensure_workflow_app_is_published( ImportTarget("tenant-1", "target", "account-1", "owner@example.com"), account, app_id, session=database.session, ) - - assert database.session.get(App, app_id).workflow_id == "published-workflow" + publisher.publish.assert_called_once() + context, published_app_id = publisher.publish.call_args.args + assert context.account_id == account.id + assert context.active_workspace_id == "tenant-1" + assert published_app_id == app_id @pytest.mark.parametrize("id_strategy", [IdStrategy.PRESERVE_ID, IdStrategy.GENERATE_NEW_ID]) def test_workflow_tool_import_id_follows_id_strategy( - monkeypatch: pytest.MonkeyPatch, database: Database, id_strategy: IdStrategy + monkeypatch: pytest.MonkeyPatch, database: Database, id_strategy: IdStrategy, *, workflow_tools ): created_kwargs = [] _persist_tenant_account(database.session) @@ -536,8 +554,10 @@ def create_workflow_tool(**kwargs) -> None: app_id=kwargs["workflow_app_id"], ) - monkeypatch.setattr(import_service.WorkflowToolManageService, "create_workflow_tool", create_workflow_tool) - StrategyImportService()._import_workflow_tools( + monkeypatch.setattr(workflow_tools, "create_workflow_tool", create_workflow_tool) + StrategyImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._import_workflow_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -563,7 +583,7 @@ def create_workflow_tool(**kwargs) -> None: ] -def test_workflow_tool_skip_records_id_mapping(database: Database): +def test_workflow_tool_skip_records_id_mapping(database: Database, *, workflow_tools): _persist_tenant_account(database.session) source_app_id = "00000000-0000-0000-0000-000000000031" target_app_id = "00000000-0000-0000-0000-000000000032" @@ -578,7 +598,9 @@ class SkipImportService(MigrationImportService): def _ensure_workflow_app_is_published(self, target, account, app_id, session): return None - SkipImportService()._import_workflow_tools( + SkipImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._import_workflow_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -600,7 +622,7 @@ def _ensure_workflow_app_is_published(self, target, account, app_id, session): @pytest.mark.parametrize("conflict_strategy", [ConflictStrategy.SKIP, ConflictStrategy.UPDATE]) def test_api_tool_existing_provider_records_id_mapping( - monkeypatch: pytest.MonkeyPatch, database: Database, conflict_strategy: ConflictStrategy + monkeypatch: pytest.MonkeyPatch, database: Database, conflict_strategy: ConflictStrategy, *, workflow_tools ): target_provider = _persist_api_provider(database.session, provider_id="target-api-provider-id") _persist_api_provider(database.session, provider_id="other-tenant-provider-id", tenant_id="tenant-2") @@ -613,7 +635,9 @@ def test_api_tool_existing_provider_records_id_mapping( ) monkeypatch.setattr(import_service.ApiToolManageService, "update_api_tool_provider", lambda **kwargs: None) - MigrationImportService()._import_api_tools( + MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._import_api_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -644,7 +668,7 @@ def test_api_tool_existing_provider_records_id_mapping( ) -def test_api_tool_create_records_id_mapping(monkeypatch: pytest.MonkeyPatch, database: Database): +def test_api_tool_create_records_id_mapping(monkeypatch: pytest.MonkeyPatch, database: Database, *, workflow_tools): id_mapping = {} monkeypatch.setattr( @@ -656,7 +680,9 @@ def create_api_tool_provider(**_kwargs) -> None: monkeypatch.setattr(import_service.ApiToolManageService, "create_api_tool_provider", create_api_tool_provider) - MigrationImportService()._import_api_tools( + MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._import_api_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -680,7 +706,9 @@ def create_api_tool_provider(**_kwargs) -> None: assert id_mapping["source-api-provider-id"] == "target-api-provider-id" -def test_mcp_tool_import_restores_exported_tool_list(monkeypatch: pytest.MonkeyPatch, database: Database): +def test_mcp_tool_import_restores_exported_tool_list( + monkeypatch: pytest.MonkeyPatch, database: Database, *, workflow_tools +): provider = _persist_mcp_provider(database.session, provider_id="target-provider-id") report_items = [] @@ -693,7 +721,9 @@ def update_provider(self, **kwargs): monkeypatch.setattr(import_service, "MCPToolManageService", StubMCPToolManageService) - MigrationImportService()._import_mcp_tools( + MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._import_mcp_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -729,7 +759,7 @@ def update_provider(self, **kwargs): @pytest.mark.parametrize("conflict_strategy", [ConflictStrategy.SKIP, ConflictStrategy.UPDATE]) def test_mcp_tool_existing_provider_records_id_mapping( - monkeypatch: pytest.MonkeyPatch, database: Database, conflict_strategy: ConflictStrategy + monkeypatch: pytest.MonkeyPatch, database: Database, conflict_strategy: ConflictStrategy, *, workflow_tools ): provider = _persist_mcp_provider(database.session, provider_id="target-mcp-provider-id") _persist_mcp_provider( @@ -751,7 +781,9 @@ def update_provider(self, **kwargs): monkeypatch.setattr(import_service, "MCPToolManageService", StubMCPToolManageService) - MigrationImportService()._import_mcp_tools( + MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._import_mcp_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -787,7 +819,7 @@ def update_provider(self, **kwargs): ] -def test_mcp_tool_create_records_id_mapping(monkeypatch: pytest.MonkeyPatch, database: Database): +def test_mcp_tool_create_records_id_mapping(monkeypatch: pytest.MonkeyPatch, database: Database, *, workflow_tools): id_mapping = {} class StubMCPToolManageService: @@ -805,7 +837,9 @@ def create_provider(self, **kwargs): monkeypatch.setattr(import_service, "MCPToolManageService", StubMCPToolManageService) - MigrationImportService()._import_mcp_tools( + MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._import_mcp_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -836,7 +870,9 @@ def create_provider(self, **kwargs): assert id_mapping["source-mcp-provider-id"] == "target-mcp-provider-id" -def test_dependency_only_mcp_preflight_reports_missing_target_provider_with_workflow_context(database: Database): +def test_dependency_only_mcp_preflight_reports_missing_target_provider_with_workflow_context( + database: Database, *, workflow_tools +): report_items = [] package = MigrationPackage.from_mapping( { @@ -875,7 +911,9 @@ def test_dependency_only_mcp_preflight_reports_missing_target_provider_with_work } ) - MigrationImportService()._preflight_dependency_only_mcp( + MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._preflight_dependency_only_mcp( package, ImportTarget( tenant_id="tenant-1", @@ -899,7 +937,9 @@ def test_dependency_only_mcp_preflight_reports_missing_target_provider_with_work ] -def test_dependency_only_mcp_lookup_does_not_compare_non_uuid_identifier_to_uuid_id(database: Database): +def test_dependency_only_mcp_lookup_does_not_compare_non_uuid_identifier_to_uuid_id( + database: Database, *, workflow_tools +): provider = _persist_mcp_provider(database.session, provider_id="provider-1", server_identifier="my-test-mcp-server") statements = [] @@ -908,7 +948,9 @@ def capture_statement(orm_execute_state) -> None: event.listen(database.session, "do_orm_execute", capture_statement) try: - result = MigrationImportService()._find_dependency_only_mcp_provider( + result = MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._find_dependency_only_mcp_provider( "tenant-1", "my-test-mcp-server", "my-test-mcp", @@ -922,7 +964,7 @@ def capture_statement(orm_execute_state) -> None: assert f"{MCPToolProvider.__tablename__}.id" not in where_clause -def test_dependency_only_mcp_preflight_reports_available_target_provider(database: Database): +def test_dependency_only_mcp_preflight_reports_available_target_provider(database: Database, *, workflow_tools): report_items = [] package = MigrationPackage.from_mapping( { @@ -943,7 +985,9 @@ def test_dependency_only_mcp_preflight_reports_available_target_provider(databas tenant_id="tenant-2", ) - MigrationImportService()._preflight_dependency_only_mcp( + MigrationImportService( + app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + )._preflight_dependency_only_mcp( package, ImportTarget( tenant_id="tenant-1", @@ -966,7 +1010,7 @@ def test_dependency_only_mcp_preflight_reports_available_target_provider(databas ] -def test_import_package_imports_workflow_tool_provider_apps_before_consumers(database: Database): +def test_import_package_imports_workflow_tool_provider_apps_before_consumers(database: Database, *, workflow_tools): events = [] class StubResolver(ImportTargetResolver): @@ -1041,9 +1085,12 @@ def _import_mcp_tools( } ) - OrderedImportService(target_resolver=StubResolver()).import_package( - ImportRequest(package=package), session=database.session - ) + OrderedImportService( + app_dsl=Mock(), + workflows=Mock(spec=WorkflowPublisher), + target_resolver=StubResolver(), + workflow_tools=workflow_tools, + ).import_package(ImportRequest(package=package), session=database.session) assert events == [ ("api_tools", "imported"), diff --git a/api/tests/unit_tests/services/dataset_service_test_helpers.py b/api/tests/unit_tests/services/dataset_service_test_helpers.py index 07364ea972c080..79ca3a97c2afc1 100644 --- a/api/tests/unit_tests/services/dataset_service_test_helpers.py +++ b/api/tests/unit_tests/services/dataset_service_test_helpers.py @@ -31,26 +31,19 @@ DocumentSegment, ) from models.model import UploadFile +from repositories.knowledge.collection_binding_repository import DatasetCollectionBindingRepository from services.entities.knowledge_entities.rag_pipeline_entities import ( IconInfo as PipelineIconInfo, ) from services.entities.knowledge_entities.rag_pipeline_entities import ( - KnowledgeConfiguration, RagPipelineDatasetCreateEntity, ) -from services.entities.knowledge_entities.rag_pipeline_entities import ( - RerankingModelConfig as RagPipelineRerankingModelConfig, -) -from services.entities.knowledge_entities.rag_pipeline_entities import ( - RetrievalSetting as RagPipelineRetrievalSetting, -) from services.errors.base import NoPermissionError from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingError from services.errors.dataset import DatasetNameDuplicateError from services.errors.document import DocumentIndexingError from services.errors.file import FileNotExistsError from services.knowledge.dataset_service import ( - DatasetCollectionBindingService, DatasetPermissionService, DatasetService, DocumentService, @@ -81,7 +74,7 @@ "CloudPlan", "DataSource", "Dataset", - "DatasetCollectionBindingService", + "DatasetCollectionBindingRepository", "DatasetNameDuplicateError", "DatasetPermissionEnum", "DatasetPermissionService", @@ -98,7 +91,6 @@ "IndexStructureType", "InfoList", "KnowledgeConfig", - "KnowledgeConfiguration", "LLMBadRequestError", "MagicMock", "Mock", @@ -114,8 +106,6 @@ "ProcessRule", "ProviderTokenNotInitError", "RagPipelineDatasetCreateEntity", - "RagPipelineRerankingModelConfig", - "RagPipelineRetrievalSetting", "RerankingModel", "RetrievalMethod", "RetrievalModel", @@ -130,7 +120,6 @@ "_make_dataset", "_make_document", "_make_features", - "_make_knowledge_configuration", "_make_lock_context", "_make_retrieval_model", "_make_segment", @@ -426,38 +415,3 @@ def _make_retrieval_model( top_k=4, score_threshold_enabled=False, ) - - -def _make_rag_pipeline_retrieval_setting() -> RagPipelineRetrievalSetting: - return RagPipelineRetrievalSetting( - search_method=RetrievalMethod.SEMANTIC_SEARCH, - top_k=4, - score_threshold=0.5, - score_threshold_enabled=True, - reranking_mode="reranking_model", - reranking_enable=True, - reranking_model=RagPipelineRerankingModelConfig( - reranking_provider_name="rerank-provider", - reranking_model_name="rerank-model", - ), - ) - - -def _make_knowledge_configuration( - *, - chunk_structure: str = "paragraph", - indexing_technique: str = "high_quality", - embedding_model_provider: str = "provider", - embedding_model: str = "embedding-model", - keyword_number: int = 8, - summary_index_setting: dict | None = None, -) -> KnowledgeConfiguration: - return KnowledgeConfiguration( - chunk_structure=chunk_structure, - indexing_technique=indexing_technique, - embedding_model_provider=embedding_model_provider, - embedding_model=embedding_model, - keyword_number=keyword_number, - retrieval_model=_make_rag_pipeline_retrieval_setting(), - summary_index_setting=summary_index_setting, - ) diff --git a/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py b/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py new file mode 100644 index 00000000000000..e6f618ee70f1bd --- /dev/null +++ b/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py @@ -0,0 +1,295 @@ +from repositories.knowledge.retrieval_repository import KnowledgeRetrievalRepository +from services.knowledge.retrieval.adapters.resource_events import DatasetIndexToolCallbackHandler +from services.tools.dataset.tool import DatasetRetrieverTool + +"""Chat and Completion release retrieval reads before model and HTTP calls.""" + +import json +from dataclasses import dataclass, field +from uuid import uuid4 + +import httpx +import pytest +from flask import Flask +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import QueuePool + +from core.app.app_config.entities import DatasetEntity, DatasetRetrieveConfigEntity, ModelConfig +from core.app.apps.chat.app_config_manager import ChatAppConfig +from core.app.apps.completion.app_config_manager import CompletionAppConfig +from core.app.entities.app_invoke_entities import ( + ChatAppGenerateEntity, + CompletionAppGenerateEntity, + InvokeFrom, + ModelConfigWithCredentialsEntity, +) +from core.db import session_factory +from core.model_manager import ModelManager +from extensions.application_services.retrieval import build_dataset_retrieval +from graphon.model_runtime.entities.llm_entities import LLMResultChunk, LLMResultChunkDelta, LLMUsage +from graphon.model_runtime.entities.message_entities import AssistantPromptMessage +from models.base import TypeBase +from models.dataset import ( + Dataset, + DatasetMetadata, + DatasetQuery, + Document, + ExternalKnowledgeApis, + ExternalKnowledgeBindings, +) +from models.model import App, AppMode, Conversation, Message +from repositories.app.generation_repository import AppGenerationRepository +from services.app.generation.adapters.chat_runner import ChatAppRunner +from services.app.generation.adapters.completion_runner import CompletionAppRunner +from services.knowledge.external.service import ExternalDatasetService +from services.knowledge.retrieval.dataset_retrieval import DatasetRetrieval +from tests.unit_tests.model_factories import make_dataset + + +@dataclass +class Queue: + events: list = field(default_factory=list) + + def publish(self, event, _source): + self.events.append(event) + + +class NoAnnotations: + def query(self, **_kwargs): + return None + + +@pytest.mark.parametrize("mode", [AppMode.CHAT, AppMode.COMPLETION, "agent-tool"]) +@pytest.mark.parametrize("strategy", ["single", "multiple"]) +def test_runner_releases_connections_during_metadata_and_external_retrieval(tmp_path, monkeypatch, mode, strategy): + engine = create_engine( + f"sqlite:///{tmp_path / 'retrieval.db'}", + poolclass=QueuePool, + pool_size=1, + max_overflow=0, + ) + sessions = sessionmaker(engine) # Production default: expire_on_commit=True. + TypeBase.metadata.create_all( + engine, + tables=[ + model.__table__ + for model in ( + Dataset, + Document, + DatasetMetadata, + DatasetQuery, + ExternalKnowledgeApis, + ExternalKnowledgeBindings, + ) + ], + ) + tenant_id, dataset_id, user_id, app_id = [str(uuid4()) for _ in range(4)] + with sessions.begin() as session: + dataset = make_dataset( + dataset_id=dataset_id, + tenant_id=tenant_id, + created_by=user_id, + provider="external", + indexing_technique="high_quality", + ) + dataset.retrieval_model = {"top_k": 3} + api = ExternalKnowledgeApis( + name="External", + description="", + tenant_id=tenant_id, + created_by=user_id, + settings=json.dumps({"endpoint": "https://knowledge.example", "api_key": "secret"}), + updated_by=None, + ) + session.add_all([dataset, api]) + session.flush() + session.add_all( + [ + ExternalKnowledgeBindings( + tenant_id=tenant_id, + dataset_id=dataset_id, + external_knowledge_api_id=api.id, + external_knowledge_id="collection", + created_by=user_id, + ), + DatasetMetadata( + tenant_id=tenant_id, dataset_id=dataset_id, name="category", type="string", created_by=user_id + ), + ] + ) + calls = [] + + def no_global_database(*_args, **_kwargs): + raise AssertionError("retrieval used the global database") + + monkeypatch.setattr(session_factory, "create_session", no_global_database) + + @dataclass + class Schema: + features: list = field(default_factory=list) + + class Model: + model_name = "model" + credentials = {} + provider_model_bundle = object() + + @property + def model_type_instance(self): + return self + + def get_model_schema(self, **_kwargs): + return Schema() + + def get_model_instance(self, **_kwargs): + return self + + def invoke_llm(self, **_kwargs): + assert engine.pool.checkedout() == 0 + calls.append("metadata-model") + yield LLMResultChunk( + model="model", + delta=LLMResultChunkDelta( + index=0, + message=AssistantPromptMessage( + content=json.dumps( + { + "metadata_map": [ + { + "metadata_field_name": "category", + "metadata_field_value": "docs", + "comparison_operator": "is", + } + ] + } + ) + ), + usage=LLMUsage.empty_usage(), + ), + ) + + model = Model() + model_config = ModelConfigWithCredentialsEntity.model_construct( + provider="provider", + model="model", + mode="chat", + provider_model_bundle=model.provider_model_bundle, + credentials={}, + parameters={}, + stop=[], + model_schema=Schema(), + ) + monkeypatch.setattr(ModelManager, "for_tenant", lambda **_kwargs: model) + monkeypatch.setattr(DatasetRetrieval, "_fetch_model_config", lambda *_args, **_kwargs: (model, model_config)) + + def metadata_prompt(_self, **kwargs): + assert kwargs["metadata_fields"] == ["category"] # A real database read happened before model I/O. + return [], [] + + monkeypatch.setattr(DatasetRetrieval, "_get_prompt_template", metadata_prompt) + + class Router: + def __init__(self, **_kwargs): + pass + + def invoke(self, *_args, **_kwargs): + assert engine.pool.checkedout() == 0 + calls.append("router") + return dataset_id, LLMUsage.empty_usage() + + monkeypatch.setattr("services.knowledge.retrieval.dataset_retrieval.ReactMultiDatasetRouter", Router) + + def external_request(request, _files): + assert engine.pool.checkedout() == 0 + assert request.url == "https://knowledge.example/retrieval" + assert request.headers["Authorization"] == "Bearer secret" + assert request.params["metadata_condition"]["conditions"][0]["value"] == "docs" + calls.append("external-knowledge") + return httpx.Response( + 200, json={"records": [{"content": "retrieved context", "title": "Docs", "score": 0.9, "metadata": {}}]} + ) + + monkeypatch.setattr(ExternalDatasetService, "process_external_api", external_request) + config_type = ChatAppConfig if mode == AppMode.CHAT else CompletionAppConfig + config = config_type.model_construct( + tenant_id=tenant_id, + app_id=app_id, + app_mode=mode, + app_model_config_dict={}, + prompt_template=None, + dataset=DatasetEntity( + dataset_ids=[dataset_id], + retrieve_config=DatasetRetrieveConfigEntity( + retrieve_strategy=strategy, + top_k=3, + reranking_enabled=False, + metadata_filtering_mode="automatic", + metadata_model_config=ModelConfig(provider="provider", name="model", mode="chat"), + ), + ), + ) + entity_type = ChatAppGenerateEntity if mode == AppMode.CHAT else CompletionAppGenerateEntity + entity = entity_type.model_construct( + task_id="task", + app_config=config, + model_conf=model_config, + inputs={}, + files=[], + user_id=user_id, + query="question", + invoke_from=InvokeFrom.DEBUGGER, + stream=False, + ) + runner_type = ChatAppRunner if mode == AppMode.CHAT else CompletionAppRunner + runner = runner_type(records=AppGenerationRepository(sessions), retrieval=build_dataset_retrieval(sessions)) + prompts = [] + + def organize_prompt(**kwargs): + prompts.append(kwargs.get("context")) + return [], [] + + monkeypatch.setattr(runner, "organize_prompt_messages", organize_prompt) + monkeypatch.setattr(runner, "moderation_for_inputs", lambda **_kwargs: (None, {}, "question")) + # Stop after retrieval and prompt preparation; the assertions cover all retrieval I/O. + monkeypatch.setattr(runner, "check_hosting_moderation", lambda **_kwargs: True) + queue = Queue() + app = App(id=app_id, tenant_id=tenant_id, mode=mode) + message = Message(id=str(uuid4())) + try: + with Flask(__name__).app_context(): + if mode == "agent-tool": + tools = DatasetRetrieverTool.get_dataset_tools( + records=KnowledgeRetrievalRepository(sessions), + retrieval=build_dataset_retrieval(sessions), + tenant_id=tenant_id, + app_id=app_id, + dataset_ids=[dataset_id], + retrieve_config=config.dataset.retrieve_config, + return_resource=True, + invoke_from=InvokeFrom.DEBUGGER, + hit_callback=DatasetIndexToolCallbackHandler(queue), + user_id=user_id, + inputs={}, + ) + prompts.append( + list(tools[0].invoke(user_id=user_id, tool_parameters={"query": "question"}))[0].message.text + ) + elif mode == AppMode.CHAT: + runner.run(entity, queue, Conversation(), message, app, NoAnnotations()) + else: + runner.run(entity, queue, message, app) + assert calls == [ + "metadata-model", + *(["router"] if strategy == "single" and mode != "agent-tool" else []), + "external-knowledge", + ] + assert prompts[-1] == "retrieved context" + assert len(queue.events) == 1 + with sessions() as session: + audit = session.scalar(select(DatasetQuery)) + assert audit is not None + assert audit.dataset_id == dataset_id + assert audit.source_app_id == app_id + assert engine.pool.checkedout() == 0 + finally: + engine.dispose() diff --git a/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py b/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py index 939b54b1559511..f3c0f5ec06c450 100644 --- a/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py +++ b/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py @@ -1,18 +1,13 @@ -from collections.abc import Callable - import pytest from pytest_mock import MockerFixture from sqlalchemy.orm import Session -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom -from core.rag.index_processor.constant.index_type import IndexStructureType -from models.dataset import Dataset, Document, Pipeline -from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus -from models.model import Account, App, AppMode, EndUser +from models.dataset import Dataset, Pipeline +from models.model import Account, EndUser from models.workflow import Workflow, WorkflowType -from services.knowledge.resource_scope import DatasetRef from services.rag_pipeline.pipeline_generate_service import PipelineGenerateService +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator @pytest.fixture @@ -32,17 +27,6 @@ def _make_account(*, account_id: str = "user-1") -> Account: return account -def _make_app(*, max_active_requests: int) -> App: - return App( - tenant_id="tenant-1", - name="Pipeline App", - mode=AppMode.RAG_PIPELINE, - enable_site=False, - enable_api=False, - max_active_requests=max_active_requests, - ) - - def _make_workflow(*, workflow_id: str = "wf-1") -> Workflow: return Workflow( id=workflow_id, @@ -66,74 +50,7 @@ def _make_dataset(*, dataset_id: str = "dataset-1", tenant_id: str = "tenant-1") ) -def _make_document( - *, document_id: str = "doc-1", dataset_id: str = "dataset-1", tenant_id: str = "tenant-1" -) -> Document: - return Document( - id=document_id, - tenant_id=tenant_id, - dataset_id=dataset_id, - position=1, - data_source_type=DataSourceType.LOCAL_FILE, - batch="batch", - name="Document", - created_from=DocumentCreatedFrom.API, - created_by="user-1", - indexing_status=IndexingStatus.COMPLETED, - doc_form=IndexStructureType.PARAGRAPH_INDEX, - ) - - -def test_get_max_active_requests_uses_smallest_non_zero_limit( - config_overrides: Callable[..., None], -) -> None: - config_overrides(APP_DEFAULT_ACTIVE_REQUESTS=5, APP_MAX_ACTIVE_REQUESTS=3) - - app_model = _make_app(max_active_requests=10) - - result = PipelineGenerateService._get_max_active_requests(app_model) - - assert result == 3 - - -def test_get_max_active_requests_returns_zero_when_all_unlimited( - config_overrides: Callable[..., None], -) -> None: - config_overrides(APP_DEFAULT_ACTIVE_REQUESTS=0, APP_MAX_ACTIVE_REQUESTS=0) - - app_model = _make_app(max_active_requests=0) - - result = PipelineGenerateService._get_max_active_requests(app_model) - - assert result == 0 - - -@pytest.mark.parametrize( - ("invoke_from", "workflow", "expected_error"), - [ - (InvokeFrom.DEBUGGER, None, "Workflow not initialized"), - (InvokeFrom.WEB_APP, None, "Workflow not published"), - (InvokeFrom.DEBUGGER, _make_workflow(), None), - ], -) -def test_get_workflow(mocker: MockerFixture, invoke_from, workflow, expected_error, sqlite_session: Session) -> None: - rag_pipeline_service_cls = mocker.patch("services.rag_pipeline.pipeline_generate_service.RagPipelineService") - rag_pipeline_service = rag_pipeline_service_cls.return_value - rag_pipeline_service.get_draft_workflow.return_value = workflow - rag_pipeline_service.get_published_workflow.return_value = workflow - - pipeline = _make_pipeline() - session = sqlite_session - - if expected_error: - with pytest.raises(ValueError, match=expected_error): - PipelineGenerateService._get_workflow(pipeline, invoke_from, session) - else: - result = PipelineGenerateService._get_workflow(pipeline, invoke_from, session) - assert result == workflow - - -def test_generate_updates_document_status_and_returns_event_stream( +def test_generate_delegates_original_document_and_returns_event_stream( mocker: MockerFixture, sqlite_session: Session, pipeline_generator ) -> None: dataset = _make_dataset() @@ -143,13 +60,13 @@ def test_generate_updates_document_status_and_returns_event_stream( user: Account | EndUser = _make_account() args = {"original_document_id": "doc-1", "query": "hello"} - mocker.patch.object(PipelineGenerateService, "_get_workflow", return_value=_make_workflow()) - update_status_mock = mocker.patch.object(PipelineGenerateService, "update_document_status") + pipeline_generator.load_workflow.return_value = _make_workflow() - generator_cls = mocker.patch("services.rag_pipeline.pipeline_generate_service.PipelineGenerator") generator_instance = pipeline_generator generator_instance.generate.return_value = "raw-events" - generator_cls.convert_to_event_stream.return_value = "stream-events" + mocker.patch( + "services.rag_pipeline.pipeline_generate_service.convert_to_event_stream", return_value="stream-events" + ) result = PipelineGenerateService.generate( pipeline=pipeline, @@ -157,156 +74,80 @@ def test_generate_updates_document_status_and_returns_event_stream( args=args, invoke_from=InvokeFrom.WEB_APP, streaming=True, - session=sqlite_session, generator=pipeline_generator, ) assert result == "stream-events" - document_ref = update_status_mock.call_args.args[0] - assert document_ref.dataset.tenant_id == "tenant-1" - assert document_ref.dataset.dataset_id == "dataset-1" - assert document_ref.document_id == "doc-1" - update_status_mock.assert_called_once_with(document_ref, session=sqlite_session) - assert generator_instance.generate.call_args.kwargs["session"] is sqlite_session - - -def test_generate_rejects_pipeline_dataset_from_another_tenant( - mocker: MockerFixture, sqlite_session: Session, pipeline_generator -) -> None: - dataset = _make_dataset(tenant_id="tenant-2") - pipeline = _make_pipeline() - sqlite_session.add_all([dataset, pipeline]) - sqlite_session.commit() - mocker.patch.object(PipelineGenerateService, "_get_workflow", return_value=_make_workflow()) - update_status_mock = mocker.patch.object(PipelineGenerateService, "update_document_status") - - with pytest.raises(ValueError, match="Pipeline dataset is required"): - PipelineGenerateService.generate( - pipeline=pipeline, - user=_make_account(), - args={"original_document_id": "doc-1"}, - invoke_from=InvokeFrom.WEB_APP, - session=sqlite_session, - generator=pipeline_generator, - ) - - update_status_mock.assert_not_called() - - -def test_generate_rejects_original_document_outside_pipeline_dataset_before_dispatch( - mocker: MockerFixture, - sqlite_session: Session, - pipeline_generator, -) -> None: - dataset = _make_dataset() - pipeline = _make_pipeline() - outside_document = _make_document(document_id="foreign-doc", dataset_id="other-dataset", tenant_id="tenant-2") - sqlite_session.add_all([dataset, pipeline, outside_document]) - sqlite_session.commit() - mocker.patch.object(PipelineGenerateService, "_get_workflow", return_value=_make_workflow()) - generator_cls = mocker.patch("services.rag_pipeline.pipeline_generate_service.PipelineGenerator") - - with pytest.raises(ValueError, match="Pipeline document not found"): - PipelineGenerateService.generate( - pipeline=pipeline, - user=_make_account(), - args={"original_document_id": "foreign-doc"}, - invoke_from=InvokeFrom.PUBLISHED_PIPELINE, - session=sqlite_session, - generator=pipeline_generator, - ) - - sqlite_session.refresh(outside_document) - assert outside_document.indexing_status == IndexingStatus.COMPLETED - pipeline_generator.generate.assert_not_called() - - -def test_update_document_status_updates_existing_document(sqlite_session: Session) -> None: - document = _make_document() - dataset = _make_dataset() - sqlite_session.add_all([dataset, document]) - sqlite_session.commit() - dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id) - document_ref = dataset_ref.document("doc-1") - - PipelineGenerateService.update_document_status(document_ref, session=sqlite_session) - - assert document.indexing_status == IndexingStatus.WAITING - - -@pytest.mark.parametrize( - ("document_tenant_id", "document_dataset_id"), - [ - pytest.param("other-tenant", "dataset-1", id="other-tenant"), - pytest.param("tenant-1", "other-dataset", id="other-dataset"), - ], -) -def test_update_document_status_rejects_document_outside_owner( - document_tenant_id: str, - document_dataset_id: str, - sqlite_session: Session, -) -> None: - dataset = _make_dataset() - dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id) - document_ref = dataset_ref.document("doc-1") - outside_document = _make_document(tenant_id=document_tenant_id, dataset_id=document_dataset_id) - sqlite_session.add(outside_document) - sqlite_session.commit() - - with pytest.raises(ValueError, match="Pipeline document not found"): - PipelineGenerateService.update_document_status(document_ref, session=sqlite_session) + assert generator_instance.generate.call_args.kwargs["args"] == args + assert "session" not in generator_instance.generate.call_args.kwargs - sqlite_session.refresh(outside_document) - assert outside_document.indexing_status == IndexingStatus.COMPLETED +def test_generate_single_iteration_delegates(mocker: MockerFixture, pipeline_generator) -> None: + pipeline_generator.load_workflow.return_value = _make_workflow() -# --- generate_single_iteration --- - - -def test_generate_single_iteration_delegates( - mocker: MockerFixture, sqlite_session: Session, pipeline_generator -) -> None: - mocker.patch.object(PipelineGenerateService, "_get_workflow", return_value=_make_workflow()) - - generator_cls = mocker.patch("services.rag_pipeline.pipeline_generate_service.PipelineGenerator") generator_instance = pipeline_generator generator_instance.single_iteration_generate.return_value = "raw-iter" - generator_cls.convert_to_event_stream.return_value = "stream-iter" + mocker.patch("services.rag_pipeline.pipeline_generate_service.convert_to_event_stream", return_value="stream-iter") pipeline = _make_pipeline() pipeline.id = "p1" user = _make_account(account_id="u1") - session = sqlite_session result = PipelineGenerateService.generate_single_iteration( - pipeline, user, "node-1", {"key": "val"}, session, generator=pipeline_generator + pipeline, user, "node-1", {"key": "val"}, generator=pipeline_generator ) assert result == "stream-iter" generator_instance.single_iteration_generate.assert_called_once() - assert generator_instance.single_iteration_generate.call_args.kwargs["session"] is session + assert "session" not in generator_instance.single_iteration_generate.call_args.kwargs # --- generate_single_loop --- -def test_generate_single_loop_delegates(mocker: MockerFixture, sqlite_session: Session, pipeline_generator) -> None: - mocker.patch.object(PipelineGenerateService, "_get_workflow", return_value=_make_workflow()) +def test_generate_single_loop_delegates(mocker: MockerFixture, pipeline_generator) -> None: + pipeline_generator.load_workflow.return_value = _make_workflow() - generator_cls = mocker.patch("services.rag_pipeline.pipeline_generate_service.PipelineGenerator") generator_instance = pipeline_generator generator_instance.single_loop_generate.return_value = "raw-loop" - generator_cls.convert_to_event_stream.return_value = "stream-loop" + mocker.patch("services.rag_pipeline.pipeline_generate_service.convert_to_event_stream", return_value="stream-loop") pipeline = _make_pipeline() pipeline.id = "p1" user = _make_account(account_id="u1") - session = sqlite_session result = PipelineGenerateService.generate_single_loop( - pipeline, user, "node-1", {"key": "val"}, session, generator=pipeline_generator + pipeline, user, "node-1", {"key": "val"}, generator=pipeline_generator ) assert result == "stream-loop" generator_instance.single_loop_generate.assert_called_once() - assert generator_instance.single_loop_generate.call_args.kwargs["session"] is session + assert "session" not in generator_instance.single_loop_generate.call_args.kwargs + + +@pytest.mark.parametrize("draft", [True, False]) +def test_pipeline_execution_definition_is_owned_and_detached_before_generation( + sqlite_session_factory, + workflow_application, + draft: bool, +) -> None: + from sqlalchemy import inspect + + from services.errors.app import WorkflowNotFoundError + + pipeline = _make_pipeline() + workflow = _make_workflow() + if not draft: + workflow.version = "published" + pipeline.workflow_id = workflow.id + with sqlite_session_factory.begin() as session: + session.add_all([pipeline, workflow]) + generator = workflow_application.knowledge.pipeline_generator + invocation = InvokeFrom.DEBUGGER if draft else InvokeFrom.PUBLISHED_PIPELINE + loaded = generator.load_workflow(pipeline, invocation) + assert loaded.id == workflow.id + assert inspect(loaded).detached + assert sqlite_session_factory.kw["bind"].pool.checkedout() == 0 + pipeline.tenant_id = "other-tenant" + with pytest.raises(WorkflowNotFoundError): + generator.load_workflow(pipeline, invocation) diff --git a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py index abfcf2e34378c3..6b1c79679994e0 100644 --- a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py +++ b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py @@ -3,18 +3,19 @@ from collections.abc import Callable from dataclasses import dataclass from datetime import datetime +from functools import partial from types import SimpleNamespace from unittest import mock from unittest.mock import create_autospec import pytest from pytest_mock import MockerFixture -from sqlalchemy.dialects import postgresql from sqlalchemy.orm import Session, sessionmaker -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom from core.rag.index_processor.constant.index_type import IndexStructureType +from core.repositories.factory import DifyCoreRepositoryFactory +from graphon.entities.workflow_node_execution import WorkflowNodeExecution from graphon.enums import ( BuiltinNodeTypes, ErrorStrategy, @@ -23,6 +24,7 @@ ) from graphon.graph_events import NodeRunFailedEvent from graphon.node_events.base import NodeRunResult +from graphon.variable_loader import VariableLoader from models import Account, Tenant from models.dataset import ( Dataset, @@ -34,13 +36,21 @@ ) from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus from models.workflow import Workflow, WorkflowRun +from repositories.factory import DifyAPIRepositoryFactory from repositories.knowledge.dataset_read_repository import get_pipeline_dataset +from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository +from repositories.workflow.definition_repository import WorkflowDefinitionRepository from services.credentials.query import CredentialQuery from services.data_source.credential_gateway import DatasourceProviderCredentialStore from services.entities.knowledge_entities.rag_pipeline_entities import IconInfo, PipelineTemplateInfoEntity from services.errors.rag_pipeline import RagPipelineResourceNotFoundError +from services.rag_pipeline import execution_service as execution_module from services.rag_pipeline import rag_pipeline as rag_pipeline_module +from services.rag_pipeline.execution_service import RagPipelineExecutionService from services.rag_pipeline.rag_pipeline import RagPipelineService +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables from services.workflow_ref_service import WorkflowRef @@ -52,6 +62,7 @@ def pipeline_generator(mocker: MockerFixture): @dataclass class RagPipelineServiceTestContext: service: RagPipelineService + execution: RagPipelineExecutionService session: Session session_maker: sessionmaker[Session] @@ -61,6 +72,9 @@ def rag_pipeline_service( mocker: MockerFixture, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], + workflow_variables: WorkflowExecutionVariables, + *, + workflow_runtime: WorkflowRuntime, ) -> RagPipelineServiceTestContext: mocker.patch( "services.rag_pipeline.rag_pipeline.DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository", @@ -70,10 +84,20 @@ def rag_pipeline_service( "services.rag_pipeline.rag_pipeline.DifyAPIRepositoryFactory.create_api_workflow_run_repository", return_value=MockRepo(), ) - mocker.patch("services.rag_pipeline.rag_pipeline.db", SimpleNamespace(engine=sqlite_session.get_bind())) service = RagPipelineService(session=sqlite_session, session_maker=sqlite_session_factory) return RagPipelineServiceTestContext( service=service, + execution=RagPipelineExecutionService( + workflows=WorkflowDefinitionRepository(session_factory=sqlite_session_factory), + variables=workflow_variables, + executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(sqlite_session_factory), + writer_factory=partial( + DifyCoreRepositoryFactory.create_workflow_node_execution_repository, + session_factory=sqlite_session_factory, + ), + documents=SQLAlchemyDocumentRepository(session_factory=sqlite_session_factory), + runtime=workflow_runtime, + ), session=sqlite_session, session_maker=sqlite_session_factory, ) @@ -83,22 +107,6 @@ class MockRepo: pass -def test_get_published_workflow_by_id_locks_restore_source(mocker: MockerFixture) -> None: - session = mocker.Mock(spec=Session) - workflow = _make_workflow() - workflow.version = "v1" - session.scalar.return_value = workflow - service = RagPipelineService.__new__(RagPipelineService) - service._session = session - - result = service.get_published_workflow_by_id(_make_pipeline(), workflow.id) - - stmt = session.scalar.call_args.args[0] - sql = str(stmt.compile(dialect=postgresql.dialect())) - assert result is workflow - assert "FOR UPDATE" in sql - - def _make_account(account_id: str = "u1", tenant_id: str = "t1") -> Account: account = Account(name="Test User", email=f"{account_id}@example.com") account.id = account_id @@ -384,73 +392,6 @@ def test_get_all_published_workflow_applies_limit_and_has_more( # --- sync_draft_workflow --- -def test_sync_draft_workflow_creates_new_when_none_exists( - rag_pipeline_service: RagPipelineServiceTestContext, -) -> None: - pipeline = _make_pipeline(workflow_id=None) - account = _make_account() - - result = rag_pipeline_service.service.sync_draft_workflow( - pipeline=pipeline, - graph={"nodes": []}, - unique_hash=None, - account=account, - environment_variables=[], - conversation_variables=[], - rag_pipeline_variables=[], - ) - - assert result.app_id == "p1" - assert pipeline.workflow_id == result.id - - -def test_sync_draft_workflow_raises_on_hash_mismatch( - rag_pipeline_service: RagPipelineServiceTestContext, -) -> None: - from services.errors.app import WorkflowHashNotEqualError - - existing_wf = _make_workflow(graph={"nodes": [{"id": "old"}]}) - _persist(rag_pipeline_service.session, existing_wf) - - pipeline = _make_pipeline() - account = _make_account() - - with pytest.raises(WorkflowHashNotEqualError): - rag_pipeline_service.service.sync_draft_workflow( - pipeline=pipeline, - graph={"nodes": []}, - unique_hash="hash-different", - account=account, - environment_variables=[], - conversation_variables=[], - rag_pipeline_variables=[], - ) - - -def test_sync_draft_workflow_updates_existing( - rag_pipeline_service: RagPipelineServiceTestContext, -) -> None: - existing_wf = _make_workflow(graph={"nodes": [{"id": "old"}]}) - _persist(rag_pipeline_service.session, existing_wf) - - pipeline = _make_pipeline() - account = _make_account() - - result = rag_pipeline_service.service.sync_draft_workflow( - pipeline=pipeline, - graph={"nodes": [{"id": "n1"}]}, - unique_hash=existing_wf.unique_hash, - account=account, - environment_variables=[], - conversation_variables=[], - rag_pipeline_variables=[], - ) - - assert result is existing_wf - assert result.updated_by == "u1" - assert result.graph_dict == {"nodes": [{"id": "n1"}]} - - # --- get_default_block_config --- @@ -486,7 +427,7 @@ def test_update_workflow_updates_allowed_fields( rag_pipeline_service: RagPipelineServiceTestContext, ) -> None: workflow = _make_workflow() - _persist(rag_pipeline_service.session, workflow) + _persist(rag_pipeline_service.session, workflow, _make_account()) workflow_ref = WorkflowRef(tenant_id="t1", owner_id="pipeline-1", workflow_id="wf-1") workflow.app_id = "pipeline-1" rag_pipeline_service.session.commit() @@ -501,7 +442,9 @@ def test_update_workflow_updates_allowed_fields( assert result.marked_name == "v1" assert result.marked_comment == "release" assert not hasattr(result, "disallowed") - assert result.updated_by == "u1" + assert result.updated_by is not None + assert result.updated_by["id"] == "u1" + assert workflow.updated_by == "u1" def test_update_workflow_returns_none_when_not_found( @@ -532,7 +475,9 @@ def test_update_workflow_with_ref_scopes_lookup_to_pipeline( workflow_ref=workflow_ref, ) - assert result is workflow + assert result is not None + assert result.id == workflow.id + assert result.marked_name == workflow.marked_name == "v1" assert other_owner_workflow.marked_name == "" @@ -580,89 +525,6 @@ def test_get_rag_pipeline_workflow_run_delegates( repo_mock.get_workflow_run_by_id.assert_called_once_with(tenant_id="t1", app_id="p1", run_id="run-1") -# --- is_workflow_exist --- - - -def test_is_workflow_exist_returns_true_when_draft_exists( - rag_pipeline_service: RagPipelineServiceTestContext, -) -> None: - _persist(rag_pipeline_service.session, _make_workflow()) - - pipeline = _make_pipeline() - assert rag_pipeline_service.service.is_workflow_exist(pipeline) is True - - -def test_is_workflow_exist_returns_false_when_no_draft( - rag_pipeline_service: RagPipelineServiceTestContext, -) -> None: - pipeline = _make_pipeline() - assert rag_pipeline_service.service.is_workflow_exist(pipeline) is False - - -# --- publish_workflow --- - - -def test_publish_workflow_success(mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext) -> None: - graph = { - "nodes": [ - { - "data": { - "type": "knowledge-index", - "dataset_id": "d1", - "chunk_structure": "paragraph", - "indexing_technique": "high_quality", - "process_rule": {"mode": "automatic"}, - "retrieval_model": {"search_method": "hybrid_search", "top_k": 3}, - } - } - ] - } - draft_workflow = _make_workflow(workflow_id="wf-draft", graph=graph) - pipeline = _make_pipeline(workflow_id="wf-old-published") - dataset = _make_dataset() - _persist(rag_pipeline_service.session, draft_workflow, pipeline, dataset) - mocker.patch("services.rag_pipeline.rag_pipeline.KnowledgeConfiguration.model_validate", return_value=mocker.Mock()) - mock_dataset_service_class = mocker.patch("services.knowledge.dataset_service.DatasetService") - - result = rag_pipeline_service.service.publish_workflow( - session=rag_pipeline_service.session, pipeline=pipeline, account=_make_account() - ) - - assert result.app_id == pipeline.id - assert result.graph_dict == graph - mock_dataset_service_class.update_rag_pipeline_dataset_settings.assert_called_once() - - -def test_publish_workflow_rejects_missing_llm_environment_reference( - rag_pipeline_service: RagPipelineServiceTestContext, -) -> None: - draft_workflow = _make_workflow( - workflow_id="wf-draft", - tenant_id="tenant", - app_id="pipeline", - graph={ - "nodes": [ - { - "id": "llm-node", - "data": { - "type": "llm", - "model": {"provider": "provider", "name": "model", "mode": "chat"}, - "model_selector": ["env", "missing_model"], - }, - } - ] - }, - ) - _persist(rag_pipeline_service.session, draft_workflow) - - with pytest.raises(ValueError, match="missing_model.*not found"): - rag_pipeline_service.service.publish_workflow( - session=rag_pipeline_service.session, - pipeline=_make_pipeline(pipeline_id="pipeline", tenant_id="tenant"), - account=_make_account(account_id="account", tenant_id="tenant"), - ) - - # --- run_datasource_workflow_node --- @@ -690,7 +552,7 @@ def test_run_datasource_workflow_node_website_crawl( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) # 2. Mock DatasourceManager and Runtime mock_runtime = mocker.Mock() @@ -712,22 +574,22 @@ def mock_crawl_gen(**kwargs): # 3. Mock DatasourceProviderService mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value={"api_key": "sk-123"}, ) # 4. Mock Enums to avoid import issues or for consistency - mocker.patch("services.rag_pipeline.rag_pipeline.DatasourceProviderType", DatasourceProviderType) + mocker.patch("services.rag_pipeline.execution_service.DatasourceProviderType", DatasourceProviderType) # 5. Run test - gen = rag_pipeline_service.service.run_datasource_workflow_node( + gen = rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=pipeline, node_id="node-1", user_inputs={"url": "https://example.com"}, account=_make_account(), datasource_type="website_crawl", is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -774,7 +636,7 @@ def test_run_datasource_node_preview_online_document( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) # 2. Mock Runtime and results mock_runtime = mocker.Mock() @@ -798,20 +660,20 @@ def mock_doc_gen(**kwargs): return_value=mock_runtime, ) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value={"token": "abc"}, ) - mocker.patch("services.rag_pipeline.rag_pipeline.DatasourceProviderType", DatasourceProviderType) + mocker.patch("services.rag_pipeline.execution_service.DatasourceProviderType", DatasourceProviderType) # 3. Run test - result = rag_pipeline_service.service.run_datasource_node_preview( + result = rag_pipeline_service.execution.run_datasource_node_preview( pipeline=pipeline, node_id="node-1", user_inputs={}, account=_make_account(), datasource_type="online_document", is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -854,7 +716,7 @@ def mock_getter(): yield event # 2. Run test - result = rag_pipeline_service.service._handle_node_run_result( + result = rag_pipeline_service.execution._handle_node_run_result( getter=lambda: (node_instance, mock_getter()), start_at=time.perf_counter(), tenant_id="t1", node_id="node-1" ) @@ -1057,40 +919,54 @@ def test_set_datasource_variables_success( draft_wf = mocker.Mock() draft_wf.id = "wf-1" draft_wf.get_enclosing_node_type_and_id.return_value = None # Avoid unpacking error - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=draft_wf) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=draft_wf) execution = mocker.Mock(spec=WorkflowNodeExecution) execution.id = "exec-1" execution.process_data = {} execution.inputs = {} execution.outputs = {} - mocker.patch.object(rag_pipeline_service.service, "_handle_node_run_result", return_value=execution) + mocker.patch.object(rag_pipeline_service.execution, "_handle_node_run_result", return_value=execution) # Mock Repository mock_repo_instance = mocker.Mock() - mocker.patch( - "services.rag_pipeline.rag_pipeline.SQLAlchemyWorkflowNodeExecutionRepository", + mocker.patch.object( + rag_pipeline_service.execution, + "_writer_factory", return_value=mock_repo_instance, ) - # Repository._to_db_model is also called + # Read the committed execution through the injected repository. mock_db_exec = mocker.Mock() mock_db_exec.node_id = "node-1" mock_db_exec.node_type = "datasource" - mock_repo_instance._to_db_model.return_value = mock_db_exec + rag_pipeline_service.execution._node_execution_service_repo = mocker.Mock( + get_execution_by_id=mocker.Mock(return_value=mock_db_exec) + ) - # Mock DraftVariableSaver + # Inject through the variable service used by the pipeline. mock_saver_instance = mocker.Mock() - mocker.patch("services.rag_pipeline.rag_pipeline.DraftVariableSaver", return_value=mock_saver_instance) + saver_factory = mocker.Mock(return_value=mock_saver_instance) + factory_provider = mocker.patch.object( + rag_pipeline_service.execution._variables, "saver_factory", return_value=saver_factory + ) # 2. Run test args = {"start_node_id": "node-1"} user = mocker.Mock() user.id = "user-1" - rag_pipeline_service.service.set_datasource_variables(pipeline, args, user) + rag_pipeline_service.execution.set_datasource_variables(pipeline, args, user) # 3. Assertions - mock_repo_instance.save.assert_called_once() - mock_saver_instance.save.assert_called_once() + mock_repo_instance.save_synchronously.assert_called_once() + factory_provider.assert_called_once_with(pipeline.tenant_id, user) + saver_factory.assert_called_once_with( + app_id=pipeline.id, + node_id="node-1", + node_type="datasource", + enclosing_node_id=None, + node_execution_id=execution.id, + ) + mock_saver_instance.save.assert_called_once_with(process_data=execution.process_data, outputs=execution.outputs) # --- Utility Methods --- @@ -1139,18 +1015,6 @@ def test_get_default_block_config_success(rag_pipeline_service: RagPipelineServi assert result["type"] == "llm" -def test_publish_workflow_raises_when_draft_workflow_missing( - rag_pipeline_service: RagPipelineServiceTestContext, -) -> None: - pipeline = _make_pipeline() - account = _make_account() - - with pytest.raises(ValueError, match="No valid workflow found"): - rag_pipeline_service.service.publish_workflow( - session=rag_pipeline_service.session, pipeline=pipeline, account=account - ) - - def test_get_default_block_config_returns_none_when_mapped_type_missing( mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext ) -> None: @@ -1183,10 +1047,10 @@ def test_run_draft_workflow_node_raises_when_workflow_missing( ) -> None: pipeline = _make_pipeline() account = _make_account() - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=None) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=None) with pytest.raises(ValueError, match="Workflow not initialized"): - rag_pipeline_service.service.run_draft_workflow_node(pipeline, "node-1", {}, account) + rag_pipeline_service.execution.run_draft_workflow_node(pipeline, "node-1", {}, account) def test_run_draft_workflow_node_seeds_llm_environment_variable( @@ -1213,25 +1077,43 @@ def test_run_draft_workflow_node_seeds_llm_environment_variable( draft_workflow.environment_variables = [llm_environment_variable] mocker.patch.object(draft_workflow, "get_node_config_by_id", return_value={"id": "node-1"}) mocker.patch.object(draft_workflow, "get_enclosing_node_type_and_id", return_value=None) - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=draft_workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=draft_workflow) - execution = SimpleNamespace(id="exec-1", node_id="node-1", node_type="llm", process_data={}, outputs={}) + execution = WorkflowNodeExecution( + id="exec-1", + workflow_id=draft_workflow.id, + created_at=datetime(2026, 1, 1), + node_id="node-1", + node_type="llm", + title="LLM", + index=1, + status=WorkflowNodeExecutionStatus.SUCCEEDED, + process_data={}, + outputs={}, + ) handle_node_run_result = mocker.patch.object( - rag_pipeline_service.service, "_handle_node_run_result", return_value=execution + rag_pipeline_service.execution, "_handle_node_run_result", return_value=execution ) - single_step_run = mocker.patch("services.rag_pipeline.rag_pipeline.WorkflowEntry.single_step_run") + single_step_run = mocker.patch("services.rag_pipeline.execution_service.WorkflowEntry.single_step_run") repo = mocker.Mock() - mocker.patch( - "services.rag_pipeline.rag_pipeline.DifyCoreRepositoryFactory.create_workflow_node_execution_repository", + mocker.patch.object( + rag_pipeline_service.execution, + "_writer_factory", return_value=repo, ) - rag_pipeline_service.service._node_execution_service_repo = mocker.Mock( + rag_pipeline_service.execution._node_execution_service_repo = mocker.Mock( get_execution_by_id=mocker.Mock(return_value="db") ) - mocker.patch("services.rag_pipeline.rag_pipeline.DraftVariableSaver", return_value=mocker.Mock()) - rag_pipeline_service.service.run_draft_workflow_node(pipeline, "node-1", {}, account) + saver = mocker.Mock() + saver_factory = mocker.Mock(return_value=saver) + factory_provider = mocker.patch.object( + rag_pipeline_service.execution._variables, "saver_factory", return_value=saver_factory + ) + rag_pipeline_service.execution.run_draft_workflow_node(pipeline, "node-1", {}, account) + factory_provider.assert_called_once_with(pipeline.tenant_id, account) + saver.save.assert_called_once_with(process_data=execution.process_data, outputs=execution.outputs) getter = handle_node_run_result.call_args.kwargs["getter"] getter() variable_pool = single_step_run.call_args.kwargs["variable_pool"] @@ -1252,45 +1134,191 @@ def test_run_draft_workflow_node_saves_execution_and_variables( draft_workflow = _make_workflow(workflow_id="wf-1") mocker.patch.object(draft_workflow, "get_node_config_by_id", return_value={"id": "node-1"}) mocker.patch.object(draft_workflow, "get_enclosing_node_type_and_id", return_value=("loop", "enclosing-node")) - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=draft_workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=draft_workflow) - execution = SimpleNamespace(id="exec-1", node_id="node-1", node_type="llm", process_data={}, outputs={}) - mocker.patch.object(rag_pipeline_service.service, "_handle_node_run_result", return_value=execution) + execution = WorkflowNodeExecution( + id="exec-1", + workflow_id=draft_workflow.id, + created_at=datetime(2026, 1, 1), + node_id="node-1", + node_type="llm", + title="LLM", + index=1, + status=WorkflowNodeExecutionStatus.SUCCEEDED, + process_data={}, + outputs={}, + ) + mocker.patch.object(rag_pipeline_service.execution, "_handle_node_run_result", return_value=execution) repo = mocker.Mock() - mocker.patch( - "services.rag_pipeline.rag_pipeline.DifyCoreRepositoryFactory.create_workflow_node_execution_repository", + mocker.patch.object( + rag_pipeline_service.execution, + "_writer_factory", return_value=repo, ) - rag_pipeline_service.service._node_execution_service_repo = mocker.Mock( + rag_pipeline_service.execution._node_execution_service_repo = mocker.Mock( get_execution_by_id=mocker.Mock(return_value="db") ) saver = mocker.Mock() - mocker.patch("services.rag_pipeline.rag_pipeline.DraftVariableSaver", return_value=saver) + saver_factory = mocker.Mock(return_value=saver) + factory_provider = mocker.patch.object( + rag_pipeline_service.execution._variables, "saver_factory", return_value=saver_factory + ) - result = rag_pipeline_service.service.run_draft_workflow_node(pipeline, "node-1", {"q": "x"}, account) + result = rag_pipeline_service.execution.run_draft_workflow_node(pipeline, "node-1", {"q": "x"}, account) assert result == "db" assert execution.workflow_id == "wf-1" - repo.save.assert_called_once_with(execution) - saver.save.assert_called_once() + repo.save_synchronously.assert_called_once_with(execution) + factory_provider.assert_called_once_with(pipeline.tenant_id, account) + saver_factory.assert_called_once_with( + app_id=pipeline.id, + node_id="node-1", + node_type="llm", + enclosing_node_id="enclosing-node", + node_execution_id=execution.id, + ) + saver.save.assert_called_once_with(process_data=execution.process_data, outputs=execution.outputs) + + +def test_single_node_uses_the_injected_variable_store( + rag_pipeline_service: RagPipelineServiceTestContext, + workflow_variables: WorkflowExecutionVariables, + mocker: MockerFixture, +) -> None: + from repositories.sqlalchemy_api_workflow_node_execution_repository import ( + DifyAPISQLAlchemyWorkflowNodeExecutionRepository, + ) + + pipeline = _make_pipeline() + workflow = _make_workflow(graph={"nodes": [{"id": "node-1", "data": {"type": "code"}}], "edges": []}) + account = _make_account() + assert rag_pipeline_service.execution._variables is workflow_variables + workflow_variables.saver_factory(pipeline.tenant_id, account)( + pipeline.id, "previous", BuiltinNodeTypes.CODE, "previous-execution" + ).save(process_data=None, outputs={"text": "injected input"}) + execution = WorkflowNodeExecution( + id="exec-1", + workflow_id=workflow.id, + created_at=datetime(2026, 1, 1), + node_id="node-1", + node_type="code", + title="Code", + index=1, + status=WorkflowNodeExecutionStatus.SUCCEEDED, + outputs={"text": "injected output"}, + ) + loaded: list[str] = [] + + def execute(*, variable_loader: VariableLoader, **_kwargs: object) -> None: + value = variable_loader.load_variables([["previous", "text"]])[0].value + assert isinstance(value, str) + loaded.append(value) + + def result(getter: Callable[[], object], **_kwargs: object) -> WorkflowNodeExecution: + getter() + return execution + + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution, "_handle_node_run_result", side_effect=result) + mocker.patch("services.rag_pipeline.execution_service.WorkflowEntry.single_step_run", side_effect=execute) + rag_pipeline_service.execution._node_execution_service_repo = DifyAPISQLAlchemyWorkflowNodeExecutionRepository( + rag_pipeline_service.session_maker + ) + saved = rag_pipeline_service.execution.run_draft_workflow_node(pipeline, "node-1", {}, account) + + assert saved is not None + assert saved.id == execution.id + assert saved.tenant_id == pipeline.tenant_id + assert saved.app_id == pipeline.id + assert loaded == ["injected input"] + loader = workflow_variables.workflow_loader(workflow, account.id) + assert loader.load_variables([["node-1", "text"]])[0].value == "injected output" + + +@pytest.mark.parametrize("entry", ["datasource", "draft"]) +def test_debug_execution_commits_before_variable_save_with_celery_repository( + rag_pipeline_service: RagPipelineServiceTestContext, + workflow_variables: WorkflowExecutionVariables, + monkeypatch: pytest.MonkeyPatch, + entry: str, +) -> None: + from core.repositories.celery_workflow_node_execution_repository import ( + CeleryWorkflowNodeExecutionRepository, + save_workflow_node_execution_task, + ) + from graphon.graph_events import NodeRunSucceededEvent + from models.workflow import WorkflowNodeExecutionTriggeredFrom + from repositories.sqlalchemy_api_workflow_node_execution_repository import ( + DifyAPISQLAlchemyWorkflowNodeExecutionRepository, + ) + + pipeline = _make_pipeline() + account = _make_account() + workflow = _make_workflow(graph={"nodes": [{"id": "node-1", "data": {"type": "code"}}], "edges": []}) + _persist(rag_pipeline_service.session, workflow) + executions = DifyAPISQLAlchemyWorkflowNodeExecutionRepository(rag_pipeline_service.session_maker) + writer_factory = partial(CeleryWorkflowNodeExecutionRepository, session_factory=rag_pipeline_service.session_maker) + queued = mock.Mock() + monkeypatch.setattr(save_workflow_node_execution_task, "delay", queued) + writer_factory( + tenant_id=pipeline.tenant_id, + user=account, + app_id=pipeline.id, + triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP, + ).save( + WorkflowNodeExecution( + id="queued-only", + workflow_id=workflow.id, + node_id="node-1", + node_type="code", + title="Code", + index=1, + created_at=datetime(2026, 1, 1), + ) + ) + queued.assert_called_once() + assert executions.get_execution_by_id("queued-only", tenant_id=pipeline.tenant_id) is None + + node = mock.Mock(workflow_id=workflow.id, node_type="code", title="Code") + result = NodeRunSucceededEvent( + id="event", + start_at=time.time(), + node_id="node-1", + node_type="code", + node_run_result=NodeRunResult(status=WorkflowNodeExecutionStatus.SUCCEEDED, outputs={"text": "saved"}), + route_node_id=None, + ) + monkeypatch.setattr(execution_module.WorkflowEntry, "single_step_run", lambda **_kwargs: (node, iter([result]))) + service = rag_pipeline_service.execution + service._writer_factory = writer_factory + service._node_execution_service_repo = executions + if entry == "datasource": + saved = service.set_datasource_variables(pipeline, {"start_node_id": "node-1"}, account) + else: + saved = service.run_draft_workflow_node(pipeline, "node-1", {}, account) + assert saved is not None + assert executions.get_execution_by_id(saved.id, tenant_id=pipeline.tenant_id) is not None + queued.assert_called_once() # Debug writes did not depend on the queued task running. + loaded = workflow_variables.workflow_loader(workflow, account.id).load_variables([["node-1", "text"]]) + assert loaded[0].value == "saved" def test_run_datasource_workflow_node_returns_error_when_workflow_missing( mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext ) -> None: pipeline = _make_pipeline() - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=None) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=None) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=pipeline, node_id="node-1", user_inputs={}, account=_make_account(), datasource_type="online_document", is_published=False, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -1321,7 +1349,7 @@ def test_run_datasource_workflow_node_online_document_success( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() runtime.runtime = SimpleNamespace(credentials=None) @@ -1329,19 +1357,19 @@ def test_run_datasource_workflow_node_online_document_success( runtime.get_online_document_pages.return_value = [SimpleNamespace(result=[{"id": "pg-1"}])] mocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value={"token": "x"}, ) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=pipeline, node_id="node-1", user_inputs={}, account=_make_account(), datasource_type=DatasourceProviderType.ONLINE_DOCUMENT, is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -1376,7 +1404,7 @@ def test_run_datasource_workflow_node_online_drive_success( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() runtime.runtime = SimpleNamespace(credentials=None) @@ -1384,19 +1412,19 @@ def test_run_datasource_workflow_node_online_drive_success( runtime.online_drive_browse_files.return_value = [SimpleNamespace(result=[{"name": "f1"}])] mocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value={}, ) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=pipeline, node_id="node-1", user_inputs={"bucket": "bucket-1"}, account=_make_account(), datasource_type=DatasourceProviderType.ONLINE_DRIVE, is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -1411,7 +1439,6 @@ def test_handle_node_run_result_default_value_strategy( ) -> None: from datetime import datetime - from graphon.graph_events import NodeRunFailedEvent from graphon.node_events.base import NodeRunResult node_instance = SimpleNamespace( @@ -1440,7 +1467,7 @@ def _events(): node_run_result=failed_result, ) - result = rag_pipeline_service.service._handle_node_run_result( + result = rag_pipeline_service.execution._handle_node_run_result( getter=lambda: (node_instance, _events()), start_at=time.perf_counter(), tenant_id="t1", @@ -1494,20 +1521,20 @@ def test_get_second_step_parameters_handles_string_and_list_variable_references( def test_get_rag_pipeline_workflow_run_node_executions_empty_when_run_missing( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext + mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext, *, tool_providers ) -> None: pipeline = _make_pipeline() mocker.patch.object(rag_pipeline_service.service, "get_rag_pipeline_workflow_run", return_value=None) result = rag_pipeline_service.service.get_rag_pipeline_workflow_run_node_executions( - pipeline=pipeline, run_id="run-1", user=_make_account() + pipeline=pipeline, run_id="run-1", user=_make_account(), tool_providers=tool_providers ) assert result == [] def test_get_rag_pipeline_workflow_run_node_executions_assembles_configured_repository_results( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext + mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext, *, tool_providers ) -> None: pipeline = _make_pipeline() mocker.patch.object( @@ -1523,7 +1550,7 @@ def test_get_rag_pipeline_workflow_run_node_executions_assembles_configured_repo ) result = rag_pipeline_service.service.get_rag_pipeline_workflow_run_node_executions( - pipeline=pipeline, run_id="run-1", user=_make_account() + pipeline=pipeline, run_id="run-1", user=_make_account(), tool_providers=tool_providers ) assert result == expected_traces @@ -1532,13 +1559,17 @@ def test_get_rag_pipeline_workflow_run_node_executions_assembles_configured_repo app_id=pipeline.id, workflow_run_id="run-1", ) - mock_assemble.assert_called_once_with(expected_executions, node_repo, session=mock.ANY) + mock_assemble.assert_called_once_with( + expected_executions, node_repo, session=mock.ANY, tool_providers=tool_providers + ) def test_get_recommended_plugins_returns_empty_when_no_active_plugins( - rag_pipeline_service: RagPipelineServiceTestContext, + rag_pipeline_service: RagPipelineServiceTestContext, *, tool_providers ) -> None: - result = rag_pipeline_service.service.get_recommended_plugins("all", _make_account(), "t1") + result = rag_pipeline_service.service.get_recommended_plugins( + "all", _make_account(), "t1", tool_providers=tool_providers + ) assert result == { "installed_recommended_plugins": [], @@ -1547,7 +1578,7 @@ def test_get_recommended_plugins_returns_empty_when_no_active_plugins( def test_get_recommended_plugins_returns_installed_and_uninstalled( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext + mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext, *, tool_providers ) -> None: plugin_a = _make_recommended_plugin("plugin-a") plugin_b = _make_recommended_plugin("plugin-b") @@ -1561,7 +1592,9 @@ def test_get_recommended_plugins_returns_installed_and_uninstalled( return_value=[{"plugin_id": "plugin-b", "name": "Plugin B"}], ) - result = rag_pipeline_service.service.get_recommended_plugins("tool", _make_account(), "t1") + result = rag_pipeline_service.service.get_recommended_plugins( + "tool", _make_account(), "t1", tool_providers=tool_providers + ) assert result["installed_recommended_plugins"] == [{"plugin_id": "plugin-a"}] assert result["uninstalled_recommended_plugins"] == [{"plugin_id": "plugin-b", "name": "Plugin B"}] @@ -1586,10 +1619,10 @@ def test_set_datasource_variables_raises_when_node_id_missing( ) -> None: pipeline = _make_pipeline() workflow = _make_workflow() - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=workflow) with pytest.raises(ValueError, match="Node id is required"): - rag_pipeline_service.service.set_datasource_variables(pipeline, {"start_node_id": ""}, _make_account()) + rag_pipeline_service.execution.set_datasource_variables(pipeline, {"start_node_id": ""}, _make_account()) def test_get_default_block_configs_skips_empty_configs( @@ -1622,17 +1655,17 @@ def test_run_datasource_workflow_node_returns_error_when_node_missing( ) -> None: pipeline = _make_pipeline() workflow = _make_workflow(graph={"nodes": []}) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=pipeline, node_id="missing-node", user_inputs={}, account=_make_account(), datasource_type="online_document", is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -1661,7 +1694,7 @@ def test_run_datasource_workflow_node_online_document_exception( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() @@ -1677,19 +1710,19 @@ def __next__(self): mocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=pipeline, node_id="node-1", user_inputs={}, account=_make_account(), datasource_type="online_document", is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -1721,7 +1754,7 @@ def test_run_datasource_node_preview_raises_for_stream_non_string( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() @@ -1736,19 +1769,19 @@ def _bad_stream_generator[**P](*args: P.args, **kwargs: P.kwargs): mocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) with pytest.raises(RuntimeError, match="must be a string"): - rag_pipeline_service.service.run_datasource_node_preview( + rag_pipeline_service.execution.run_datasource_node_preview( pipeline=pipeline, node_id="node-1", user_inputs={}, account=_make_account(), datasource_type="online_document", is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -1844,7 +1877,7 @@ def _event_generator(): yield object() with pytest.raises(ValueError, match="Node run failed with no run result"): - rag_pipeline_service.service._handle_node_run_result( + rag_pipeline_service.execution._handle_node_run_result( getter=lambda: (node_instance, _event_generator()), start_at=time.perf_counter(), tenant_id="t1", @@ -1864,7 +1897,7 @@ def test_handle_node_run_result_marks_document_error_for_published_invoke( document, ) - result = rag_pipeline_service.service._handle_node_run_result( + result = rag_pipeline_service.execution._handle_node_run_result( getter=lambda: (node_instance, iter([event])), start_at=time.perf_counter(), tenant_id="t1", @@ -1884,7 +1917,7 @@ def test_handle_node_run_result_does_not_write_when_pipeline_dataset_is_missing( ) -> None: node_instance, event = _make_failed_published_node_run() - result = rag_pipeline_service.service._handle_node_run_result( + result = rag_pipeline_service.execution._handle_node_run_result( getter=lambda: (node_instance, iter([event])), start_at=time.perf_counter(), tenant_id="t1", @@ -1914,24 +1947,24 @@ def test_run_datasource_node_preview_raises_for_unsupported_provider( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() runtime.datasource_provider_type.return_value = "unsupported" mocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) with pytest.raises(RuntimeError, match="Unsupported datasource provider"): - rag_pipeline_service.service.run_datasource_node_preview( + rag_pipeline_service.execution.run_datasource_node_preview( pipeline=pipeline, node_id="node-1", user_inputs={}, account=_make_account(), datasource_type="website_crawl", is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -1965,8 +1998,10 @@ def test_get_pipeline_raises_when_pipeline_missing( rag_pipeline_service.service.get_pipeline("t1", "d1") -def test_init_uses_default_sessionmaker_when_none(mocker: MockerFixture, sqlite_session: Session) -> None: - mocker.patch("services.rag_pipeline.rag_pipeline.db", SimpleNamespace(engine=sqlite_session.get_bind())) +def test_init_uses_default_sessionmaker_when_none( + mocker: MockerFixture, + sqlite_session: Session, +) -> None: create_exec_repo = mocker.patch( "services.rag_pipeline.rag_pipeline.DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository" ) @@ -2034,27 +2069,10 @@ def test_get_all_published_workflow_without_filters_has_no_more( named_only=False, ) - assert workflows == [workflow] + assert [record.id for record in workflows] == [workflow.id] assert has_more is False -def test_publish_workflow_skips_dataset_update_for_non_knowledge_nodes( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext -) -> None: - draft = _make_workflow(graph={"nodes": [{"data": {"type": "start"}}]}) - _persist(rag_pipeline_service.session, draft) - dataset_service = mocker.patch("services.knowledge.dataset_service.DatasetService") - - result = rag_pipeline_service.service.publish_workflow( - session=rag_pipeline_service.session, - pipeline=_make_pipeline(), - account=_make_account(), - ) - - assert result.graph_dict == {"nodes": [{"data": {"type": "start"}}]} - dataset_service.update_rag_pipeline_dataset_settings.assert_not_called() - - def test_get_default_block_config_returns_none_when_default_empty( mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext ) -> None: @@ -2094,7 +2112,7 @@ def test_run_datasource_workflow_node_handles_variable_parameter_types( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() def crawl_gen(**kwargs): @@ -2104,19 +2122,19 @@ def crawl_gen(**kwargs): runtime.datasource_provider_type.return_value = DatasourceProviderType.WEBSITE_CRAWL mocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=_make_pipeline(), node_id="node-1", user_inputs={"k": "mapped"}, account=_make_account(), datasource_type="website_crawl", is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -2146,7 +2164,7 @@ def test_run_datasource_workflow_node_online_drive_branch( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() def drive_gen(**kwargs): @@ -2156,19 +2174,19 @@ def drive_gen(**kwargs): runtime.datasource_provider_type.return_value = DatasourceProviderType.ONLINE_DRIVE mocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=_make_pipeline(), node_id="node-1", user_inputs={}, account=_make_account(), datasource_type="online_drive", is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -2198,7 +2216,9 @@ def test_run_datasource_node_preview_not_published_uses_draft( ] } ) - get_draft = mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=workflow) + get_draft = mocker.patch.object( + rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=workflow + ) runtime = mocker.Mock() def doc_gen(**kwargs): @@ -2210,18 +2230,18 @@ def doc_gen(**kwargs): runtime.get_online_document_page_content.side_effect = doc_gen mocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) - result = rag_pipeline_service.service.run_datasource_node_preview( + result = rag_pipeline_service.execution.run_datasource_node_preview( pipeline=_make_pipeline(), node_id="n1", user_inputs={}, account=_make_account(), datasource_type="online_document", is_published=False, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -2230,24 +2250,6 @@ def doc_gen(**kwargs): get_draft.assert_called_once() -def test_run_free_workflow_node_delegates_to_handle_result( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext -) -> None: - expected = SimpleNamespace(id="exec-1") - handle = mocker.patch.object(rag_pipeline_service.service, "_handle_node_run_result", return_value=expected) - - result = rag_pipeline_service.service.run_free_workflow_node( - node_data={"type": "start"}, - tenant_id="t1", - user_id="u1", - node_id="n1", - user_inputs={}, - ) - - assert result is expected - handle.assert_called_once() - - @pytest.mark.parametrize(("workflow_tenant_id", "workflow_app_id"), [("t2", "p1"), ("t1", "p2")]) def test_publish_customized_pipeline_template_rejects_unowned_workflow_before_export( mocker: MockerFixture, @@ -2313,14 +2315,16 @@ def test_publish_customized_pipeline_template_rejects_missing_or_unowned_draft_b def test_get_recommended_plugins_skips_manifest_when_missing( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext + mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext, *, tool_providers ) -> None: plugin = _make_recommended_plugin("plugin-a") _persist(rag_pipeline_service.session, plugin) mocker.patch("services.rag_pipeline.rag_pipeline.BuiltinToolManageService.list_builtin_tools", return_value=[]) mocker.patch("services.rag_pipeline.rag_pipeline.marketplace.batch_fetch_plugin_by_ids", return_value=[]) - result = rag_pipeline_service.service.get_recommended_plugins("all", _make_account(), "t1") + result = rag_pipeline_service.service.get_recommended_plugins( + "all", _make_account(), "t1", tool_providers=tool_providers + ) assert result["installed_recommended_plugins"] == [] assert result["uninstalled_recommended_plugins"] == [] @@ -2391,34 +2395,20 @@ def test_get_datasource_plugins_returns_empty_for_non_datasource_nodes( ) -def test_publish_workflow_raises_when_knowledge_index_dataset_missing( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext -) -> None: - draft = _make_workflow(graph={"nodes": [{"data": {"type": "knowledge-index"}}]}) - _persist(rag_pipeline_service.session, draft) - mocker.patch("services.rag_pipeline.rag_pipeline.KnowledgeConfiguration.model_validate", return_value=mocker.Mock()) - pipeline = _make_pipeline() - - with pytest.raises(ValueError, match="Dataset not found"): - rag_pipeline_service.service.publish_workflow( - session=rag_pipeline_service.session, pipeline=pipeline, account=_make_account() - ) - - def test_run_datasource_node_preview_raises_when_workflow_missing( mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext ) -> None: - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=None) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=None) with pytest.raises(RuntimeError, match="Workflow not initialized"): - rag_pipeline_service.service.run_datasource_node_preview( + rag_pipeline_service.execution.run_datasource_node_preview( pipeline=_make_pipeline(), node_id="n1", user_inputs={}, account=_make_account(), datasource_type="online_document", is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -2428,18 +2418,20 @@ def test_run_datasource_node_preview_raises_when_node_missing( mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext ) -> None: mocker.patch.object( - rag_pipeline_service.service, "get_published_workflow", return_value=_make_workflow(graph={"nodes": []}) + rag_pipeline_service.execution._workflows, + "get_published_workflow", + return_value=_make_workflow(graph={"nodes": []}), ) with pytest.raises(RuntimeError, match="Datasource node data not found"): - rag_pipeline_service.service.run_datasource_node_preview( + rag_pipeline_service.execution.run_datasource_node_preview( pipeline=_make_pipeline(), node_id="missing", user_inputs={}, account=_make_account(), datasource_type="online_document", is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -2465,7 +2457,7 @@ def test_run_datasource_node_preview_keeps_existing_user_input( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() def gen(**kwargs): @@ -2479,18 +2471,18 @@ def gen(**kwargs): runtime.get_online_document_page_content.side_effect = gen mocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) - result = rag_pipeline_service.service.run_datasource_node_preview( + result = rag_pipeline_service.execution.run_datasource_node_preview( pipeline=_make_pipeline(), node_id="n1", user_inputs={"workspace_id": "existing"}, account=_make_account(), datasource_type="online_document", is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -2515,7 +2507,7 @@ def test_run_datasource_node_preview_ignores_non_variable_messages( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() def gen(**kwargs): @@ -2524,18 +2516,18 @@ def gen(**kwargs): runtime.get_online_document_page_content.side_effect = gen mocker.patch("core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) - result = rag_pipeline_service.service.run_datasource_node_preview( + result = rag_pipeline_service.execution.run_datasource_node_preview( pipeline=_make_pipeline(), node_id="n1", user_inputs={}, account=_make_account(), datasource_type="online_document", is_published=True, - datasource_providers=rag_pipeline_module.DatasourceProviderService( + datasource_providers=execution_module.DatasourceProviderService( credentials=create_autospec(DatasourceProviderCredentialStore, instance=True) ), ) @@ -2545,10 +2537,10 @@ def gen(**kwargs): def test_set_datasource_variables_raises_when_workflow_missing( mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext ) -> None: - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=None) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=None) with pytest.raises(ValueError, match="Workflow not initialized"): - rag_pipeline_service.service.set_datasource_variables( + rag_pipeline_service.execution.set_datasource_variables( _make_pipeline(), {"start_node_id": "n1"}, _make_account(), diff --git a/api/tests/unit_tests/services/test_agent_tool_inner_service.py b/api/tests/unit_tests/services/test_agent_tool_inner_service.py deleted file mode 100644 index 8f222b3309356e..00000000000000 --- a/api/tests/unit_tests/services/test_agent_tool_inner_service.py +++ /dev/null @@ -1,198 +0,0 @@ -"""Unit tests for the Agent tool inner invoke service with SQLite-backed app lookup.""" - -from collections.abc import Generator -from unittest.mock import MagicMock, patch - -import pytest -from sqlalchemy.orm import Session - -from core.tools.entities.tool_entities import ToolInvokeMessage, ToolProviderType -from core.tools.errors import ( - ToolInvokeError, - ToolParameterValidationError, - ToolProviderCredentialValidationError, - ToolProviderNotFoundError, -) -from models.enums import AppStatus -from models.model import App, AppMode -from services.agent_tool_inner_service import AgentToolInnerService -from services.entities.agent_tool_inner import AgentToolInvokeRequest -from services.errors.agent_tool_inner import AgentToolInnerServiceError - -TENANT_ID = "11111111-1111-1111-1111-111111111111" -OTHER_TENANT_ID = "22222222-2222-2222-2222-222222222222" -USER_ID = "33333333-3333-3333-3333-333333333333" -APP_ID = "44444444-4444-4444-4444-444444444444" - - -def _persist_app(sqlite_session: Session, *, tenant_id: str = TENANT_ID) -> App: - app = App( - id=APP_ID, - tenant_id=tenant_id, - name="Test App", - description="", - mode=AppMode.CHAT, - status=AppStatus.NORMAL, - enable_site=False, - enable_api=False, - max_active_requests=None, - ) - sqlite_session.add(app) - sqlite_session.commit() - sqlite_session.expunge_all() - return app - - -def _request() -> AgentToolInvokeRequest: - return AgentToolInvokeRequest.model_validate( - { - "caller": { - "tenant_id": TENANT_ID, - "user_id": USER_ID, - "user_from": "account", - "app_id": APP_ID, - "invoke_from": "service-api", - "conversation_id": "conversation-1", - "workflow_id": "workflow-1", - "workflow_run_id": "workflow-run-1", - "node_id": "node-1", - "node_execution_id": "node-exec-1", - "agent_id": "agent-1", - "agent_config_version_id": "snapshot-1", - }, - "tool": { - "provider_type": "plugin", - "provider_id": "langgenius/search/search", - "tool_name": "search", - "credential_id": "credential-1", - "tool_parameters": {"query": "dify"}, - "runtime_parameters": {"region": "us"}, - }, - } - ) - - -def _messages() -> Generator[ToolInvokeMessage, None, None]: - yield ToolInvokeMessage( - type=ToolInvokeMessage.MessageType.TEXT, - message=ToolInvokeMessage.TextMessage(text="ok"), - ) - - -@pytest.mark.parametrize("sqlite_session", [(App,)], indirect=True) -def test_invoke_uses_agent_tool_runtime_and_returns_observation(sqlite_session: Session) -> None: - fake_tool = MagicMock() - _persist_app(sqlite_session) - - with ( - patch( - "services.agent_tool_inner_service.ToolManager.get_agent_tool_runtime", - return_value=fake_tool, - ) as mock_get_runtime, - patch("services.agent_tool_inner_service.ToolEngine.generic_invoke", return_value=_messages()) as mock_invoke, - patch( - "services.agent_tool_inner_service.ToolFileMessageTransformer.transform_tool_invoke_messages", - side_effect=lambda messages, **_kwargs: messages, - ), - ): - response = AgentToolInnerService().invoke(_request(), session=sqlite_session) - - assert response.observation == "ok" - assert response.metadata == { - "provider_type": "plugin", - "provider_id": "langgenius/search/search", - "tool_name": "search", - } - agent_tool = mock_get_runtime.call_args.kwargs["agent_tool"] - assert agent_tool.provider_type is ToolProviderType.PLUGIN - assert agent_tool.tool_parameters == {"region": "us"} - mock_invoke.assert_called_once() - assert mock_invoke.call_args.kwargs["session"] is sqlite_session - assert sqlite_session.in_transaction() - - -@pytest.mark.parametrize("sqlite_session", [(App,)], indirect=True) -def test_invoke_raises_app_not_found_when_session_has_no_app(sqlite_session: Session) -> None: - with pytest.raises(AgentToolInnerServiceError) as exc_info: - AgentToolInnerService().invoke(_request(), session=sqlite_session) - - assert exc_info.value.error_code == "app_not_found" - assert exc_info.value.status_code == 404 - assert exc_info.value.description == "App not found." - assert sqlite_session.in_transaction() - - -@pytest.mark.parametrize("sqlite_session", [(App,)], indirect=True) -def test_invoke_raises_app_tenant_mismatch_when_app_belongs_to_other_tenant(sqlite_session: Session) -> None: - _persist_app(sqlite_session, tenant_id=OTHER_TENANT_ID) - - with pytest.raises(AgentToolInnerServiceError) as exc_info: - AgentToolInnerService().invoke(_request(), session=sqlite_session) - - assert exc_info.value.error_code == "app_tenant_mismatch" - assert exc_info.value.status_code == 403 - assert exc_info.value.description == "App does not belong to the caller tenant." - assert sqlite_session.in_transaction() - - -@pytest.mark.parametrize("sqlite_session", [(App,)], indirect=True) -def test_invoke_maps_tool_runtime_app_not_found_value_error_to_specific_error_code( - sqlite_session: Session, -) -> None: - fake_tool = MagicMock() - _persist_app(sqlite_session) - - with ( - patch("services.agent_tool_inner_service.ToolManager.get_agent_tool_runtime", return_value=fake_tool), - patch("services.agent_tool_inner_service.ToolEngine.generic_invoke", side_effect=ValueError("app not found")), - ): - with pytest.raises(AgentToolInnerServiceError) as exc_info: - AgentToolInnerService().invoke(_request(), session=sqlite_session) - - assert exc_info.value.error_code == "app_not_found" - assert exc_info.value.status_code == 404 - assert exc_info.value.description == "App not found." - assert sqlite_session.in_transaction() - - -@pytest.mark.parametrize("sqlite_session", [(App,)], indirect=True) -def test_invoke_maps_tool_invoke_error_without_private_tool_engine_helper(sqlite_session: Session) -> None: - fake_tool = MagicMock() - _persist_app(sqlite_session) - - with ( - patch("services.agent_tool_inner_service.ToolManager.get_agent_tool_runtime", return_value=fake_tool), - patch( - "services.agent_tool_inner_service.ToolEngine.generic_invoke", - side_effect=ToolInvokeError("workflow crashed"), - ), - ): - with pytest.raises(AgentToolInnerServiceError) as exc_info: - AgentToolInnerService().invoke(_request(), session=sqlite_session) - - assert exc_info.value.error_code == "agent_tool_invoke_failed" - assert sqlite_session.in_transaction() - - -@pytest.mark.parametrize( - ("error", "expected_code"), - [ - (ToolProviderNotFoundError("provider gone"), "agent_tool_declaration_not_found"), - (ToolProviderCredentialValidationError("credential invalid"), "agent_tool_credential_invalid"), - (ToolParameterValidationError("query is required"), "tool_parameters_invalid"), - ], -) -@pytest.mark.parametrize("sqlite_session", [(App,)], indirect=True) -def test_invoke_maps_runtime_lookup_errors_to_service_error_codes( - error: Exception, - expected_code: str, - sqlite_session: Session, -) -> None: - _persist_app(sqlite_session) - - with patch("services.agent_tool_inner_service.ToolManager.get_agent_tool_runtime", side_effect=error): - with pytest.raises(AgentToolInnerServiceError) as exc_info: - AgentToolInnerService().invoke(_request(), session=sqlite_session) - - assert exc_info.value.error_code == expected_code - assert sqlite_session.in_transaction() diff --git a/api/tests/unit_tests/services/test_annotation_service.py b/api/tests/unit_tests/services/test_annotation_service.py index ac2a69b0f40448..e06bc462e79226 100644 --- a/api/tests/unit_tests/services/test_annotation_service.py +++ b/api/tests/unit_tests/services/test_annotation_service.py @@ -19,7 +19,6 @@ from sqlalchemy import func, select from sqlalchemy.orm import Session, sessionmaker from werkzeug.datastructures import FileStorage -from werkzeug.exceptions import NotFound import services.annotation_service as annotation_service_module from enums import DeploymentEdition @@ -36,6 +35,7 @@ Message, MessageAnnotation, ) +from services.annotation.errors import AnnotationResourceNotFoundError from services.annotation_service import AppAnnotationService from services.app_ref_service import AnnotationRef, AppRef from tests.unit_tests.config_override import config_overrides_context @@ -221,7 +221,7 @@ class TestAppAnnotationServiceUpsert: def test_rejects_missing_or_cross_tenant_app(self, sqlite_session: Session, current_user: Account) -> None: _persist_app(sqlite_session, app_id="other-app", tenant_id=OTHER_TENANT_ID) - with pytest.raises(NotFound): + with pytest.raises(AnnotationResourceNotFoundError): AppAnnotationService.up_insert_app_annotation_from_message( {"answer": "hello", "question": "q"}, "other-app", sqlite_session ) @@ -239,7 +239,7 @@ def test_rejects_message_from_another_app(self, sqlite_session: Session, current other_app = _persist_app(sqlite_session, app_id="app-2") message = _persist_message(sqlite_session, other_app) - with pytest.raises(NotFound, match="Message"): + with pytest.raises(AnnotationResourceNotFoundError, match="Message"): AppAnnotationService.up_insert_app_annotation_from_message( {"answer": "hello", "message_id": message.id}, app.id, sqlite_session ) @@ -374,7 +374,7 @@ class TestAppAnnotationServiceListAndExport: def test_list_rejects_cross_tenant_app(self, sqlite_session: Session, current_user: Account) -> None: app = _persist_app(sqlite_session, tenant_id=OTHER_TENANT_ID) - with pytest.raises(NotFound): + with pytest.raises(AnnotationResourceNotFoundError): AppAnnotationService.get_annotation_list_by_app_id(app.id, 1, 10, "", sqlite_session) def test_list_filters_orders_and_paginates(self, sqlite_session: Session, current_user: Account) -> None: @@ -433,7 +433,7 @@ def test_insert_rejects_cross_tenant_app_and_missing_question( self, sqlite_session: Session, current_user: Account ) -> None: other_app = _persist_app(sqlite_session, tenant_id=OTHER_TENANT_ID) - with pytest.raises(NotFound): + with pytest.raises(AnnotationResourceNotFoundError): AppAnnotationService.insert_app_annotation_directly( {"answer": "hello", "question": "q"}, other_app.id, sqlite_session ) @@ -471,7 +471,7 @@ def test_update_is_app_scoped_and_validates_fields(self, sqlite_session: Session other_app = _persist_app(sqlite_session, app_id="app-2") annotation = _persist_annotation(sqlite_session, other_app) - with pytest.raises(NotFound): + with pytest.raises(AnnotationResourceNotFoundError): AppAnnotationService.update_app_annotation_directly( {"answer": "a", "question": "q"}, _annotation_ref(app, annotation.id), sqlite_session ) @@ -512,7 +512,7 @@ def test_delete_is_app_scoped(self, sqlite_session: Session, current_user: Accou other_app = _persist_app(sqlite_session, app_id="app-2") annotation = _persist_annotation(sqlite_session, other_app) - with pytest.raises(NotFound): + with pytest.raises(AnnotationResourceNotFoundError): AppAnnotationService.delete_app_annotation(_annotation_ref(app, annotation.id), sqlite_session) assert sqlite_session.get(MessageAnnotation, annotation.id) is not None @@ -609,7 +609,7 @@ def _invoke( def test_rejects_cross_tenant_app(self, sqlite_session: Session, current_user: Account) -> None: app = _persist_app(sqlite_session, tenant_id=OTHER_TENANT_ID) - with pytest.raises(NotFound): + with pytest.raises(AnnotationResourceNotFoundError): AppAnnotationService.batch_import_app_annotations(app.id, _file(b"question,answer\nq,a\n"), sqlite_session) @pytest.mark.parametrize( @@ -775,7 +775,7 @@ def test_hit_histories_are_annotation_and_app_scoped(self, sqlite_session: Sessi other_app = _persist_app(sqlite_session, app_id="app-2") other_annotation = _persist_annotation(sqlite_session, other_app) - with pytest.raises(NotFound): + with pytest.raises(AnnotationResourceNotFoundError): AppAnnotationService.get_annotation_hit_histories( _annotation_ref(app, other_annotation.id), 1, 10, sqlite_session ) @@ -797,49 +797,10 @@ def test_hit_histories_are_annotation_and_app_scoped(self, sqlite_session: Sessi assert total == 2 assert [item.id for item in items] == [old.id] - def test_get_annotation_by_id_uses_real_identity_lookup( - self, sqlite_session: Session, current_user: Account - ) -> None: - app = _persist_app(sqlite_session) - annotation = _persist_annotation(sqlite_session, app) - - assert AppAnnotationService.get_annotation_by_id("missing", sqlite_session) is None - assert AppAnnotationService.get_annotation_by_id(annotation.id, sqlite_session) is annotation - - def test_add_history_increments_count_and_flushes_row(self, sqlite_session: Session, current_user: Account) -> None: - app = _persist_app(sqlite_session) - annotation = _persist_annotation(sqlite_session, app) - - AppAnnotationService.add_annotation_history( - annotation_id=annotation.id, - app_id=app.id, - annotation_question="q", - annotation_content="a", - query="user q", - user_id=current_user.id, - message_id="msg-1", - from_source="chat", - score=0.8, - session=sqlite_session, - ) - - sqlite_session.refresh(annotation) - assert annotation.hit_count == 1 - history = sqlite_session.scalar( - select(AppAnnotationHitHistory).where(AppAnnotationHitHistory.annotation_id == annotation.id) - ) - assert history is not None - assert (history.question, history.annotation_question, history.annotation_content, history.score) == ( - "user q", - "q", - "a", - 0.8, - ) - def test_get_setting_rejects_cross_tenant_app(self, sqlite_session: Session, current_user: Account) -> None: app = _persist_app(sqlite_session, tenant_id=OTHER_TENANT_ID) - with pytest.raises(NotFound): + with pytest.raises(AnnotationResourceNotFoundError): AppAnnotationService.get_app_annotation_setting_by_app_id(app.id, sqlite_session) def test_get_setting_returns_disabled_without_row(self, sqlite_session: Session, current_user: Account) -> None: @@ -884,7 +845,7 @@ def test_update_setting_is_app_scoped(self, sqlite_session: Session, current_use other_app = _persist_app(sqlite_session, app_id="app-2") other_setting = _persist_setting(sqlite_session, other_app) - with pytest.raises(NotFound): + with pytest.raises(AnnotationResourceNotFoundError): AppAnnotationService.update_app_annotation_setting( app.id, other_setting.id, {"score_threshold": 0.8}, sqlite_session ) @@ -960,7 +921,7 @@ def test_clear_all_deletes_only_app_rows_and_enqueues_indexes( def test_clear_all_rejects_cross_tenant_app(self, sqlite_session: Session, current_user: Account) -> None: app = _persist_app(sqlite_session, tenant_id=OTHER_TENANT_ID) - with pytest.raises(NotFound): + with pytest.raises(AnnotationResourceNotFoundError): AppAnnotationService.clear_all_annotations(app.id, sqlite_session) assert sqlite_session.scalar(select(func.count()).select_from(MessageAnnotation)) == 0 diff --git a/api/tests/unit_tests/services/test_app_definition_query_service.py b/api/tests/unit_tests/services/test_app_definition_query_service.py index 9987d4e50d37e3..eb5ae55580c6e3 100644 --- a/api/tests/unit_tests/services/test_app_definition_query_service.py +++ b/api/tests/unit_tests/services/test_app_definition_query_service.py @@ -3,7 +3,7 @@ import pytest import services.app_definition_query_service as module -from core.app.apps.agent_app.errors import AgentAppGeneratorError, AgentAppNotPublishedError +from services.app.generation.errors import AgentAppGeneratorError, AgentAppNotPublishedError from services.app_definition_query_service import ( AppDefinitionNotPublishedError, AppDefinitionQuery, diff --git a/api/tests/unit_tests/services/test_app_dsl_service.py b/api/tests/unit_tests/services/test_app_dsl_service.py index 66d2f0067b41c8..8caf53a2ad9811 100644 --- a/api/tests/unit_tests/services/test_app_dsl_service.py +++ b/api/tests/unit_tests/services/test_app_dsl_service.py @@ -12,16 +12,18 @@ from constants.dsl_version import CURRENT_APP_DSL_VERSION from core.rbac import RBACPermission, RBACResourceScope from core.workflow.llm_environment_variable import LLMEnvironmentVariable +from extensions.application_services.workflow import build_app_dsl_service from models import Account, App, AppMode from models.enums import CustomizeTokenStrategy from models.model import AppModelConfig, AppModelConfigDict, IconType, Site from models.workflow import Workflow from services.agent.dsl_entities import AgentPackage from services.app_dsl_service import AppDslService, PendingData -from services.entities.dsl_entities import ImportStatus +from services.entities.dsl_entities import AppDslOverwriteStore, AppDslOverwriteTarget, ImportStatus from services.entities.site_dsl import SiteDsl from services.errors.app import WorkflowNotFoundError from services.errors.base import NoPermissionError +from services.workflow.draft_service import WorkflowDraftService from tests.unit_tests.config_override import apply_config_overrides from tests.unit_tests.model_factories import make_account, make_app, make_tenant, make_workflow @@ -190,7 +192,9 @@ def test_import_app_rejects_oversized_yaml_content_before_parsing( monkeypatch: pytest.MonkeyPatch, unbound_session: Session ) -> None: monkeypatch.setattr("services.app_dsl_service.DSL_MAX_SIZE", 3) - service = AppDslService(session=unbound_session) + service = AppDslService( + session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + ) account = _account() result = service.import_app(account=account, import_mode="yaml-content", yaml_content="你你") @@ -208,7 +212,9 @@ def test_import_app_rejects_oversized_yaml_url_bytes_before_decode( response.raise_for_status.return_value = None response.content = b"\xff\xff" monkeypatch.setattr("services.app_dsl_service.remote_fetcher.make_request", Mock(return_value=response)) - service = AppDslService(session=unbound_session) + service = AppDslService( + session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + ) result = service.import_app( account=_account(), @@ -228,7 +234,9 @@ def test_import_app_returns_decode_error_for_invalid_yaml_url_bytes( response.raise_for_status.return_value = None response.content = b"\xff" monkeypatch.setattr("services.app_dsl_service.remote_fetcher.make_request", Mock(return_value=response)) - service = AppDslService(session=unbound_session) + service = AppDslService( + session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + ) result = service.import_app( account=_account(), @@ -258,7 +266,7 @@ def deny_before_transaction(*_args: object, **_kwargs: object) -> bool: monkeypatch.setattr("services.app_dsl_service.redis_client.setex", setex) with pytest.raises(NoPermissionError, match="permission to overwrite"): - AppDslService(sqlite_session).import_app( + build_app_dsl_service(sqlite_session).import_app( account=account, import_mode="yaml-content", yaml_content=_PENDING_WORKFLOW_DSL, @@ -283,7 +291,7 @@ def test_confirm_import_rechecks_overwrite_rbac_before_database_access( redis_delete = Mock() monkeypatch.setattr("services.app_dsl_service.redis_client.delete", redis_delete) create_or_update = Mock() - service = AppDslService(sqlite_session) + service = build_app_dsl_service(sqlite_session) monkeypatch.setattr(service, "_create_or_update_app", create_or_update) def deny_before_transaction(*_args: object, **_kwargs: object) -> bool: @@ -319,7 +327,7 @@ def test_confirm_import_does_not_create_when_overwrite_target_disappeared( monkeypatch.setattr("services.app_dsl_service.RBACService.CheckAccess.check", Mock(return_value=True)) redis_delete = Mock() monkeypatch.setattr("services.app_dsl_service.redis_client.delete", redis_delete) - service = AppDslService(sqlite_session) + service = build_app_dsl_service(sqlite_session) create_or_update = Mock() monkeypatch.setattr(service, "_create_or_update_app", create_or_update) @@ -340,7 +348,9 @@ def test_pending_import_is_scoped_to_its_owner(monkeypatch: pytest.MonkeyPatch, "services.app_dsl_service.redis_client.setex", lambda key, _expiry, value: pending_imports.__setitem__(key, value), ) - service = AppDslService(session=unbound_session) + service = AppDslService( + session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + ) creator = _account() pending = service.import_app( @@ -386,7 +396,9 @@ def test_pending_import_requires_current_tenant(monkeypatch: pytest.MonkeyPatch, account = _account() account._current_tenant = None - result = AppDslService(session=unbound_session).import_app( + result = AppDslService( + session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + ).import_app( account=account, import_mode="yaml-content", yaml_content="version: 99.0.0\nkind: app\napp: {name: Test, mode: workflow}\n", @@ -451,7 +463,7 @@ def test_create_or_update_app_loads_existing_model_config_with_service_session( app = _app(app_model_config_id=app_model_config_id) with sqlite_session_factory() as service_session: - result = AppDslService(session=service_session)._create_or_update_app( + result = build_app_dsl_service(session=service_session)._create_or_update_app( app=app, data={"app": {"mode": AppMode.CHAT}, "model_config": {"model": {}}}, account=_account(), @@ -465,7 +477,7 @@ def test_create_or_update_app_loads_existing_model_config_with_service_session( def test_create_or_update_app_silently_discards_invalid_image_icon(sqlite_session: Session) -> None: app = _app(tenant_id=_TENANT_ID) - service = AppDslService(session=sqlite_session) + service = build_app_dsl_service(session=sqlite_session) result = service._create_or_update_app( app=app, @@ -502,7 +514,7 @@ def test_create_or_update_app_applies_site_settings_without_changing_access( sqlite_session.add_all([app, site]) sqlite_session.flush() - service = AppDslService(session=sqlite_session) + service = build_app_dsl_service(session=sqlite_session) service._create_or_update_app( app=app, data={ @@ -536,7 +548,9 @@ def test_create_or_update_app_rejects_null_required_site_setting_before_mutation app = _app() with pytest.raises(ValueError, match="Required Site settings cannot be null"): - AppDslService(unbound_session)._create_or_update_app( + AppDslService( + unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + )._create_or_update_app( app=app, data={"app": {"mode": AppMode.CHAT.value}, "site": {"title": None}}, account=_account(), @@ -548,12 +562,13 @@ def test_create_or_update_app_rejects_null_required_site_setting_before_mutation def test_import_app_resolves_site_entitlement_before_database_writes( monkeypatch: pytest.MonkeyPatch, unbound_session: Session ) -> None: - service = AppDslService(unbound_session) + service = AppDslService( + unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + ) create_or_update = Mock(return_value=_app()) entitlement = Mock(return_value=False) monkeypatch.setattr(service, "_create_or_update_app", create_or_update) monkeypatch.setattr("services.app_dsl_service.FeatureService.can_import_premium_site_settings", entitlement) - monkeypatch.setattr("services.app_dsl_service.WorkflowDraftVariableService", Mock()) result = service.import_app( account=_account(), @@ -602,7 +617,9 @@ def test_confirm_import_rechecks_site_entitlement(monkeypatch: pytest.MonkeyPatc monkeypatch.setattr("services.app_dsl_service.redis_client.delete", Mock()) entitlement = Mock(return_value=False) monkeypatch.setattr("services.app_dsl_service.FeatureService.can_import_premium_site_settings", entitlement) - service = AppDslService(unbound_session) + service = AppDslService( + unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + ) create_or_update = Mock(return_value=_app()) monkeypatch.setattr(service, "_create_or_update_app", create_or_update) @@ -683,7 +700,7 @@ def record_signal(*_args: object, **_kwargs: object) -> None: app = _app() try: - AppDslService(session=sqlite_session)._create_or_update_app( + build_app_dsl_service(session=sqlite_session)._create_or_update_app( app=app, data={"app": {"mode": AppMode.CHAT}, "model_config": {"model": {}}}, account=_account(account_id="22222222-2222-2222-2222-222222222222"), @@ -714,7 +731,7 @@ def test_chat_dsl_import_rejects_invalid_feature_fields( app = _app() with pytest.raises(ValueError, match=field): - AppDslService(session=sqlite_session)._create_or_update_app( + build_app_dsl_service(session=sqlite_session)._create_or_update_app( app=app, data={"app": {"mode": AppMode.CHAT}, "model_config": {feature: {"enabled": True, field: value}}}, account=_account(), @@ -731,7 +748,7 @@ def test_chat_dsl_import_preserves_valid_feature_fields(sqlite_session: Session) "text_to_speech": {"enabled": True, "voice": "alloy", "language": "en", "autoPlay": "disabled"}, } - AppDslService(session=sqlite_session)._create_or_update_app( + build_app_dsl_service(session=sqlite_session)._create_or_update_app( app=app, data={"app": {"mode": AppMode.CHAT}, "model_config": model_config}, account=_account(), @@ -744,23 +761,10 @@ def test_chat_dsl_import_preserves_valid_feature_fields(sqlite_session: Session) assert persisted.text_to_speech_dict == model_config["text_to_speech"] -def test_create_or_update_app_removes_imported_workflow_viewport(monkeypatch: pytest.MonkeyPatch) -> None: - session = cast(Session, SimpleNamespace(add=Mock(), flush=Mock(), get=Mock())) - service = AppDslService(session=session) - app = SimpleNamespace( - id="app-1", - tenant_id="tenant-1", - name="Workflow", - description="", - icon_type=IconType.EMOJI, - icon="robot", - icon_background="#FFFFFF", - ) - workflow_service = SimpleNamespace( - get_draft_workflow=Mock(return_value=None), - sync_draft_workflow=Mock(return_value=SimpleNamespace(id="workflow-1")), - ) - monkeypatch.setattr("services.app_dsl_service.WorkflowService", Mock(return_value=workflow_service)) +def test_create_or_update_app_removes_imported_workflow_viewport(unbound_session: Session) -> None: + drafts = Mock(spec=WorkflowDraftService) + service = AppDslService(session=unbound_session, drafts=drafts, overwrites=Mock(spec=AppDslOverwriteStore)) + app = make_app(mode=AppMode.WORKFLOW) imported_graph: dict[str, object] = { "nodes": [], "edges": [], @@ -768,74 +772,21 @@ def test_create_or_update_app_removes_imported_workflow_viewport(monkeypatch: py } service._create_or_update_app( - app=cast(App, app), + app=app, data={ "app": {"mode": AppMode.WORKFLOW.value}, "workflow": {"graph": imported_graph}, }, - account=Mock(id="account-1"), + account=make_account(), ) - assert workflow_service.sync_draft_workflow.call_args.kwargs["graph"] == { + assert drafts.sync.call_args.args[2].graph == { "nodes": [], "edges": [], } assert imported_graph["viewport"] == {"x": 100, "y": 200, "zoom": 1.5} -def test_create_or_update_app_forwards_imported_agent_purge_ids(monkeypatch: pytest.MonkeyPatch) -> None: - session = cast(Session, SimpleNamespace(add=Mock(), flush=Mock(), commit=Mock(), get=Mock())) - service = AppDslService(session=session) - app = SimpleNamespace( - id="app-1", - tenant_id="tenant-1", - name="Workflow", - description="", - icon_type=IconType.EMOJI, - icon="robot", - icon_background="#FFFFFF", - ) - workflow = SimpleNamespace(id="workflow-1") - workflow_service = SimpleNamespace( - get_draft_workflow=Mock(return_value=None), - sync_draft_workflow=Mock(return_value=workflow), - ) - monkeypatch.setattr("services.app_dsl_service.WorkflowService", Mock(return_value=workflow_service)) - monkeypatch.setattr( - "services.app_dsl_service.AgentDslService.graph_without_package_bindings", - Mock(return_value={"nodes": [], "edges": []}), - ) - monkeypatch.setattr( - "services.app_dsl_service.AgentDslService.import_workflow_packages", - Mock(return_value=(workflow, [], {"retired-agent"})), - ) - monkeypatch.setattr( - "services.app_dsl_service.WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync", - Mock(), - ) - retire_unowned = Mock() - monkeypatch.setattr( - "services.app_dsl_service.WorkflowAgentRetirementService.retire_unowned", - retire_unowned, - ) - - service._create_or_update_app( - app=cast(App, app), - data={ - "app": {"mode": AppMode.WORKFLOW.value}, - "workflow": {"graph": {"nodes": [], "edges": []}}, - "agent_packages": {"package-1": {}}, - }, - account=Mock(id="account-1"), - ) - - retire_unowned.assert_called_once_with( - tenant_id="tenant-1", - agent_ids={"retired-agent"}, - account_id="account-1", - ) - - def test_export_dsl_loads_model_config_and_annotation_reply_with_request_session( monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], @@ -879,7 +830,9 @@ def test_agent_import_of_new_agent_checks_function_scope( monkeypatch.setattr("services.app_dsl_service.RBACService.CheckAccess.check", check) account = _account() - AppDslService(session=unbound_session)._ensure_agent_import_permission(account, app=None) + AppDslService( + session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + )._ensure_agent_import_permission(account, app=None) check.assert_called_once_with( account.current_tenant_id, @@ -897,7 +850,9 @@ def test_create_or_update_app_gates_agent_mode_before_creation( ) -> None: config_overrides(RBAC_ENABLED=True) monkeypatch.setattr("services.app_dsl_service.RBACService.CheckAccess.check", Mock(return_value=False)) - service = AppDslService(session=unbound_session) + service = AppDslService( + session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + ) with pytest.raises(NoPermissionError): service._create_or_update_app( @@ -916,7 +871,9 @@ def test_import_app_reraises_permission_denial_instead_of_failed_result( ) -> None: config_overrides(RBAC_ENABLED=True) monkeypatch.setattr("services.app_dsl_service.RBACService.CheckAccess.check", Mock(return_value=False)) - service = AppDslService(session=unbound_session) + service = AppDslService( + session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + ) with pytest.raises(NoPermissionError): service.import_app( @@ -1016,8 +973,9 @@ def append_model(export_data: dict[str, object], *_args: object, **_kwargs: obje def test_overwrite_rejects_incompatible_nodes_before_mutation(mode: AppMode, node_type: str) -> None: session = Mock() target = App(id="target", tenant_id="tenant", mode=mode, name="Original") - service = AppDslService(session) - service._load_app_for_overwrite = Mock(return_value=target) + overwrites = Mock(spec=AppDslOverwriteStore) + overwrites.snapshot.return_value = AppDslOverwriteTarget(mode, None) + service = AppDslService(session, drafts=Mock(spec=WorkflowDraftService), overwrites=overwrites) result = service.import_app( account=_account(), import_mode="yaml-content", @@ -1036,6 +994,7 @@ def test_overwrite_rejects_incompatible_nodes_before_mutation(mode: AppMode, nod assert "incompatible" in result.error assert target.name == "Original" session.add.assert_not_called() + overwrites.load.assert_not_called() @pytest.mark.parametrize( @@ -1050,9 +1009,9 @@ def test_overwrite_rejects_incompatible_nodes_before_mutation(mode: AppMode, nod def test_missing_app_section_names_the_keys_that_were_present( unbound_session: Session, content: dict[str, object], found: str ) -> None: - result = AppDslService(session=unbound_session).import_app( - account=_account(), import_mode="yaml-content", yaml_content=yaml.safe_dump(content, sort_keys=False) - ) + result = AppDslService( + session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + ).import_app(account=_account(), import_mode="yaml-content", yaml_content=yaml.safe_dump(content, sort_keys=False)) assert result.status == ImportStatus.FAILED assert result.error is not None assert result.error.startswith("Missing app data in YAML content.") diff --git a/api/tests/unit_tests/services/test_app_generate_service.py b/api/tests/unit_tests/services/test_app_generate_service.py index 5b20675c9c83b1..4f5e6df3c5bd46 100644 --- a/api/tests/unit_tests/services/test_app_generate_service.py +++ b/api/tests/unit_tests/services/test_app_generate_service.py @@ -10,7 +10,6 @@ - generate_single_loop (ADVANCED_CHAT / WORKFLOW / invalid mode) - generate_more_like_this - _get_workflow (debugger / non-debugger / specific id / invalid format / not found) - - get_response_generator (ended / non-ended workflow run) """ import json @@ -27,17 +26,18 @@ import services.app_generate_service as ags_module from core.app.entities.app_invoke_entities import InvokeFrom from enums import DeploymentEdition, QuotaType -from graphon.enums import WorkflowExecutionStatus from models.account import Account -from models.enums import AppStatus, CreatorUserRole +from models.enums import AppStatus from models.model import App, AppMode -from models.workflow import Workflow, WorkflowRun, WorkflowRunTriggeredFrom, WorkflowType +from models.workflow import Workflow, WorkflowType from services.app_generate_service import AppGenerateService from services.errors.app import ( TriggerWorkflowServiceModeUnavailableError, WorkflowIdFormatError, WorkflowNotFoundError, ) +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables # --------------------------------------------------------------------------- @@ -99,8 +99,8 @@ def _make_user() -> Account: class _RealSessionTest: @pytest.fixture(autouse=True) - def _bind_unbound_session(self, unbound_session: Session) -> None: - self.session = unbound_session + def _bind_session(self, sqlite_session: Session) -> None: + self.session = sqlite_session def _make_workflow( @@ -123,29 +123,13 @@ def _make_workflow( "edges": [], } ), - features={}, + features="{}", created_by=created_by, environment_variables=[], conversation_variables=[], ) -def _make_workflow_run(*, run_id: str, ended: bool) -> WorkflowRun: - run = WorkflowRun( - tenant_id="tenant-id", - app_id="app-id", - workflow_id="workflow-id", - type=WorkflowType.WORKFLOW, - triggered_from=WorkflowRunTriggeredFrom.APP_RUN, - version="published", - status=WorkflowExecutionStatus.SUCCEEDED if ended else WorkflowExecutionStatus.RUNNING, - created_by_role=CreatorUserRole.ACCOUNT, - created_by="user-id", - ) - run.id = run_id - return run - - @contextmanager def _noop_rate_limit_context(rate_limit, request_id): """Drop-in replacement for rate_limit_context that doesn't touch Redis.""" @@ -325,13 +309,95 @@ def _common(self, mocker: MockerFixture, config_overrides: Callable[..., None]): ) # -- COMPLETION --------------------------------------------------------- - def test_completion_mode(self, mocker: MockerFixture): + @pytest.mark.parametrize("mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]) + @pytest.mark.parametrize("entry", ["generate", "iteration", "loop"]) + def test_debug_generators_use_injected_loader_and_saver( + self, + monkeypatch: pytest.MonkeyPatch, + workflow_variables: WorkflowExecutionVariables, + mode: AppMode, + entry: str, + *, + workflow_runtime: WorkflowRuntime, + ) -> None: + from graphon.enums import BuiltinNodeTypes + from services.app.generation.input_adapter import AppInputAdapter + from services.workflow.execution.adapters.chatflow.app_generator import AdvancedChatAppGenerator + from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator + + workflow = _make_workflow() + user = _make_user() + monkeypatch.setattr(AppGenerateService, "_get_workflow", lambda *_args, **_kwargs: workflow) + calls: list[str] = [] + + def generate(generator: AppInputAdapter, **_kwargs: object) -> dict[str, str]: + # Keep the real generator constructor and debugger saver selection. + assert generator._draft_variable_loader == workflow_variables.workflow_loader + assert generator._draft_variable_saver == workflow_variables.saver_factory + loader = workflow_variables.workflow_loader(workflow, user.id) + saver = generator._get_draft_var_saver_factory(InvokeFrom.DEBUGGER, user, tenant_id=workflow.tenant_id) + saver(workflow.app_id, "node", BuiltinNodeTypes.CODE, "execution").save( + process_data=None, outputs={"text": "injected value"} + ) + assert loader.load_variables([["node", "text"]])[0].value == "injected value" + calls.append(entry) + return {"result": "ready"} + + generator_type = WorkflowAppGenerator if mode == AppMode.WORKFLOW else AdvancedChatAppGenerator + method = {"generate": "generate", "iteration": "single_iteration_generate", "loop": "single_loop_generate"}[ + entry + ] + monkeypatch.setattr(generator_type, method, generate) + app = _make_app(mode) + if entry == "generate": + result = AppGenerateService.generate( + app_model=app, + user=user, + args={}, + invoke_from=InvokeFrom.DEBUGGER, + session=self.session, + streaming=False, + variables=workflow_variables, + runtime=workflow_runtime, + ) + elif entry == "iteration": + result = AppGenerateService.generate_single_iteration( + workflow=workflow, + app_model=app, + user=user, + node_id="node", + args={}, + streaming=False, + variables=workflow_variables, + runtime=workflow_runtime, + ) + else: + result = AppGenerateService.generate_single_loop( + workflow=workflow, + app_model=app, + user=user, + node_id="node", + args={}, + streaming=False, + variables=workflow_variables, + runtime=workflow_runtime, + ) + assert result == {"result": "ready"} + assert calls == [entry] + + def test_completion_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): gen_spy = mocker.patch( "services.app_generate_service.CompletionAppGenerator.generate", return_value={"result": "ok"}, ) mocker.patch( - "services.app_generate_service.CompletionAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) result = AppGenerateService.generate( @@ -341,18 +407,26 @@ def test_completion_mode(self, mocker: MockerFixture): invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) assert result == {"result": "ok"} gen_spy.assert_called_once() # -- AGENT_CHAT via mode ------------------------------------------------ - def test_agent_chat_mode(self, mocker: MockerFixture): + def test_agent_chat_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): gen_spy = mocker.patch( "services.app_generate_service.AgentChatAppGenerator.generate", return_value={"result": "agent"}, ) mocker.patch( - "services.app_generate_service.AgentChatAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) result = AppGenerateService.generate( @@ -362,18 +436,26 @@ def test_agent_chat_mode(self, mocker: MockerFixture): invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) assert result == {"result": "agent"} gen_spy.assert_called_once() # -- AGENT_CHAT via is_agent flag (non-AGENT_CHAT mode) ----------------- - def test_agent_via_is_agent_flag(self, mocker: MockerFixture): + def test_agent_via_is_agent_flag( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): gen_spy = mocker.patch( "services.app_generate_service.AgentChatAppGenerator.generate", return_value={"result": "agent-via-flag"}, ) mocker.patch( - "services.app_generate_service.AgentChatAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) app = _make_app(AppMode.CHAT) @@ -386,19 +468,27 @@ def test_agent_via_is_agent_flag(self, mocker: MockerFixture): invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=session, + variables=workflow_variables, + runtime=workflow_runtime, ) assert result == {"result": "agent-via-flag"} gen_spy.assert_called_once() is_agent.assert_called_once_with(session=session) # -- AGENT -------------------------------------------------------------- - def test_agent_mode_passes_session(self, mocker: MockerFixture): + def test_agent_mode_passes_session( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): gen_spy = mocker.patch( "services.app_generate_service.AgentAppGenerator.generate", return_value={"result": "agent"}, ) mocker.patch( - "services.app_generate_service.AgentAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) session = self.session @@ -410,19 +500,26 @@ def test_agent_mode_passes_session(self, mocker: MockerFixture): invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=session, + variables=workflow_variables, + runtime=workflow_runtime, ) assert result == {"result": "agent"} - assert gen_spy.call_args.kwargs["session"] is session # -- CHAT --------------------------------------------------------------- - def test_chat_mode(self, mocker: MockerFixture): + def test_chat_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): gen_spy = mocker.patch( "services.app_generate_service.ChatAppGenerator.generate", return_value={"result": "chat"}, ) mocker.patch( - "services.app_generate_service.ChatAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) app = _make_app(AppMode.CHAT) @@ -433,22 +530,30 @@ def test_chat_mode(self, mocker: MockerFixture): invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) assert result == {"result": "chat"} gen_spy.assert_called_once() # -- ADVANCED_CHAT blocking --------------------------------------------- - def test_advanced_chat_blocking(self, mocker: MockerFixture): + def test_advanced_chat_blocking( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) - retrieve_spy = mocker.patch("services.app_generate_service.AdvancedChatAppGenerator.retrieve_events") + retrieve_spy = mocker.patch("services.app_generate_service.WorkflowEventStream.retrieve_events") gen_spy = mocker.patch( "services.app_generate_service.AdvancedChatAppGenerator.generate", return_value={"result": "advanced-blocking"}, ) mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) @@ -460,15 +565,24 @@ def test_advanced_chat_blocking(self, mocker: MockerFixture): invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=session, + variables=workflow_variables, + runtime=workflow_runtime, ) assert result == {"result": "advanced-blocking"} call_kwargs = gen_spy.call_args.kwargs assert call_kwargs.get("streaming") is False - assert call_kwargs["session"] is session + assert "session" not in call_kwargs retrieve_spy.assert_not_called() # -- ADVANCED_CHAT streaming -------------------------------------------- - def test_advanced_chat_streaming(self, mocker: MockerFixture, config_overrides: Callable[..., None]): + def test_advanced_chat_streaming( + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): config_overrides(PUBSUB_REDIS_CHANNEL_TYPE="streams") workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) @@ -481,8 +595,8 @@ def test_advanced_chat_streaming(self, mocker: MockerFixture, config_overrides: gen_instance.retrieve_events.return_value = iter([]) gen_instance.convert_to_event_stream.side_effect = lambda x: x mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator", - return_value=gen_instance, + "services.app_generate_service.WorkflowEventStream", + new=gen_instance, ) result = AppGenerateService.generate( @@ -492,6 +606,8 @@ def test_advanced_chat_streaming(self, mocker: MockerFixture, config_overrides: invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) # In streaming mode it should go through retrieve_events, not generate gen_instance.retrieve_events.assert_called_once() @@ -502,7 +618,13 @@ def test_advanced_chat_streaming(self, mocker: MockerFixture, config_overrides: delay_spy.assert_called_once() # -- WORKFLOW blocking -------------------------------------------------- - def test_workflow_blocking(self, mocker: MockerFixture): + def test_workflow_blocking( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) gen_spy = mocker.patch( @@ -510,7 +632,7 @@ def test_workflow_blocking(self, mocker: MockerFixture): return_value={"result": "workflow-blocking"}, ) mocker.patch( - "services.app_generate_service.WorkflowAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) @@ -522,6 +644,8 @@ def test_workflow_blocking(self, mocker: MockerFixture): invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=session, + variables=workflow_variables, + runtime=workflow_runtime, ) assert result == {"result": "workflow-blocking"} call_kwargs = gen_spy.call_args.kwargs @@ -538,6 +662,9 @@ def test_trigger_workflow_rejects_manual_service_surfaces( invoke_from: InvokeFrom, node_type: str, mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, ) -> None: workflow = _make_workflow(node_types=(node_type,)) mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) @@ -551,11 +678,19 @@ def test_trigger_workflow_rejects_manual_service_surfaces( invoke_from=invoke_from, streaming=False, session=MagicMock(), + variables=workflow_variables, + runtime=workflow_runtime, ) generate.assert_not_called() - def test_trigger_workflow_allows_trigger_execution(self, mocker: MockerFixture) -> None: + def test_trigger_workflow_allows_trigger_execution( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ) -> None: workflow = _make_workflow(node_types=("trigger-webhook",)) mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) generate = mocker.patch( @@ -563,7 +698,7 @@ def test_trigger_workflow_allows_trigger_execution(self, mocker: MockerFixture) return_value={"result": "trigger"}, ) mocker.patch( - "services.app_generate_service.WorkflowAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda value: value, ) @@ -574,12 +709,20 @@ def test_trigger_workflow_allows_trigger_execution(self, mocker: MockerFixture) invoke_from=InvokeFrom.TRIGGER, streaming=False, session=MagicMock(), + variables=workflow_variables, + runtime=workflow_runtime, ) assert result == {"result": "trigger"} generate.assert_called_once() - def test_specific_start_workflow_version_remains_runnable(self, mocker: MockerFixture) -> None: + def test_specific_start_workflow_version_remains_runnable( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ) -> None: workflow_id = str(uuid.uuid4()) workflow = _make_workflow(workflow_id=workflow_id, node_types=("start",)) get_workflow = mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) @@ -588,7 +731,7 @@ def test_specific_start_workflow_version_remains_runnable(self, mocker: MockerFi return_value={"result": "version"}, ) mocker.patch( - "services.app_generate_service.WorkflowAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda value: value, ) app = _make_app(AppMode.WORKFLOW) @@ -601,13 +744,22 @@ def test_specific_start_workflow_version_remains_runnable(self, mocker: MockerFi invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=session, + variables=workflow_variables, + runtime=workflow_runtime, ) assert result == {"result": "version"} - get_workflow.assert_called_once_with(app, InvokeFrom.SERVICE_API, workflow_id, session=session) + get_workflow.assert_called_once_with(app, InvokeFrom.SERVICE_API, workflow_id, session=session, snapshot=None) # -- WORKFLOW streaming ------------------------------------------------- - def test_workflow_streaming(self, mocker: MockerFixture, config_overrides: Callable[..., None]): + def test_workflow_streaming( + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): config_overrides(PUBSUB_REDIS_CHANNEL_TYPE="streams") workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) @@ -617,11 +769,11 @@ def test_workflow_streaming(self, mocker: MockerFixture, config_overrides: Calla ) delay_spy = mocker.patch("services.app_generate_service.workflow_based_app_execution_task.delay") retrieve_spy = mocker.patch( - "services.app_generate_service.MessageBasedAppGenerator.retrieve_events", + "services.app_generate_service.WorkflowEventStream.retrieve_events", return_value=iter([]), ) mocker.patch( - "services.app_generate_service.WorkflowAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) @@ -632,6 +784,8 @@ def test_workflow_streaming(self, mocker: MockerFixture, config_overrides: Calla invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) retrieve_spy.assert_called_once() # Dispatch is gated on subscribe; simulate the SSE layer entering the @@ -641,7 +795,13 @@ def test_workflow_streaming(self, mocker: MockerFixture, config_overrides: Calla delay_spy.assert_called_once() # -- Invalid mode ------------------------------------------------------- - def test_invalid_mode_raises(self, mocker: MockerFixture): + def test_invalid_mode_raises( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): app = _make_app("invalid-mode") with pytest.raises(ValueError, match="Invalid app mode"): AppGenerateService.generate( @@ -651,6 +811,8 @@ def test_invalid_mode_raises(self, mocker: MockerFixture): invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) @@ -666,7 +828,14 @@ def _common(self, mocker: MockerFixture): _noop_rate_limit_context, ) - def test_cloud_edition_consumes_quota(self, mocker: MockerFixture, config_overrides: Callable[..., None]): + def test_cloud_edition_consumes_quota( + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) quota_charge = MagicMock() reserve_mock = mocker.patch( @@ -678,7 +847,7 @@ def test_cloud_edition_consumes_quota(self, mocker: MockerFixture, config_overri return_value={"ok": True}, ) mocker.patch( - "services.app_generate_service.CompletionAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) @@ -689,12 +858,19 @@ def test_cloud_edition_consumes_quota(self, mocker: MockerFixture, config_overri invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) reserve_mock.assert_called_once_with(QuotaType.WORKFLOW, "tenant-id") quota_charge.commit.assert_called_once() def test_billing_quota_exceeded_raises_rate_limit_error( - self, mocker: MockerFixture, config_overrides: Callable[..., None] + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, ): from services.errors.app import QuotaExceededError from services.errors.llm import InvokeRateLimitError @@ -713,10 +889,17 @@ def test_billing_quota_exceeded_raises_rate_limit_error( invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) def test_exception_refunds_quota_and_exits_rate_limit( - self, mocker: MockerFixture, config_overrides: Callable[..., None] + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, ): config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) quota_charge = MagicMock() @@ -729,7 +912,7 @@ def test_exception_refunds_quota_and_exits_rate_limit( side_effect=RuntimeError("boom"), ) mocker.patch( - "services.app_generate_service.CompletionAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) @@ -741,11 +924,18 @@ def test_exception_refunds_quota_and_exits_rate_limit( invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) quota_charge.refund.assert_called_once() def test_rate_limit_exit_called_in_finally_for_blocking( - self, mocker: MockerFixture, config_overrides: Callable[..., None] + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, ): """For non-streaming (blocking) calls, rate_limit.exit should be called in finally.""" config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) @@ -762,7 +952,7 @@ def exit(self, request_id: str) -> None: return_value={"ok": True}, ) mocker.patch( - "services.app_generate_service.CompletionAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) @@ -773,11 +963,20 @@ def exit(self, request_id: str) -> None: invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) # exit is called in finally block for non-streaming assert exit_calls == ["dummy-request-id"] - def test_blocking_failure_exits_rate_limit_once(self, mocker: MockerFixture, config_overrides: Callable[..., None]): + def test_blocking_failure_exits_rate_limit_once( + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) quota_charge = MagicMock() mocker.patch( @@ -804,13 +1003,20 @@ def exit(self, request_id: str) -> None: invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) quota_charge.refund.assert_called_once() assert exit_calls == ["dummy-request-id"] def test_streaming_failure_exits_rate_limit_once( - self, mocker: MockerFixture, config_overrides: Callable[..., None] + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, ): config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) quota_charge = MagicMock() @@ -838,6 +1044,8 @@ def exit(self, request_id: str) -> None: invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) quota_charge.refund.assert_called_once() @@ -848,86 +1056,64 @@ def exit(self, request_id: str) -> None: # _get_workflow # --------------------------------------------------------------------------- class TestGetWorkflow(_RealSessionTest): - def test_debugger_fetches_draft(self, mocker: MockerFixture): - draft_wf = _make_workflow() - ws = MagicMock() - ws.get_draft_workflow.return_value = draft_wf - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) - - result = AppGenerateService._get_workflow( - _make_app(AppMode.WORKFLOW), InvokeFrom.DEBUGGER, session=self.session + def test_debugger_fetches_draft(self): + workflow = _make_workflow() + self.session.add(workflow) + self.session.commit() + assert ( + AppGenerateService._get_workflow(_make_app(AppMode.WORKFLOW), InvokeFrom.DEBUGGER, session=self.session) + is workflow ) - assert result is draft_wf - ws.get_draft_workflow.assert_called_once() - - def test_debugger_raises_when_no_draft(self, mocker: MockerFixture): - ws = MagicMock() - ws.get_draft_workflow.return_value = None - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) + def test_debugger_raises_when_no_draft(self): with pytest.raises(ValueError, match="Workflow not initialized"): AppGenerateService._get_workflow(_make_app(AppMode.WORKFLOW), InvokeFrom.DEBUGGER, session=self.session) - def test_non_debugger_fetches_published(self, mocker: MockerFixture): - pub_wf = _make_workflow() - ws = MagicMock() - ws.get_published_workflow.return_value = pub_wf - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) - - result = AppGenerateService._get_workflow( - _make_app(AppMode.WORKFLOW), InvokeFrom.SERVICE_API, session=self.session - ) - assert result is pub_wf - ws.get_published_workflow.assert_called_once() - - def test_non_debugger_raises_when_no_published(self, mocker: MockerFixture): - ws = MagicMock() - ws.get_published_workflow.return_value = None - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) + def test_non_debugger_fetches_published(self): + workflow = _make_workflow() + workflow.version = "published" + self.session.add(workflow) + self.session.commit() + app = _make_app(AppMode.WORKFLOW) + app.workflow_id = workflow.id + assert AppGenerateService._get_workflow(app, InvokeFrom.SERVICE_API, session=self.session) is workflow + def test_non_debugger_raises_when_no_published(self): with pytest.raises(ValueError, match="Workflow not published"): AppGenerateService._get_workflow(_make_app(AppMode.WORKFLOW), InvokeFrom.SERVICE_API, session=self.session) - def test_specific_workflow_id_valid_uuid(self, mocker: MockerFixture): - valid_uuid = str(uuid.uuid4()) - specific_wf = _make_workflow(workflow_id=valid_uuid) - ws = MagicMock() - ws.get_published_workflow_by_id.return_value = specific_wf - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) - - result = AppGenerateService._get_workflow( - _make_app(AppMode.WORKFLOW), - InvokeFrom.SERVICE_API, - workflow_id=valid_uuid, - session=self.session, + def test_specific_workflow_id_valid_uuid(self): + workflow = _make_workflow(workflow_id=str(uuid.uuid4())) + workflow.version = "published" + self.session.add(workflow) + self.session.commit() + assert ( + AppGenerateService._get_workflow( + _make_app(AppMode.WORKFLOW), InvokeFrom.SERVICE_API, workflow_id=workflow.id, session=self.session + ) + is workflow ) - assert result is specific_wf - ws.get_published_workflow_by_id.assert_called_once() - - def test_specific_workflow_id_invalid_uuid(self, mocker: MockerFixture): - ws = MagicMock() - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) + def test_specific_workflow_id_invalid_uuid(self): with pytest.raises(WorkflowIdFormatError): AppGenerateService._get_workflow( - _make_app(AppMode.WORKFLOW), - InvokeFrom.SERVICE_API, - workflow_id="not-a-uuid", - session=self.session, + _make_app(AppMode.WORKFLOW), InvokeFrom.SERVICE_API, workflow_id="not-a-uuid", session=self.session ) - def test_specific_workflow_id_not_found(self, mocker: MockerFixture): - valid_uuid = str(uuid.uuid4()) - ws = MagicMock() - ws.get_published_workflow_by_id.return_value = None - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) - + @pytest.mark.parametrize("owner", ["missing", "other-tenant", "other-app"]) + def test_specific_workflow_id_not_found(self, owner: str): + workflow = _make_workflow(workflow_id=str(uuid.uuid4())) + workflow.version = "published" + if owner != "missing": + if owner == "other-tenant": + workflow.tenant_id = owner + else: + workflow.app_id = owner + self.session.add(workflow) + self.session.commit() with pytest.raises(WorkflowNotFoundError): AppGenerateService._get_workflow( - _make_app(AppMode.WORKFLOW), - InvokeFrom.SERVICE_API, - workflow_id=valid_uuid, - session=self.session, + _make_app(AppMode.WORKFLOW), InvokeFrom.SERVICE_API, workflow_id=workflow.id, session=self.session ) @@ -935,11 +1121,17 @@ def test_specific_workflow_id_not_found(self, mocker: MockerFixture): # generate_single_iteration # --------------------------------------------------------------------------- class TestGenerateSingleIteration(_RealSessionTest): - def test_advanced_chat_mode(self, mocker: MockerFixture): + def test_advanced_chat_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) gen_spy = mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) iter_spy = mocker.patch( @@ -949,21 +1141,28 @@ def test_advanced_chat_mode(self, mocker: MockerFixture): app = _make_app(AppMode.ADVANCED_CHAT) session = self.session result = AppGenerateService.generate_single_iteration( + workflow=workflow, app_model=app, user=_make_user(), node_id="n1", args={"k": "v"}, - session=session, + variables=workflow_variables, + runtime=workflow_runtime, ) iter_spy.assert_called_once() - assert iter_spy.call_args.kwargs["session"] is session assert result == {"event": "iteration"} - def test_workflow_mode(self, mocker: MockerFixture): + def test_workflow_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) iter_spy = mocker.patch( @@ -973,21 +1172,35 @@ def test_workflow_mode(self, mocker: MockerFixture): app = _make_app(AppMode.WORKFLOW) session = self.session result = AppGenerateService.generate_single_iteration( + workflow=workflow, app_model=app, user=_make_user(), node_id="n1", args={"k": "v"}, - session=session, + variables=workflow_variables, + runtime=workflow_runtime, ) iter_spy.assert_called_once() - assert iter_spy.call_args.kwargs["session"] is session + assert "session" not in iter_spy.call_args.kwargs assert result == {"event": "wf-iteration"} - def test_invalid_mode_raises(self, mocker: MockerFixture): + def test_invalid_mode_raises( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): app = _make_app(AppMode.CHAT) with pytest.raises(ValueError, match="Invalid app mode"): AppGenerateService.generate_single_iteration( - app_model=app, user=_make_user(), node_id="n1", args={}, session=self.session + workflow=_make_workflow(), + app_model=app, + user=_make_user(), + node_id="n1", + args={}, + variables=workflow_variables, + runtime=workflow_runtime, ) @@ -995,11 +1208,17 @@ def test_invalid_mode_raises(self, mocker: MockerFixture): # generate_single_loop # --------------------------------------------------------------------------- class TestGenerateSingleLoop(_RealSessionTest): - def test_advanced_chat_mode(self, mocker: MockerFixture): + def test_advanced_chat_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) loop_spy = mocker.patch( @@ -1009,21 +1228,28 @@ def test_advanced_chat_mode(self, mocker: MockerFixture): app = _make_app(AppMode.ADVANCED_CHAT) session = self.session result = AppGenerateService.generate_single_loop( + workflow=workflow, app_model=app, user=_make_user(), node_id="n1", args=MagicMock(), - session=session, + variables=workflow_variables, + runtime=workflow_runtime, ) loop_spy.assert_called_once() - assert loop_spy.call_args.kwargs["session"] is session assert result == {"event": "loop"} - def test_workflow_mode(self, mocker: MockerFixture): + def test_workflow_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.convert_to_event_stream", + "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, ) loop_spy = mocker.patch( @@ -1033,21 +1259,35 @@ def test_workflow_mode(self, mocker: MockerFixture): app = _make_app(AppMode.WORKFLOW) session = self.session result = AppGenerateService.generate_single_loop( + workflow=workflow, app_model=app, user=_make_user(), node_id="n1", args=MagicMock(), - session=session, + variables=workflow_variables, + runtime=workflow_runtime, ) loop_spy.assert_called_once() - assert loop_spy.call_args.kwargs["session"] is session + assert "session" not in loop_spy.call_args.kwargs assert result == {"event": "wf-loop"} - def test_invalid_mode_raises(self, mocker: MockerFixture): + def test_invalid_mode_raises( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): app = _make_app(AppMode.COMPLETION) with pytest.raises(ValueError, match="Invalid app mode"): AppGenerateService.generate_single_loop( - app_model=app, user=_make_user(), node_id="n1", args=MagicMock(), session=self.session + workflow=_make_workflow(), + app_model=app, + user=_make_user(), + node_id="n1", + args=MagicMock(), + variables=workflow_variables, + runtime=workflow_runtime, ) @@ -1055,13 +1295,18 @@ def test_invalid_mode_raises(self, mocker: MockerFixture): # generate_more_like_this # --------------------------------------------------------------------------- class TestGenerateMoreLikeThis(_RealSessionTest): - def test_delegates_to_completion_generator(self, mocker: MockerFixture): + def test_delegates_to_completion_generator( + self, mocker: MockerFixture, app_records, annotation_replies, workflow_runtime + ): gen_spy = mocker.patch( "services.app_generate_service.CompletionAppGenerator.generate_more_like_this", return_value={"result": "similar"}, ) session = self.session result = AppGenerateService.generate_more_like_this( + retrieval=workflow_runtime.retrieval, + records=app_records, + annotations=annotation_replies, app_model=_make_app(AppMode.COMPLETION), user=_make_user(), message_id="msg-1", @@ -1071,42 +1316,4 @@ def test_delegates_to_completion_generator(self, mocker: MockerFixture): ) assert result == {"result": "similar"} gen_spy.assert_called_once() - assert gen_spy.call_args.kwargs["session"] is session assert gen_spy.call_args.kwargs["stream"] is True - - -# --------------------------------------------------------------------------- -# get_response_generator -# --------------------------------------------------------------------------- -class TestGetResponseGenerator: - def test_non_ended_workflow_run(self, mocker: MockerFixture): - app = _make_app(AppMode.ADVANCED_CHAT) - workflow_run = _make_workflow_run(run_id="run-1", ended=False) - - gen_instance = MagicMock() - gen_instance.retrieve_events.return_value = iter([{"event": "started"}]) - gen_instance.convert_to_event_stream.side_effect = lambda x: x - mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator", - return_value=gen_instance, - ) - - result = AppGenerateService.get_response_generator(app_model=app, workflow_run=workflow_run) - gen_instance.retrieve_events.assert_called_once() - - def test_ended_workflow_run_still_returns_generator(self, mocker: MockerFixture): - """Even when the run is ended, the current code still returns a generator (TODO branch).""" - app = _make_app(AppMode.WORKFLOW) - workflow_run = _make_workflow_run(run_id="run-2", ended=True) - - gen_instance = MagicMock() - gen_instance.retrieve_events.return_value = iter([]) - gen_instance.convert_to_event_stream.side_effect = lambda x: x - mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator", - return_value=gen_instance, - ) - - result = AppGenerateService.get_response_generator(app_model=app, workflow_run=workflow_run) - # current impl falls through the TODO and still creates a generator - gen_instance.retrieve_events.assert_called_once() diff --git a/api/tests/unit_tests/services/test_app_generate_service_streaming_integration.py b/api/tests/unit_tests/services/test_app_generate_service_streaming_integration.py index 1595a1db952969..ace28aedcb872a 100644 --- a/api/tests/unit_tests/services/test_app_generate_service_streaming_integration.py +++ b/api/tests/unit_tests/services/test_app_generate_service_streaming_integration.py @@ -5,10 +5,9 @@ import pytest -from core.app.apps.message_based_app_generator import MessageBasedAppGenerator -from core.app.apps.message_generator import MessageGenerator from models.model import AppMode from services.app_generate_service import AppGenerateService +from services.workflow.execution.adapters.response_stream import WorkflowEventStream from tests.unit_tests.config_override import apply_config_overrides @@ -111,10 +110,14 @@ def _patch_get_channel_streams(monkeypatch: pytest.MonkeyPatch): def _get_channel(): return chan - # Patch the source and the imported aliases used by MessageGenerator / MessageBasedAppGenerator + # Patch the source and the imported aliases used by WorkflowEventStream / MessageBasedAppGenerator monkeypatch.setattr("extensions.ext_redis.get_pubsub_broadcast_channel", lambda: chan) - monkeypatch.setattr("core.app.apps.message_generator.get_pubsub_broadcast_channel", lambda: chan) - monkeypatch.setattr("core.app.apps.message_based_app_generator.get_pubsub_broadcast_channel", lambda: chan) + monkeypatch.setattr( + "services.workflow.execution.adapters.response_stream.get_pubsub_broadcast_channel", lambda: chan + ) + monkeypatch.setattr( + "services.workflow.execution.adapters.response_stream.get_pubsub_broadcast_channel", lambda: chan + ) apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE="streams") @@ -129,15 +132,17 @@ def _patch_get_channel_pubsub(monkeypatch: pytest.MonkeyPatch): def _get_channel(): return chan - # Patch both the source and the imported alias used by MessageGenerator + # Patch both the source and the imported alias used by WorkflowEventStream monkeypatch.setattr("extensions.ext_redis.get_pubsub_broadcast_channel", lambda: chan) - monkeypatch.setattr("core.app.apps.message_generator.get_pubsub_broadcast_channel", lambda: chan) + monkeypatch.setattr( + "services.workflow.execution.adapters.response_stream.get_pubsub_broadcast_channel", lambda: chan + ) apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE="pubsub") def _publish_events(app_mode: AppMode, run_id: str, events: list[dict]): - # Publish events to the same topic used by MessageGenerator - topic = MessageGenerator.get_response_topic(app_mode, run_id) + # Publish events to the same topic used by WorkflowEventStream + topic = WorkflowEventStream.get_response_topic(app_mode, run_id) for ev in events: topic.publish(json.dumps(ev).encode()) @@ -164,7 +169,7 @@ def start_task(): # ADVANCED_CHAT streaming runs through in production. on_subscribe = AppGenerateService._build_streaming_task_on_subscribe(start_task) - gen = MessageBasedAppGenerator.retrieve_events(app_mode, run_id, idle_timeout=2.0, on_subscribe=on_subscribe) + gen = WorkflowEventStream.retrieve_events(app_mode, run_id, idle_timeout=2.0, on_subscribe=on_subscribe) received = [] for msg in gen: @@ -199,7 +204,7 @@ def start_task(): # Producer not started yet; only when subscribe happens assert published_order == [] - gen = MessageGenerator.retrieve_events(app_mode, run_id, idle_timeout=2.0, on_subscribe=on_subscribe) + gen = WorkflowEventStream.retrieve_events(app_mode, run_id, idle_timeout=2.0, on_subscribe=on_subscribe) received = [] for msg in gen: diff --git a/api/tests/unit_tests/services/test_app_preview_details_adapters.py b/api/tests/unit_tests/services/test_app_preview_details_adapters.py index d83a24d2859265..8fad993c761d17 100644 --- a/api/tests/unit_tests/services/test_app_preview_details_adapters.py +++ b/api/tests/unit_tests/services/test_app_preview_details_adapters.py @@ -14,14 +14,13 @@ from sqlalchemy.orm import Session, SessionTransaction, UOWTransaction, sessionmaker from sqlalchemy.pool import QueuePool -import core.agent.tool_configuration as tool_configuration_module import core.helper.tool_parameter_cache as cache_module +import services.tools.agent_configuration as tool_configuration_module from core.agent.entities import AgentToolEntity from core.helper import encrypter from core.plugin.plugin_service import PluginService from core.tools.__base.tool import Tool from core.tools.entities.tool_entities import ApiProviderSchemaType, ToolEntity, ToolParameter -from core.tools.tool_manager import ToolManager from extensions.ext_database import db from graphon.variables import SecretVariable from models import Account, App @@ -37,6 +36,7 @@ from services.app_preview_details_adapters import AppPreviewDetailsRuntime from services.app_preview_details_service import AppPreviewDetail from services.app_preview_query_service import AppPreviewRef, AppPreviewSiteUnavailableError, AppPreviewUnavailableError +from services.tools.tool_manager import ToolManager @dataclass(frozen=True) @@ -83,7 +83,9 @@ def assert_read_only(self) -> None: @pytest.fixture -def harness(sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session]) -> Iterator[_Harness]: +def harness( + sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], *, tool_providers, workflow_queries +) -> Iterator[_Harness]: target = App( id=str(uuid4()), tenant_id=str(uuid4()), name="Preview", mode="chat", enable_site=True, enable_api=True ) @@ -193,7 +195,11 @@ def after_begin(_session: Session, _transaction: SessionTransaction, connection: decoy_workflow, sqlite_session_factory, sqlite_engine, - AppPreviewDetailsRuntime(details=AppPreviewQueryRepository(session_factory=factory)), + AppPreviewDetailsRuntime( + details=AppPreviewQueryRepository(session_factory=factory), + tool_providers=tool_providers, + workflow_queries=workflow_queries, + ), closed, mutated, commits, @@ -553,7 +559,12 @@ def setex(self, key: str, expiry: int, value: str) -> None: @pytest.mark.parametrize("failure", ["none", "runtime", "decrypt"]) def test_agent_tool_masking_returns_a_copy_without_dirtying_or_flushing_models( - harness: _Harness, monkeypatch: pytest.MonkeyPatch, failure: Literal["none", "runtime", "decrypt"] + harness: _Harness, + monkeypatch: pytest.MonkeyPatch, + failure: Literal["none", "runtime", "decrypt"], + *, + tool_providers, + workflow_queries, ) -> None: provider_id = str(uuid4()) agent_mode = { @@ -586,7 +597,10 @@ def test_agent_tool_masking_returns_a_copy_without_dirtying_or_flushing_models( calls: list[str] = [] checkedout_during_io: list[int] = [] - def runtime(*, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str) -> Tool: + def runtime( + *, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str, tool_providers, workflow_queries + ) -> Tool: + del workflow_queries, tool_providers assert isinstance(harness.engine.pool, QueuePool) checkedout_during_io.append(harness.engine.pool.checkedout()) harness.assert_closed() @@ -621,6 +635,8 @@ def decrypt(tenant_id: str, token: str) -> str: app_id=harness.target.id, tenant_id=harness.active_workspace.id, user_id=harness.account.id, + tool_providers=tool_providers, + workflow_queries=workflow_queries, ) assert result.model_config is not None masked_agent_mode = result.model_config["agent_mode"] diff --git a/api/tests/unit_tests/services/test_app_service.py b/api/tests/unit_tests/services/test_app_service.py index 5e5a03505832c8..19652b31c6d68c 100644 --- a/api/tests/unit_tests/services/test_app_service.py +++ b/api/tests/unit_tests/services/test_app_service.py @@ -12,17 +12,19 @@ from sqlalchemy.orm import Session, sessionmaker from enums import DeploymentEdition +from enums.agent import WorkflowAgentBindingType from graphon.model_runtime.entities.model_entities import ModelType from models import Account, Tenant from models.account import TenantAccountJoin, TenantAccountRole from models.agent import ( Agent, AgentConfigSnapshot, + AgentHomeSnapshot, AgentIconType, AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, + AgentWorkingResourceStatus, WorkflowAgentNodeBinding, ) from models.agent_config_entities import AgentSoulConfig @@ -588,7 +590,12 @@ def record_sql(_conn, _cursor, statement, _parameters, _context, _executemany) - class TestAppResponseViewAgentConfig: def test_masking_persistent_agent_config_does_not_change_database( - self, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session] + self, + sqlite_session: Session, + sqlite_session_factory: sessionmaker[Session], + *, + tool_providers, + workflow_queries, ): tenant_id = str(uuid4()) app = _persist_app(sqlite_session, tenant_id=tenant_id) @@ -609,7 +616,13 @@ def test_masking_persistent_agent_config_does_not_change_database( patch("services.app_service.mask_agent_tool_parameters", return_value=masked_agent_mode), patch.object(sqlite_session, "refresh", side_effect=AssertionError("App refresh is unnecessary")), ): - response_config = AppResponseView(app, session=sqlite_session, account=account).app_model_config + response_config = AppResponseView( + app, + session=sqlite_session, + account=account, + tool_providers=tool_providers, + workflow_queries=workflow_queries, + ).app_model_config assert response_config is not None assert response_config.agent_mode_dict == masked_agent_mode assert config.agent_mode_dict == original_agent_mode @@ -848,6 +861,10 @@ def test_update_agent_app_duplicate_name_rolls_back_and_raises_conflict(self, sq def test_delete_agent_app_archives_backing_agent(self, sqlite_session: Session): app, backing_agent = _persist_agent_app(sqlite_session) + home = AgentHomeSnapshot( + id="home-1", tenant_id=app.tenant_id, agent_id=backing_agent.id, snapshot_ref="snapshot-ref" + ) + sqlite_session.add(home) workflow_app = _persist_app(sqlite_session, tenant_id=app.tenant_id, name="Workflow") workflow_app.mode = AppMode.WORKFLOW referencing_workflow = Workflow.new( @@ -904,10 +921,6 @@ def test_delete_agent_app_archives_backing_agent(self, sqlite_session: Session): "services.app_service.remove_app_and_related_data_task.delay", side_effect=lambda **_kwargs: events.append("enqueue-app-cleanup"), ), - patch( - "services.app_service.AgentHomeSnapshotService.retire_all_for_agent", - return_value=["home-1"], - ) as mock_retire_homes, patch( "services.app_service.AgentWorkspaceService.retire_all_for_app", side_effect=lambda **_kwargs: events.append("retire-app-workspaces") or ["workspace-1"], @@ -950,11 +963,8 @@ def test_delete_agent_app_archives_backing_agent(self, sqlite_session: Session): tenant_id=app.tenant_id, app_id=app.id, ) - mock_retire_homes.assert_called_once_with( - session=sqlite_session, - tenant_id=app.tenant_id, - agent_id=backing_agent.id, - ) + assert home.status == AgentWorkingResourceStatus.RETIRED + assert home.retired_at is not None mock_enqueue_collection.assert_called_once_with( tenant_id=app.tenant_id, workspace_ids=["workspace-1"], diff --git a/api/tests/unit_tests/services/test_conversation_variable_updater.py b/api/tests/unit_tests/services/test_conversation_variable_updater.py index 6710da8b26687b..a9428cc8413aaa 100644 --- a/api/tests/unit_tests/services/test_conversation_variable_updater.py +++ b/api/tests/unit_tests/services/test_conversation_variable_updater.py @@ -3,7 +3,7 @@ from graphon.variables import StringVariable from models import ConversationVariable -from services.conversation_variable_updater import ConversationVariableUpdater +from repositories.workflow.conversation_variable_repository import WorkflowConversationVariableRepository @pytest.mark.parametrize("sqlite_session", [(ConversationVariable,)], indirect=True) @@ -24,7 +24,7 @@ def test_runtime_update_preserves_authored_id_and_other_conversations( ] sqlite_session.add_all(rows) sqlite_session.commit() - updater = ConversationVariableUpdater(sessionmaker(bind=sqlite_session.get_bind())) + updater = WorkflowConversationVariableRepository(sessionmaker(bind=sqlite_session.get_bind())) updater.update(conversation_id, variable.model_copy(update={"value": "updated"})) diff --git a/api/tests/unit_tests/services/test_dataset_service_dataset.py b/api/tests/unit_tests/services/test_dataset_service_dataset.py index 8ae38e09757f67..631900602dfae4 100644 --- a/api/tests/unit_tests/services/test_dataset_service_dataset.py +++ b/api/tests/unit_tests/services/test_dataset_service_dataset.py @@ -28,19 +28,19 @@ ExternalKnowledgeBindings, Pipeline, ) +from repositories.knowledge.collection_binding_repository import DatasetCollectionBindingRepository from services.entities.knowledge_entities.rag_pipeline_entities import ( IconInfo, RagPipelineDatasetCreateEntity, ) from services.errors.base import NoPermissionError from services.errors.dataset import DatasetNameDuplicateError -from services.knowledge.dataset_service import DatasetCollectionBindingService, DatasetPermissionService, DatasetService +from services.knowledge.dataset_service import DatasetPermissionService, DatasetService from tests.unit_tests.model_factories import make_account, make_dataset, make_tenant from .dataset_service_test_helpers import ( MagicMock, TenantAccountRole, - _make_knowledge_configuration, _make_retrieval_model, ) @@ -326,15 +326,17 @@ def test_get_datasets_by_ids_intersects_requested_and_accessible_ids( accessible = _dataset(dataset_id="accessible", name="Accessible", maintainer="other") owned = _dataset(dataset_id="owned", name="Owned", maintainer=user.id) hidden = _dataset(dataset_id="hidden", name="Hidden", maintainer="other") - sqlite_session.add_all([accessible, owned, hidden]) + foreign = _dataset(dataset_id="foreign", tenant_id="other", maintainer=user.id) + unrequested = _dataset(dataset_id="not-requested", maintainer=user.id) + sqlite_session.add_all([accessible, owned, hidden, foreign, unrequested]) sqlite_session.commit() config_overrides(RBAC_ENABLED=True) datasets, total = DatasetService.get_datasets_by_ids( - [accessible.id, owned.id, hidden.id], + [accessible.id, owned.id, hidden.id, foreign.id], "tenant-1", user=user, - accessible_dataset_ids=[accessible.id, "not-requested"], + accessible_dataset_ids=[accessible.id, unrequested.id, foreign.id], include_own_datasets=True, session=sqlite_session, ) @@ -691,8 +693,8 @@ def test_update_pipeline_node_data_rolls_back_real_session_on_failure(self, sqli lambda _session, _previous_transaction: transaction_events.append("rollback"), ) - with patch("services.knowledge.dataset_service.RagPipelineService") as service_cls: - service_cls.return_value.get_published_workflow.side_effect = RuntimeError("boom") + with patch("services.knowledge.dataset_service.WorkflowDefinitionStore") as service_cls: + service_cls.get_published_workflow.side_effect = RuntimeError("boom") with pytest.raises(RuntimeError, match="boom"): DatasetService._update_pipeline_knowledge_base_node_data(dataset, "user-1", sqlite_session) @@ -823,7 +825,7 @@ def test_apply_new_embedding_settings_uses_real_collection_binding(self, unbound patch("services.knowledge.dataset_service.current_user", account), patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls, patch.object( - DatasetCollectionBindingService, + DatasetCollectionBindingRepository, "get_dataset_collection_binding", return_value=collection_binding, ), @@ -886,149 +888,6 @@ def test_check_summary_index_setting_model_changed( assert DatasetService._check_summary_index_setting_model_changed(dataset, data) is expected -class TestDatasetServiceRagPipelineSettings: - def test_requires_current_tenant(self, unbound_session: Session) -> None: - account = _account() - account._current_tenant = None - - with ( - patch("services.knowledge.dataset_service.current_user", account), - pytest.raises(ValueError, match="Current user or current tenant not found"), - ): - DatasetService.update_rag_pipeline_dataset_settings( - _dataset(), - _make_knowledge_configuration(), - session=unbound_session, - ) - - def test_unpublished_high_quality_settings_use_real_merged_dataset(self, sqlite_session: Session) -> None: - dataset = _dataset(indexing_technique=IndexTechniqueType.ECONOMY) - sqlite_session.add(dataset) - sqlite_session.commit() - account = _account() - embedding_model = SimpleNamespace(provider="provider-2", model_name="embedding-2") - collection_binding = DatasetCollectionBinding( - provider_name="provider-2", - model_name="embedding-2", - type="dataset", - collection_name="collection", - ) - collection_binding.id = "collection-binding-2" - - with ( - patch("services.knowledge.dataset_service.current_user", account), - patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls, - patch.object(DatasetService, "check_is_multimodal_model", return_value=True), - patch.object( - DatasetCollectionBindingService, - "get_dataset_collection_binding", - return_value=collection_binding, - ), - ): - model_manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model - DatasetService.update_rag_pipeline_dataset_settings( - dataset, - _make_knowledge_configuration( - embedding_model_provider="provider-2", - embedding_model="embedding-2", - summary_index_setting={"enable": True}, - ), - session=sqlite_session, - ) - - persisted = sqlite_session.get(Dataset, dataset.id) - assert persisted is not None - assert persisted.indexing_technique == IndexTechniqueType.HIGH_QUALITY - assert persisted.embedding_model == "embedding-2" - assert persisted.collection_binding_id == collection_binding.id - assert persisted.is_multimodal is True - - def test_unpublished_economy_settings_update_keyword_number(self, sqlite_session: Session) -> None: - dataset = _dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY) - sqlite_session.add(dataset) - sqlite_session.commit() - - with patch("services.knowledge.dataset_service.current_user", _account()): - DatasetService.update_rag_pipeline_dataset_settings( - dataset, - _make_knowledge_configuration( - indexing_technique=IndexTechniqueType.ECONOMY, - keyword_number=17, - ), - session=sqlite_session, - ) - - persisted = sqlite_session.get(Dataset, dataset.id) - assert persisted is not None - assert persisted.indexing_technique == IndexTechniqueType.ECONOMY - assert persisted.keyword_number == 17 - - def test_published_economy_settings_commit_keyword_change_and_dispatch_no_task( - self, sqlite_session: Session - ) -> None: - dataset = _dataset(indexing_technique=IndexTechniqueType.ECONOMY, chunk_structure="paragraph") - dataset.keyword_number = 4 - sqlite_session.add(dataset) - sqlite_session.commit() - - with ( - patch("services.knowledge.dataset_service.current_user", _account()), - patch("services.knowledge.dataset_service.deal_dataset_index_update_task.delay") as update_task, - ): - DatasetService.update_rag_pipeline_dataset_settings( - dataset, - _make_knowledge_configuration( - chunk_structure="paragraph", - indexing_technique=IndexTechniqueType.ECONOMY, - keyword_number=12, - ), - has_published=True, - session=sqlite_session, - ) - - assert sqlite_session.get(Dataset, dataset.id).keyword_number == 12 - update_task.assert_not_called() - - @pytest.mark.parametrize( - ("dataset", "configuration", "message"), - [ - ( - _dataset(chunk_structure="paragraph"), - _make_knowledge_configuration(chunk_structure="sentence"), - "Chunk structure is not allowed", - ), - ( - _dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY, chunk_structure="paragraph"), - _make_knowledge_configuration( - chunk_structure="paragraph", - indexing_technique=IndexTechniqueType.ECONOMY, - ), - "not allowed to be updated to economy", - ), - ], - ) - def test_published_settings_reject_incompatible_changes( - self, - sqlite_session: Session, - dataset: Dataset, - configuration, - message: str, - ) -> None: - sqlite_session.add(dataset) - sqlite_session.commit() - - with ( - patch("services.knowledge.dataset_service.current_user", _account()), - pytest.raises(ValueError, match=message), - ): - DatasetService.update_rag_pipeline_dataset_settings( - dataset, - configuration, - has_published=True, - session=sqlite_session, - ) - - class TestDatasetPermissions: def test_check_dataset_permission_enforces_tenant_and_partial_members(self, sqlite_session: Session) -> None: dataset = _dataset(permission=DatasetPermissionEnum.PARTIAL_TEAM, maintainer="owner") diff --git a/api/tests/unit_tests/services/test_dataset_service_document.py b/api/tests/unit_tests/services/test_dataset_service_document.py index 702e7902065477..018572f6001f59 100644 --- a/api/tests/unit_tests/services/test_dataset_service_document.py +++ b/api/tests/unit_tests/services/test_dataset_service_document.py @@ -1,3 +1,5 @@ +from services.errors.document import DocumentNotFoundError + """Unit tests for DocumentService behaviors in dataset_service.""" from collections.abc import Callable @@ -31,7 +33,6 @@ IndexStructureType, InfoList, KnowledgeConfig, - NotFound, NotionIcon, NotionInfo, NotionPage, @@ -589,7 +590,7 @@ def test_save_document_without_dataset_id_creates_high_quality_dataset_with_defa with ( patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()), patch( - "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding", + "services.knowledge.dataset_service.DatasetCollectionBindingRepository.get_dataset_collection_binding", return_value=binding, ), patch.object( @@ -723,7 +724,7 @@ def test_update_document_with_dataset_id_raises_when_document_is_missing( ), ) with patch.object(DatasetService, "check_dataset_model_setting") as check_model_setting: - with pytest.raises(NotFound, match="Document not found"): + with pytest.raises(DocumentNotFoundError, match="Document not found"): DocumentService.update_document_with_dataset_id( dataset, document_data, @@ -1803,7 +1804,7 @@ def test_save_document_with_dataset_id_initializes_high_quality_dataset_from_def patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()), patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls, patch( - "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding", + "services.knowledge.dataset_service.DatasetCollectionBindingRepository.get_dataset_collection_binding", return_value=binding, ) as get_binding, patch.object(DocumentService, "update_prepared_document", return_value=updated_document), @@ -1870,7 +1871,7 @@ def test_save_document_with_dataset_id_uses_explicit_embedding_and_retrieval_mod patch("services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features()), patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls, patch( - "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding", + "services.knowledge.dataset_service.DatasetCollectionBindingRepository.get_dataset_collection_binding", return_value=binding, ) as get_binding, patch.object(DocumentService, "update_prepared_document", return_value=updated_document), diff --git a/api/tests/unit_tests/services/test_human_input_delivery_test_service.py b/api/tests/unit_tests/services/test_human_input_delivery_test_service.py index f523de5e017253..2fb2d7c4db13ce 100644 --- a/api/tests/unit_tests/services/test_human_input_delivery_test_service.py +++ b/api/tests/unit_tests/services/test_human_input_delivery_test_service.py @@ -23,14 +23,16 @@ ) from services import human_input_delivery_test_service as service_module from services.entities.feature_entities import FeatureModel -from services.human_input_delivery_test_service import ( +from services.human_input.contracts import ( DeliveryTestContext, DeliveryTestEmailRecipient, DeliveryTestError, - DeliveryTestRegistry, DeliveryTestResult, DeliveryTestStatus, DeliveryTestUnsupportedError, +) +from services.human_input_delivery_test_service import ( + DeliveryTestRegistry, EmailDeliveryTestHandler, HumanInputDeliveryTestService, _build_form_link, diff --git a/api/tests/unit_tests/services/test_installed_app_generation_adapters.py b/api/tests/unit_tests/services/test_installed_app_generation_adapters.py index d1e2b3e87ca0fd..ceb07732d32e16 100644 --- a/api/tests/unit_tests/services/test_installed_app_generation_adapters.py +++ b/api/tests/unit_tests/services/test_installed_app_generation_adapters.py @@ -12,7 +12,6 @@ from sqlalchemy.orm import Session, sessionmaker import services.app_generate_service as generation_module -from core.app.apps import message_based_app_generator from core.app.entities.app_invoke_entities import InvokeFrom from core.app.features.rate_limiting.rate_limit import RateLimit, RateLimitGenerator from enums import DeploymentEdition @@ -24,10 +23,14 @@ from services.account_errors import AccountNotFoundError from services.app_definition_query_service import AppDefinitionUnavailableError from services.errors.app import MoreLikeThisDisabledError +from services.errors.app_model_config import AppModelConfigBrokenError from services.errors.conversation import ConversationNotExistsError from services.errors.message import MessageNotExistsError from services.installed_app_generation_adapters import AppGenerateServiceRuntime from services.installed_app_generation_service import GenerationResponse +from services.workflow.execution.adapters import response_stream +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables _ARGS: dict[str, object] = {"inputs": {"count": 0}, "query": "hello", "auto_generate_name": False} _WORKFLOW_ARGS: dict[str, object] = {"inputs": {"count": 0}, "files": []} @@ -44,7 +47,12 @@ class _RuntimeHarness: @pytest.fixture -def harness(sqlite_session_factory: sessionmaker[Session]) -> _RuntimeHarness: +def harness( + sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, +) -> _RuntimeHarness: with sqlite_session_factory.begin() as session: app = App( tenant_id=str(uuid4()), name="Original app", mode=AppMode.COMPLETION, enable_site=True, enable_api=True @@ -73,7 +81,9 @@ def record_commit(session: Session) -> None: committed_sessions.append(session) return _RuntimeHarness( - AppGenerateServiceRuntime(session_factory=cast(sessionmaker[Session], factory)), + AppGenerateServiceRuntime( + session_factory=cast(sessionmaker[Session], factory), variables=workflow_variables, runtime=workflow_runtime + ), app.id, account.id, closed_sessions, @@ -111,8 +121,10 @@ def assert_context( assert streaming is expected_streaming return generate(session) - def legacy_generate( + def boundary_generate( *, + variables: WorkflowExecutionVariables, + runtime: WorkflowRuntime, session: Session, app_model: App, user: Account, @@ -120,13 +132,18 @@ def legacy_generate( invoke_from: InvokeFrom, streaming: bool, ) -> GenerationResponse | Generator[Mapping[str, object] | str, None, None]: + assert runtime is harness.runtime._runtime + assert variables is harness.runtime._variables assert args == _ARGS return assert_context( session=session, app_model=app_model, user=user, invoke_from=invoke_from, streaming=streaming ) - def legacy_generate_more_like_this( + def boundary_generate_more_like_this( *, + retrieval, + records, + annotations, session: Session, app_model: App, user: Account, @@ -134,6 +151,9 @@ def legacy_generate_more_like_this( invoke_from: InvokeFrom, streaming: bool, ) -> GenerationResponse | Generator[Mapping[str, object] | str, None, None]: + assert records is harness.runtime._runtime.chat_records + assert annotations is harness.runtime._runtime.annotation_replies + assert retrieval is harness.runtime._runtime.retrieval assert message_id == _MESSAGE_ID return assert_context( session=session, app_model=app_model, user=user, invoke_from=invoke_from, streaming=streaming @@ -142,10 +162,10 @@ def legacy_generate_more_like_this( expected_streaming = streaming if more_like_this: monkeypatch.setattr( - generation_module.AppGenerateService, "generate_more_like_this", legacy_generate_more_like_this + generation_module.AppGenerateService, "generate_more_like_this", boundary_generate_more_like_this ) else: - monkeypatch.setattr(generation_module.AppGenerateService, "generate", legacy_generate) + monkeypatch.setattr(generation_module.AppGenerateService, "generate", boundary_generate) def _invoke_generation( @@ -481,7 +501,7 @@ def test_more_like_this_requires_historical_config_even_when_current_config_is_e assert conversation is not None conversation.app_model_config_id = missing_config_id - with pytest.raises(ValueError, match="Message app_model_config is None"): + with pytest.raises(AppModelConfigBrokenError): harness.runtime.generate_more_like_this( app_id=harness.app_id, account_id=harness.account_id, message_id=message_id, streaming=True ) @@ -746,19 +766,19 @@ def test_workflow_dispatch_starts_task_after_subscription_with_runtime_session_c channel.topic.return_value = topic def activate_subscription() -> Subscription: - assert len(harness.closed_sessions) == 2 + assert len(set(harness.closed_sessions)) == 2 assert all(not session.in_transaction() for session in harness.closed_sessions) transport_events.append("subscribe") return subscription subscription.__enter__.side_effect = activate_subscription subscription.receive.return_value = b'{"event":"workflow_finished"}' - monkeypatch.setattr(message_based_app_generator, "get_pubsub_broadcast_channel", lambda: channel) + monkeypatch.setattr(response_stream, "get_pubsub_broadcast_channel", lambda: channel) submitted: list[generation_module.AppExecutionParams] = [] def enqueue(payload_json: str) -> None: assert transport_events == ["subscribe"] - assert len(harness.closed_sessions) == 2 + assert len(set(harness.closed_sessions)) == 2 assert all(not session.in_transaction() for session in harness.closed_sessions) submitted.append(generation_module.AppExecutionParams.model_validate_json(payload_json)) transport_events.append("enqueue") @@ -768,7 +788,7 @@ def enqueue(payload_json: str) -> None: result = _generate_workflow(harness, sqlite_session_factory, args) assert isinstance(result, RateLimitGenerator) - assert len(harness.closed_sessions) == 2 + assert len(set(harness.closed_sessions)) == 2 assert harness.committed_sessions == [harness.closed_sessions[1]] assert submitted == [] subscriber.prepare_subscription.assert_called_once_with() @@ -819,7 +839,7 @@ def test_unpublished_workflow_raises_before_subscription_or_task_creation( ) channel = MagicMock(spec=BroadcastChannel) - monkeypatch.setattr(message_based_app_generator, "get_pubsub_broadcast_channel", lambda: channel) + monkeypatch.setattr(response_stream, "get_pubsub_broadcast_channel", lambda: channel) enqueue = MagicMock() monkeypatch.setattr(generation_module.workflow_based_app_execution_task, "delay", enqueue) @@ -827,7 +847,7 @@ def test_unpublished_workflow_raises_before_subscription_or_task_creation( _generate_workflow(harness, sqlite_session_factory, _WORKFLOW_ARGS) assert type(raised.value) is ValueError - assert len(harness.closed_sessions) == 2 + assert len(set(harness.closed_sessions)) == 2 assert all(not session.in_transaction() for session in harness.closed_sessions) assert harness.committed_sessions == [] channel.topic.assert_not_called() diff --git a/api/tests/unit_tests/services/test_package_icons.py b/api/tests/unit_tests/services/test_package_icons.py index b31bdf0901ef16..189d05f332bf1e 100644 --- a/api/tests/unit_tests/services/test_package_icons.py +++ b/api/tests/unit_tests/services/test_package_icons.py @@ -167,9 +167,9 @@ def test_materialized_icon_is_owned_by_destination(monkeypatch: pytest.MonkeyPat def test_roster_export_embeds_agent_icon(monkeypatch: pytest.MonkeyPatch) -> None: - from models.agent_config_entities import AgentSoulConfig + from models.agent_config_entities import AgentPackageMetadata, AgentSoulConfig from models.model import App - from services.agent.dsl_entities import AgentPackage, AgentPackageMetadata, make_agent_app_dsl + from services.agent.dsl_entities import AgentPackage, make_agent_app_dsl from services.agent.roster_package_exporter import RosterAgentPackageExporter from services.agent.roster_package_reader import RosterAgentPackageReader from services.entities.site_dsl import SiteDsl diff --git a/api/tests/unit_tests/services/test_schedule_service.py b/api/tests/unit_tests/services/test_schedule_service.py index 40f1801c561912..b70df951b27823 100644 --- a/api/tests/unit_tests/services/test_schedule_service.py +++ b/api/tests/unit_tests/services/test_schedule_service.py @@ -2,7 +2,6 @@ import unittest from datetime import UTC, datetime from typing import Any -from unittest.mock import MagicMock import pytest @@ -11,7 +10,9 @@ from core.workflow.nodes.trigger_schedule.exc import ScheduleConfigError from libs.schedule_utils import calculate_next_run_at, convert_12h_to_24h from models.workflow import Workflow +from services.trigger.schedule_policy import visual_to_cron from services.trigger.schedule_service import ScheduleService +from services.trigger.workflow_policy import schedule_config from tests.unit_tests.model_factories import make_workflow @@ -83,13 +84,13 @@ class TestVisualToCron(unittest.TestCase): def test_visual_to_cron_hourly(self): """Test converting hourly visual config to cron.""" visual_config = VisualConfig(on_minute=15) - result = ScheduleService.visual_to_cron("hourly", visual_config) + result = visual_to_cron("hourly", visual_config) assert result == "15 * * * *" def test_visual_to_cron_daily(self): """Test converting daily visual config to cron.""" visual_config = VisualConfig(time="2:30 PM") - result = ScheduleService.visual_to_cron("daily", visual_config) + result = visual_to_cron("daily", visual_config) assert result == "30 14 * * *" def test_visual_to_cron_weekly(self): @@ -98,7 +99,7 @@ def test_visual_to_cron_weekly(self): time="10:00 AM", weekdays=["mon", "wed", "fri"], ) - result = ScheduleService.visual_to_cron("weekly", visual_config) + result = visual_to_cron("weekly", visual_config) assert result == "0 10 * * 1,3,5" def test_visual_to_cron_monthly_with_specific_days(self): @@ -107,7 +108,7 @@ def test_visual_to_cron_monthly_with_specific_days(self): time="11:30 AM", monthly_days=[1, 15], ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) assert result == "30 11 1,15 * *" def test_visual_to_cron_monthly_with_last_day(self): @@ -116,7 +117,7 @@ def test_visual_to_cron_monthly_with_last_day(self): time="11:30 AM", monthly_days=[1, "last"], ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) assert result == "30 11 1,L * *" def test_visual_to_cron_monthly_only_last_day(self): @@ -125,7 +126,7 @@ def test_visual_to_cron_monthly_only_last_day(self): time="9:00 PM", monthly_days=["last"], ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) assert result == "0 21 L * *" def test_visual_to_cron_monthly_with_end_days_and_last(self): @@ -134,46 +135,46 @@ def test_visual_to_cron_monthly_with_end_days_and_last(self): time="3:45 PM", monthly_days=[29, 30, 31, "last"], ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) # Should have 29,30,31,L - the L handles all possible last days assert result == "45 15 29,30,31,L * *" def test_visual_to_cron_invalid_frequency(self): """Test converting with invalid frequency.""" with pytest.raises(ScheduleConfigError, match="Unsupported frequency: invalid"): - ScheduleService.visual_to_cron("invalid", VisualConfig()) + visual_to_cron("invalid", VisualConfig()) def test_visual_to_cron_weekly_no_weekdays(self): """Test converting weekly with no weekdays specified.""" visual_config = VisualConfig(time="10:00 AM") with pytest.raises(ScheduleConfigError, match="Weekdays are required for weekly schedules"): - ScheduleService.visual_to_cron("weekly", visual_config) + visual_to_cron("weekly", visual_config) def test_visual_to_cron_hourly_no_minute(self): """Test converting hourly with no on_minute specified.""" visual_config = VisualConfig() # on_minute defaults to 0 - result = ScheduleService.visual_to_cron("hourly", visual_config) + result = visual_to_cron("hourly", visual_config) assert result == "0 * * * *" # Should use default value 0 def test_visual_to_cron_daily_no_time(self): """Test converting daily with no time specified.""" visual_config = VisualConfig(time=None) with pytest.raises(ScheduleConfigError, match="time is required for daily schedules"): - ScheduleService.visual_to_cron("daily", visual_config) + visual_to_cron("daily", visual_config) def test_visual_to_cron_weekly_no_time(self): """Test converting weekly with no time specified.""" visual_config = VisualConfig(weekdays=["mon"]) visual_config.time = None # Override default with pytest.raises(ScheduleConfigError, match="time is required for weekly schedules"): - ScheduleService.visual_to_cron("weekly", visual_config) + visual_to_cron("weekly", visual_config) def test_visual_to_cron_monthly_no_time(self): """Test converting monthly with no time specified.""" visual_config = VisualConfig(monthly_days=[1]) visual_config.time = None # Override default with pytest.raises(ScheduleConfigError, match="time is required for monthly schedules"): - ScheduleService.visual_to_cron("monthly", visual_config) + visual_to_cron("monthly", visual_config) def test_visual_to_cron_monthly_duplicate_days(self): """Test monthly with duplicate days should be deduplicated.""" @@ -181,7 +182,7 @@ def test_visual_to_cron_monthly_duplicate_days(self): time="10:00 AM", monthly_days=[1, 15, 1, 15, 31], # Duplicates ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) assert result == "0 10 1,15,31 * *" # Should be deduplicated def test_visual_to_cron_monthly_unsorted_days(self): @@ -190,7 +191,7 @@ def test_visual_to_cron_monthly_unsorted_days(self): time="2:30 PM", monthly_days=[20, 5, 15, 1, 10], # Unsorted ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) assert result == "30 14 1,5,10,15,20 * *" # Should be sorted def test_visual_to_cron_weekly_all_weekdays(self): @@ -199,31 +200,31 @@ def test_visual_to_cron_weekly_all_weekdays(self): time="8:00 AM", weekdays=["sun", "mon", "tue", "wed", "thu", "fri", "sat"], ) - result = ScheduleService.visual_to_cron("weekly", visual_config) + result = visual_to_cron("weekly", visual_config) assert result == "0 8 * * 0,1,2,3,4,5,6" def test_visual_to_cron_hourly_boundary_values(self): """Test hourly with boundary minute values.""" # Minimum value visual_config = VisualConfig(on_minute=0) - result = ScheduleService.visual_to_cron("hourly", visual_config) + result = visual_to_cron("hourly", visual_config) assert result == "0 * * * *" # Maximum value visual_config = VisualConfig(on_minute=59) - result = ScheduleService.visual_to_cron("hourly", visual_config) + result = visual_to_cron("hourly", visual_config) assert result == "59 * * * *" def test_visual_to_cron_daily_midnight_noon(self): """Test daily at special times (midnight and noon).""" # Midnight visual_config = VisualConfig(time="12:00 AM") - result = ScheduleService.visual_to_cron("daily", visual_config) + result = visual_to_cron("daily", visual_config) assert result == "0 0 * * *" # Noon visual_config = VisualConfig(time="12:00 PM") - result = ScheduleService.visual_to_cron("daily", visual_config) + result = visual_to_cron("daily", visual_config) assert result == "0 12 * * *" def test_visual_to_cron_monthly_mixed_with_last_and_duplicates(self): @@ -232,7 +233,7 @@ def test_visual_to_cron_monthly_mixed_with_last_and_duplicates(self): time="11:45 PM", monthly_days=[15, 1, "last", 15, 30, 1, "last"], # Mixed with duplicates ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) assert result == "45 23 1,15,30,L * *" # Deduplicated and sorted with L at end def test_visual_to_cron_weekly_single_day(self): @@ -241,7 +242,7 @@ def test_visual_to_cron_weekly_single_day(self): time="6:30 PM", weekdays=["sun"], ) - result = ScheduleService.visual_to_cron("weekly", visual_config) + result = visual_to_cron("weekly", visual_config) assert result == "30 18 * * 0" def test_visual_to_cron_monthly_all_possible_days(self): @@ -251,7 +252,7 @@ def test_visual_to_cron_monthly_all_possible_days(self): time="12:01 AM", monthly_days=all_days, ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) expected_days = ",".join([str(i) for i in range(1, 32)]) + ",L" assert result == f"1 0 {expected_days} * *" @@ -259,13 +260,13 @@ def test_visual_to_cron_monthly_no_days(self): """Test monthly without any days specified should raise error.""" visual_config = VisualConfig(time="10:00 AM", monthly_days=[]) with pytest.raises(ScheduleConfigError, match="Monthly days are required for monthly schedules"): - ScheduleService.visual_to_cron("monthly", visual_config) + visual_to_cron("monthly", visual_config) def test_visual_to_cron_weekly_empty_weekdays_list(self): """Test weekly with empty weekdays list should raise error.""" visual_config = VisualConfig(time="10:00 AM", weekdays=[]) with pytest.raises(ScheduleConfigError, match="Weekdays are required for weekly schedules"): - ScheduleService.visual_to_cron("weekly", visual_config) + visual_to_cron("weekly", visual_config) class TestParseTime(unittest.TestCase): @@ -344,7 +345,7 @@ def test_extract_schedule_config_with_cron_mode(self): ), ) - config = ScheduleService.extract_schedule_config(workflow) + config = schedule_config(workflow.graph_dict) assert config is not None assert config.node_id == "schedule-node" @@ -372,7 +373,7 @@ def test_extract_schedule_config_with_visual_mode(self): ), ) - config = ScheduleService.extract_schedule_config(workflow) + config = schedule_config(workflow.graph_dict) assert config is not None assert config.node_id == "schedule-node" @@ -394,7 +395,7 @@ def test_extract_schedule_config_no_schedule_node(self): ), ) - config = ScheduleService.extract_schedule_config(workflow) + config = schedule_config(workflow.graph_dict) assert config is None def test_extract_schedule_config_invalid_graph(self): @@ -404,7 +405,7 @@ def test_extract_schedule_config_invalid_graph(self): ) with pytest.raises(ScheduleConfigError, match="Workflow graph is empty"): - ScheduleService.extract_schedule_config(workflow) + schedule_config(workflow.graph_dict) class TestScheduleWithTimezone(unittest.TestCase): @@ -423,7 +424,7 @@ def test_visual_schedule_with_timezone_integration(self): ) # Convert to cron expression - cron_expr = ScheduleService.visual_to_cron("monthly", visual_config) + cron_expr = visual_to_cron("monthly", visual_config) assert cron_expr is not None assert cron_expr == "30 10 1 * *" # Direct conversion @@ -456,7 +457,7 @@ def test_visual_schedule_different_timezones_same_local_time(self): weekdays=["mon"], ) - cron_expr = ScheduleService.visual_to_cron("weekly", visual_config) + cron_expr = visual_to_cron("weekly", visual_config) assert cron_expr is not None assert cron_expr == "0 9 * * 1" @@ -487,7 +488,7 @@ def test_visual_schedule_daily_across_dst_change(self): time="10:00 AM", ) - cron_expr = ScheduleService.visual_to_cron("daily", visual_config) + cron_expr = visual_to_cron("daily", visual_config) assert cron_expr is not None assert cron_expr == "0 10 * * *" @@ -540,7 +541,7 @@ def test_to_schedule_config_should_raise_for_cron_mode_without_expression() -> N ScheduleService.to_schedule_config(node_config=node_config) -def test_to_schedule_config_should_build_from_visual_mode(monkeypatch: pytest.MonkeyPatch) -> None: +def test_to_schedule_config_should_build_from_visual_mode() -> None: # Arrange node_config = { "id": "node-1", @@ -551,7 +552,6 @@ def test_to_schedule_config_should_build_from_visual_mode(monkeypatch: pytest.Mo "timezone": "UTC", }, } - monkeypatch.setattr(ScheduleService, "visual_to_cron", MagicMock(return_value="30 9 * * *")) # Act result = ScheduleService.to_schedule_config(node_config=node_config) @@ -575,7 +575,7 @@ def test_extract_schedule_config_should_raise_when_graph_is_empty() -> None: # Act / Assert with pytest.raises(ScheduleConfigError, match="Workflow graph is empty"): - ScheduleService.extract_schedule_config(workflow=workflow) + schedule_config(workflow.graph_dict) def test_extract_schedule_config_should_raise_when_mode_invalid() -> None: @@ -596,7 +596,7 @@ def test_extract_schedule_config_should_raise_when_mode_invalid() -> None: # Act / Assert with pytest.raises(ScheduleConfigError, match="Invalid schedule mode: invalid"): - ScheduleService.extract_schedule_config(workflow=workflow) + schedule_config(workflow.graph_dict) if __name__ == "__main__": diff --git a/api/tests/unit_tests/services/test_skill_management_service.py b/api/tests/unit_tests/services/test_skill_management_service.py index adfd2fa9b54230..0e8e28ec2f663b 100644 --- a/api/tests/unit_tests/services/test_skill_management_service.py +++ b/api/tests/unit_tests/services/test_skill_management_service.py @@ -18,6 +18,7 @@ from core.credit_usage import CreditUsageCreatedBy from core.db.session_factory import session_factory from core.tools.tool_file_manager import ToolFileManager +from enums.agent import WorkflowAgentBindingType from models.account import Account from models.agent import ( Agent, @@ -29,7 +30,6 @@ AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import ( diff --git a/api/tests/unit_tests/services/test_snippet_dsl_service.py b/api/tests/unit_tests/services/test_snippet_dsl_service.py index a6e56897b84679..f15a89ddd40a13 100644 --- a/api/tests/unit_tests/services/test_snippet_dsl_service.py +++ b/api/tests/unit_tests/services/test_snippet_dsl_service.py @@ -4,9 +4,10 @@ import pytest import yaml -from sqlalchemy import event +from sqlalchemy import event, select from sqlalchemy.orm import Session +from extensions.application_services.snippets import build_snippet_dsl_service from graphon.nodes import BuiltinNodeTypes from models import Account from models.snippet import CustomizedSnippet, SnippetType @@ -18,6 +19,7 @@ SnippetPendingData, _check_version_compatibility, ) +from services.snippet_service import SnippetService from tests.unit_tests.model_factories import make_account, make_tenant, make_workflow SQLITE_MODELS = (CustomizedSnippet,) @@ -30,7 +32,7 @@ @pytest.fixture def service(sqlite_session: Session) -> SnippetDslService: """Create the service with a real caller-owned SQLite session.""" - return SnippetDslService(session=sqlite_session) + return build_snippet_dsl_service(session=sqlite_session) def _account(*, account_id: str = "account-1", tenant_id: str = "tenant-1") -> Account: @@ -547,8 +549,9 @@ def test_check_dependencies_returns_empty_without_draft_workflow( service: SnippetDslService, monkeypatch: pytest.MonkeyPatch ): monkeypatch.setattr( - "services.snippet_dsl_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_draft_workflow=Mock(return_value=None)), + service, + "_snippets", + Mock(spec=SnippetService, get_draft_workflow=Mock(return_value=None)), ) result = service.check_dependencies(_snippet()) @@ -565,8 +568,9 @@ def test_check_dependencies_returns_generated_dependencies(service: SnippetDslSe } ] monkeypatch.setattr( - "services.snippet_dsl_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_draft_workflow=Mock(return_value=workflow)), + service, + "_snippets", + Mock(spec=SnippetService, get_draft_workflow=Mock(return_value=workflow)), ) monkeypatch.setattr(service, "_extract_dependencies_from_workflow", Mock(return_value=["langgenius/openai"])) monkeypatch.setattr( @@ -589,23 +593,17 @@ def test_create_or_update_snippet_updates_existing_snippet_and_syncs_workflow( ) sqlite_session.add(snippet) sqlite_session.commit() - draft_workflow = _workflow() - snippet_service = SimpleNamespace( - get_draft_workflow=Mock(return_value=draft_workflow), - sync_draft_workflow=Mock(return_value=draft_workflow), - ) - monkeypatch.setattr("services.snippet_dsl_service.SnippetService", lambda *_args, **_kwargs: snippet_service) monkeypatch.setattr( - "services.snippet_dsl_service.WorkflowAgentPublishService.sync_agent_bindings_for_draft", + "services.agent.workflow_publish_service.WorkflowAgentPublishService._synchronize_bindings", Mock(return_value={"retired-agent"}), ) monkeypatch.setattr( - "services.snippet_dsl_service.WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync", + "services.agent.workflow_publish_service.WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync", Mock(), ) retire_unowned = Mock() monkeypatch.setattr( - "services.snippet_dsl_service.WorkflowAgentRetirementService.retire_unowned", + "services.agent.retirement_service.WorkflowAgentRetirementService.retire_unowned", retire_unowned, ) @@ -628,14 +626,17 @@ def test_create_or_update_snippet_updates_existing_snippet_and_syncs_workflow( assert snippet.name == "New" assert snippet.type == "node" assert snippet.icon_info == {"icon": "x"} - snippet_service.sync_draft_workflow.assert_called_once() assert not sqlite_session.in_transaction() + persisted_draft = sqlite_session.scalar(select(Workflow).where(Workflow.app_id == result.id)) + assert persisted_draft is not None + assert persisted_draft.kind == "snippet" + assert persisted_draft.graph_dict == {"nodes": [], "edges": []} persisted = sqlite_session.get(CustomizedSnippet, snippet.id) assert persisted is not None assert persisted.name == "New" retire_unowned.assert_called_once_with( tenant_id="tenant-1", - agent_ids={"retired-agent"}, + agent_ids=["retired-agent"], account_id="account-1", ) @@ -643,17 +644,12 @@ def test_create_or_update_snippet_updates_existing_snippet_and_syncs_workflow( def test_create_or_update_snippet_creates_new_snippet_and_flushes( service: SnippetDslService, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session ): - snippet_service = SimpleNamespace( - get_draft_workflow=Mock(return_value=None), - sync_draft_workflow=Mock(return_value=_workflow()), - ) - monkeypatch.setattr("services.snippet_dsl_service.SnippetService", lambda *_args, **_kwargs: snippet_service) monkeypatch.setattr( - "services.snippet_dsl_service.WorkflowAgentPublishService.sync_agent_bindings_for_draft", + "services.agent.workflow_publish_service.WorkflowAgentPublishService._synchronize_bindings", Mock(return_value=set()), ) monkeypatch.setattr( - "services.snippet_dsl_service.WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync", + "services.agent.workflow_publish_service.WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync", Mock(), ) @@ -674,14 +670,18 @@ def test_create_or_update_snippet_creates_new_snippet_and_flushes( assert result.name == "New Snippet" assert result.type == "group" assert sqlite_session.get(CustomizedSnippet, result.id) is result - snippet_service.sync_draft_workflow.assert_called_once() assert not sqlite_session.in_transaction() + persisted_draft = sqlite_session.scalar(select(Workflow).where(Workflow.app_id == result.id)) + assert persisted_draft is not None + assert persisted_draft.kind == "snippet" + assert persisted_draft.graph_dict == {"nodes": [], "edges": []} def test_export_snippet_dsl_raises_without_draft_workflow(service: SnippetDslService, monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr( - "services.snippet_dsl_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_draft_workflow=Mock(return_value=None)), + service, + "_snippets", + Mock(spec=SnippetService, get_draft_workflow=Mock(return_value=None)), ) with pytest.raises(ValueError, match="Missing draft workflow"): @@ -697,8 +697,9 @@ def test_export_snippet_dsl_returns_yaml(service: SnippetDslService, monkeypatch input_fields=[{"variable": "query"}], ) monkeypatch.setattr( - "services.snippet_dsl_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_draft_workflow=Mock(return_value=workflow)), + service, + "_snippets", + Mock(spec=SnippetService, get_draft_workflow=Mock(return_value=workflow)), ) monkeypatch.setattr( "services.snippet_dsl_service.DependenciesAnalysisService.generate_dependencies", @@ -720,8 +721,10 @@ def test_export_snippet_dsl_uses_requested_published_workflow( get_published_workflow_by_id = Mock(return_value=workflow) get_draft_workflow = Mock() monkeypatch.setattr( - "services.snippet_dsl_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace( + service, + "_snippets", + Mock( + spec=SnippetService, get_draft_workflow=get_draft_workflow, get_published_workflow_by_id=get_published_workflow_by_id, ), diff --git a/api/tests/unit_tests/services/test_snippet_generate_service.py b/api/tests/unit_tests/services/test_snippet_generate_service.py index cc2a81ecdc4685..0598f7be0edbc1 100644 --- a/api/tests/unit_tests/services/test_snippet_generate_service.py +++ b/api/tests/unit_tests/services/test_snippet_generate_service.py @@ -10,9 +10,41 @@ from models.snippet import CustomizedSnippet, SnippetType from models.workflow import Workflow, WorkflowKind, WorkflowNodeExecutionModel from services.snippet_generate_service import SnippetGenerateService +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables +from services.workflow_service import WorkflowService from tests.unit_tests.model_factories import make_account, make_workflow +class SnippetReader: + workflow: Workflow | None = None + + def get_draft_workflow(self, snippet: CustomizedSnippet) -> Workflow | None: + assert snippet.id == "snippet-1" + return self.workflow + + +@pytest.fixture +def snippet_reader() -> SnippetReader: + return SnippetReader() + + +@pytest.fixture +def generation( + snippet_reader: SnippetReader, + workflow_variables: WorkflowExecutionVariables, + sqlite_session_factory: sessionmaker[Session], + *, + workflow_runtime: WorkflowRuntime, +) -> SnippetGenerateService: + return SnippetGenerateService( + snippets=snippet_reader, + variables=workflow_variables, + workflows=WorkflowService(sqlite_session_factory), + runtime=workflow_runtime, + ) + + def _workflow(graph: dict) -> Workflow: return make_workflow(workflow_id="workflow-1", app_id="snippet-1", kind=WorkflowKind.SNIPPET, graph=graph) @@ -33,6 +65,47 @@ def _account(account_id: str = "user-1") -> Account: return make_account(account_id=account_id, name="Test User", email=f"{account_id}@example.com") +@pytest.mark.parametrize("entry", ["generate", "iteration", "loop"]) +def test_debug_generators_use_injected_variable_dependencies( + generation: SnippetGenerateService, + snippet_reader: SnippetReader, + workflow_variables: WorkflowExecutionVariables, + monkeypatch: pytest.MonkeyPatch, + entry: str, +) -> None: + from core.app.entities.app_invoke_entities import InvokeFrom + from graphon.enums import BuiltinNodeTypes + from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator + + workflow = _workflow({"nodes": [{"id": "start", "data": {"type": "start"}}], "edges": []}) + snippet_reader.workflow = workflow + user = _account() + calls: list[str] = [] + + def execute(generator: WorkflowAppGenerator, **_kwargs: object) -> dict[str, str]: + assert generator._draft_variable_loader == workflow_variables.workflow_loader + assert generator._draft_variable_saver == workflow_variables.saver_factory + saver = generator._get_draft_var_saver_factory(InvokeFrom.DEBUGGER, user, tenant_id=workflow.tenant_id) + saver(workflow.app_id, "node", BuiltinNodeTypes.CODE, "execution").save( + process_data=None, outputs={"text": "injected"} + ) + loader = workflow_variables.workflow_loader(workflow, user.id) + assert loader.load_variables([["node", "text"]])[0].value == "injected" + calls.append(entry) + return {"result": "ready"} + + method = {"generate": "generate", "iteration": "single_iteration_generate", "loop": "single_loop_generate"}[entry] + monkeypatch.setattr(WorkflowAppGenerator, method, execute) + if entry == "generate": + result = generation.generate(_snippet(), user, {}, InvokeFrom.DEBUGGER, streaming=False) + elif entry == "iteration": + result = generation.generate_single_iteration(_snippet(), user, "node", {}, streaming=False) + else: + result = generation.generate_single_loop(_snippet(), user, "node", {}, streaming=False) + assert result == {"result": "ready"} + assert calls == [entry] + + def test_filter_virtual_start_events_keeps_blocking_response_unchanged(): response = {"data": {"outputs": {"text": "ok"}}} @@ -132,14 +205,14 @@ def test_parse_files_delegates_to_file_factory(monkeypatch: pytest.MonkeyPatch): build_from_mappings.assert_called_once() -def test_generate_raises_when_draft_workflow_missing(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr( - "services.snippet_generate_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_draft_workflow=Mock(return_value=None)), - ) +def test_generate_raises_when_draft_workflow_missing( + generation: SnippetGenerateService, + snippet_reader: SnippetReader, +): + snippet_reader.workflow = None with pytest.raises(ValueError, match="Workflow not initialized"): - SnippetGenerateService.generate( + generation.generate( snippet=_snippet(), user=_account(), args={"inputs": {}}, @@ -147,7 +220,11 @@ def test_generate_raises_when_draft_workflow_missing(monkeypatch: pytest.MonkeyP ) -def test_generate_delegates_to_workflow_generator_and_filters_stream(monkeypatch: pytest.MonkeyPatch): +def test_generate_delegates_to_workflow_generator_and_filters_stream( + monkeypatch: pytest.MonkeyPatch, + generation: SnippetGenerateService, + snippet_reader: SnippetReader, +): workflow = _workflow({"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []}) snippet = _snippet() user = _account() @@ -161,22 +238,21 @@ def test_generate_delegates_to_workflow_generator_and_filters_stream(monkeypatch workflow_generator_class = Mock(return_value=generator) workflow_generator_class.convert_to_event_stream = Mock(side_effect=lambda response: response) - monkeypatch.setattr( - "services.snippet_generate_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_draft_workflow=Mock(return_value=workflow)), - ) + snippet_reader.workflow = workflow ensure_start_node = Mock(return_value=workflow) monkeypatch.setattr(SnippetGenerateService, "_ensure_start_node", ensure_start_node) monkeypatch.setattr("services.snippet_generate_service.WorkflowAppGenerator", workflow_generator_class) - result = SnippetGenerateService.generate( + result = generation.generate( snippet=snippet, user=user, args={"inputs": {"query": "hello"}}, invoke_from="debugger", ) - assert list(result) == [{"event": "node_finished", "data": {"node_id": "llm-1"}}] + assert [json.loads(event.removeprefix("data: ")) for event in result] == [ + {"event": "node_finished", "data": {"node_id": "llm-1"}} + ] ensure_start_node.assert_called_once_with(workflow, snippet) generator.generate.assert_called_once() kwargs = generator.generate.call_args.kwargs @@ -185,65 +261,25 @@ def test_generate_delegates_to_workflow_generator_and_filters_stream(monkeypatch assert kwargs["user"] is user assert kwargs["streaming"] is True assert kwargs["call_depth"] == 0 - workflow_generator_class.convert_to_event_stream.assert_called_once() - - -def test_run_published_delegates_to_workflow_generator_non_streaming(monkeypatch: pytest.MonkeyPatch): - workflow = _workflow({"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []}) - snippet = _snippet() - user = _account() - generator = SimpleNamespace(generate=Mock(return_value={"data": {"outputs": {"answer": "ok"}}})) - - monkeypatch.setattr( - "services.snippet_generate_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_published_workflow=Mock(return_value=workflow)), - ) - ensure_start_node = Mock(return_value=workflow) - monkeypatch.setattr(SnippetGenerateService, "_ensure_start_node", ensure_start_node) - monkeypatch.setattr("services.snippet_generate_service.WorkflowAppGenerator", Mock(return_value=generator)) - - result = SnippetGenerateService.run_published( - snippet=snippet, - user=user, - args={"inputs": {"query": "hello"}}, - invoke_from="service-api", - ) - - assert result == {"data": {"outputs": {"answer": "ok"}}} - ensure_start_node.assert_called_once_with(workflow, snippet) - generator.generate.assert_called_once() - kwargs = generator.generate.call_args.kwargs - assert kwargs["app_model"].id == "snippet-1" - assert kwargs["streaming"] is False - assert kwargs["call_depth"] == 0 - - -def test_ensure_start_node_for_worker_delegates(monkeypatch: pytest.MonkeyPatch): - workflow = _workflow({"nodes": [], "edges": []}) - snippet = _snippet() - ensure_start_node = Mock(return_value=workflow) - monkeypatch.setattr(SnippetGenerateService, "_ensure_start_node", ensure_start_node) - - result = SnippetGenerateService.ensure_start_node_for_worker(workflow, snippet) - - assert result is workflow - ensure_start_node.assert_called_once_with(workflow, snippet) -def test_run_draft_node_delegates_to_workflow_service(monkeypatch: pytest.MonkeyPatch): +def test_run_draft_node_delegates_to_workflow_service( + *, + workflow_variables: WorkflowExecutionVariables, + generation: SnippetGenerateService, + snippet_reader: SnippetReader, +): workflow = _workflow({"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []}) snippet = _snippet() account = _account("account-1") execution = WorkflowNodeExecutionModel(id="execution-1") - workflow_service = SimpleNamespace(run_draft_workflow_node=Mock(return_value=execution)) + workflow_service = Mock(spec=WorkflowService) + workflow_service.run_draft_workflow_node.return_value = execution - monkeypatch.setattr( - "services.snippet_generate_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_draft_workflow=Mock(return_value=workflow)), - ) - monkeypatch.setattr("services.snippet_generate_service.WorkflowService", Mock(return_value=workflow_service)) + snippet_reader.workflow = workflow + generation._workflows = workflow_service - result = SnippetGenerateService.run_draft_node( + result = generation.run_draft_node( snippet=snippet, node_id="llm-1", user_inputs={"query": "hello"}, @@ -262,16 +298,18 @@ def test_run_draft_node_delegates_to_workflow_service(monkeypatch: pytest.Monkey assert kwargs["account"] is account assert kwargs["query"] == "question" assert kwargs["files"] == [] + assert kwargs["variables"] is workflow_variables -def test_run_draft_node_raises_when_draft_workflow_missing(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr( - "services.snippet_generate_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_draft_workflow=Mock(return_value=None)), - ) +def test_run_draft_node_raises_when_draft_workflow_missing( + *, + generation: SnippetGenerateService, + snippet_reader: SnippetReader, +): + snippet_reader.workflow = None with pytest.raises(ValueError, match="Workflow not initialized"): - SnippetGenerateService.run_draft_node( + generation.run_draft_node( snippet=_snippet(), node_id="llm-1", user_inputs={}, @@ -281,7 +319,8 @@ def test_run_draft_node_raises_when_draft_workflow_missing(monkeypatch: pytest.M def test_generate_single_iteration_delegates_to_workflow_generator( monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], + generation: SnippetGenerateService, + snippet_reader: SnippetReader, ) -> None: workflow = _workflow({"nodes": [{"id": "iteration-1", "data": {"type": "iteration"}}], "edges": []}) snippet = _snippet() @@ -291,21 +330,17 @@ def test_generate_single_iteration_delegates_to_workflow_generator( workflow_generator_class = Mock(return_value=generator) workflow_generator_class.convert_to_event_stream = Mock(side_effect=lambda item: item) - monkeypatch.setattr( - "services.snippet_generate_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_draft_workflow=Mock(return_value=workflow)), - ) + snippet_reader.workflow = workflow monkeypatch.setattr("services.snippet_generate_service.WorkflowAppGenerator", workflow_generator_class) - result = SnippetGenerateService.generate_single_iteration( + result = generation.generate_single_iteration( snippet=snippet, user=user, node_id="iteration-1", args={"inputs": {"items": [1]}}, - session_maker=sqlite_session_factory, ) - assert list(result) == ["event"] + assert list(result) == ["event: event\n\n"] generator.single_iteration_generate.assert_called_once() kwargs = generator.single_iteration_generate.call_args.kwargs assert kwargs["app_model"].id == "snippet-1" @@ -313,31 +348,28 @@ def test_generate_single_iteration_delegates_to_workflow_generator( assert kwargs["node_id"] == "iteration-1" assert kwargs["user"] is user assert kwargs["streaming"] is True - assert isinstance(kwargs["session"], Session) - workflow_generator_class.convert_to_event_stream.assert_called_once_with(response) + assert "session" not in kwargs def test_generate_single_iteration_raises_when_draft_workflow_missing( - monkeypatch: pytest.MonkeyPatch, unbound_session_factory: sessionmaker[Session] + generation: SnippetGenerateService, + snippet_reader: SnippetReader, ): - monkeypatch.setattr( - "services.snippet_generate_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_draft_workflow=Mock(return_value=None)), - ) + snippet_reader.workflow = None with pytest.raises(ValueError, match="Workflow not initialized"): - SnippetGenerateService.generate_single_iteration( + generation.generate_single_iteration( snippet=_snippet(), user=_account(), node_id="iteration-1", args={"inputs": {}}, - session_maker=unbound_session_factory, ) def test_generate_single_loop_delegates_to_workflow_generator( monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], + generation: SnippetGenerateService, + snippet_reader: SnippetReader, ) -> None: workflow = _workflow({"nodes": [{"id": "loop-1", "data": {"type": "loop"}}], "edges": []}) snippet = _snippet() @@ -347,21 +379,17 @@ def test_generate_single_loop_delegates_to_workflow_generator( workflow_generator_class = Mock(return_value=generator) workflow_generator_class.convert_to_event_stream = Mock(side_effect=lambda item: item) - monkeypatch.setattr( - "services.snippet_generate_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_draft_workflow=Mock(return_value=workflow)), - ) + snippet_reader.workflow = workflow monkeypatch.setattr("services.snippet_generate_service.WorkflowAppGenerator", workflow_generator_class) - result = SnippetGenerateService.generate_single_loop( + result = generation.generate_single_loop( snippet=snippet, user=user, node_id="loop-1", - args=SimpleNamespace(inputs={"items": [1]}), - session_maker=sqlite_session_factory, + args={"inputs": {"items": [1]}}, ) - assert list(result) == ["event"] + assert list(result) == ["event: event\n\n"] generator.single_loop_generate.assert_called_once() kwargs = generator.single_loop_generate.call_args.kwargs assert kwargs["app_model"].id == "snippet-1" @@ -369,38 +397,19 @@ def test_generate_single_loop_delegates_to_workflow_generator( assert kwargs["node_id"] == "loop-1" assert kwargs["user"] is user assert kwargs["streaming"] is True - assert isinstance(kwargs["session"], Session) - workflow_generator_class.convert_to_event_stream.assert_called_once_with(response) + assert "session" not in kwargs def test_generate_single_loop_raises_when_draft_workflow_missing( - monkeypatch: pytest.MonkeyPatch, unbound_session_factory: sessionmaker[Session] + generation: SnippetGenerateService, + snippet_reader: SnippetReader, ): - monkeypatch.setattr( - "services.snippet_generate_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_draft_workflow=Mock(return_value=None)), - ) + snippet_reader.workflow = None with pytest.raises(ValueError, match="Workflow not initialized"): - SnippetGenerateService.generate_single_loop( + generation.generate_single_loop( snippet=_snippet(), user=_account(), node_id="loop-1", - args=SimpleNamespace(inputs={}), - session_maker=unbound_session_factory, - ) - - -def test_run_published_raises_when_published_workflow_missing(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr( - "services.snippet_generate_service.SnippetService", - lambda *_args, **_kwargs: SimpleNamespace(get_published_workflow=Mock(return_value=None)), - ) - - with pytest.raises(ValueError, match="No published workflow found"): - SnippetGenerateService.run_published( - snippet=_snippet(), - user=_account(), args={"inputs": {}}, - invoke_from="service-api", ) diff --git a/api/tests/unit_tests/services/test_snippet_service.py b/api/tests/unit_tests/services/test_snippet_service.py index 9c6da99855d143..c49434e33cbd69 100644 --- a/api/tests/unit_tests/services/test_snippet_service.py +++ b/api/tests/unit_tests/services/test_snippet_service.py @@ -10,18 +10,13 @@ from sqlalchemy.orm import Session, sessionmaker from enums import DeploymentEdition +from enums.agent import WorkflowAgentBindingType +from extensions.application_services.snippets import build_snippet_service from extensions.storage.storage_type import StorageType from graphon.variables.segments import StringSegment from graphon.variables.types import SegmentType from models.account import Account -from models.agent import ( - Agent, - AgentScope, - AgentSource, - AgentStatus, - WorkflowAgentBindingType, - WorkflowAgentNodeBinding, -) +from models.agent import Agent, AgentScope, AgentSource, AgentStatus, WorkflowAgentNodeBinding from models.enums import AppStatus, CreatorUserRole from models.model import App, AppMode, UploadFile from models.snippet import CustomizedSnippet, SnippetType @@ -34,9 +29,9 @@ WorkflowRun, WorkflowType, ) -from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError -from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError +from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError from services.snippet_service import SnippetService +from services.workflow.snippet_policy import validate_snippet_graph_forbidden_nodes from tests.unit_tests.model_factories import make_account @@ -82,7 +77,7 @@ def test_create_snippet_allows_duplicate_names( existing.name = "shared name" sqlite_session.add(existing) sqlite_session.commit() - service = SnippetService(session_maker=sqlite_session_factory) + service = build_snippet_service(session_maker=sqlite_session_factory) snippet = service.create_snippet( tenant_id="tenant-1", @@ -104,7 +99,7 @@ def test_create_snippet_allows_duplicate_names( def test_validate_snippet_graph_forbidden_nodes_ignores_malformed_nodes() -> None: - SnippetService.validate_snippet_graph_forbidden_nodes( + validate_snippet_graph_forbidden_nodes( { "nodes": [ "not-a-node", @@ -118,7 +113,7 @@ def test_validate_snippet_graph_forbidden_nodes_ignores_malformed_nodes() -> Non def test_validate_snippet_graph_forbidden_nodes_raises_with_node_details() -> None: with pytest.raises(ValueError, match="start-1:start"): - SnippetService.validate_snippet_graph_forbidden_nodes({"nodes": [{"id": "start-1", "data": {"type": "start"}}]}) + validate_snippet_graph_forbidden_nodes({"nodes": [{"id": "start-1", "data": {"type": "start"}}]}) def test_get_snippets_returns_empty_when_tag_filter_has_no_targets( @@ -224,105 +219,6 @@ def test_update_snippet_updates_optional_fields(sqlite_session: Session) -> None assert stored.description == "new description" -def test_sync_draft_workflow_creates_draft_and_updates_input_fields( - monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], - sqlite_session: Session, -) -> None: - service = SnippetService(session_maker=sqlite_session_factory) - monkeypatch.setattr(service, "get_draft_workflow", Mock(return_value=None)) - monkeypatch.setattr( - "services.agent.legacy_workflow_publish_service.WorkflowAgentPublishService.sync_agent_bindings_for_draft", - Mock(return_value={"retired-agent"}), - ) - retire_unowned = Mock() - monkeypatch.setattr( - "services.snippet_service.WorkflowAgentRetirementService.retire_unowned", - retire_unowned, - ) - snippet = _snippet() - account = _account() - - workflow = service.sync_draft_workflow( - snippet=snippet, - graph={"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []}, - unique_hash=None, - account=account, - input_fields=[{"variable": "query"}], - ) - - assert workflow.app_id == snippet.id - assert workflow.kind == WorkflowKind.SNIPPET - assert json.loads(snippet.input_fields) == [{"variable": "query"}] - sqlite_session.expire_all() - stored_workflow = sqlite_session.scalar(select(Workflow).where(Workflow.id == workflow.id)) - stored_snippet = sqlite_session.get(CustomizedSnippet, snippet.id) - assert stored_workflow is not None - assert stored_snippet is not None - assert stored_snippet.input_fields_list == [{"variable": "query"}] - retire_unowned.assert_called_once_with( - tenant_id=snippet.tenant_id, - agent_ids={"retired-agent"}, - account_id=account.id, - ) - - -def test_sync_draft_workflow_raises_when_hash_mismatches( - sqlite_session_factory: sessionmaker[Session], -) -> None: - service = SnippetService(session_maker=sqlite_session_factory) - draft_workflow = _create_workflow( - workflow_id="workflow-1", version=Workflow.VERSION_DRAFT, graph={"nodes": []}, features={} - ) - service.get_draft_workflow = Mock(return_value=draft_workflow) - - with pytest.raises(WorkflowHashNotEqualError): - service.sync_draft_workflow( - snippet=_snippet(), - graph={"nodes": [], "edges": []}, - unique_hash="client-hash", - account=_account(), - ) - - -def test_sync_draft_workflow_updates_existing_draft_and_clears_variables( - monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], - sqlite_session: Session, -) -> None: - service = SnippetService(session_maker=sqlite_session_factory) - workflow = _create_workflow( - workflow_id="workflow-1", - version=Workflow.VERSION_DRAFT, - graph={"nodes": [], "edges": []}, - features={}, - ) - unique_hash = workflow.unique_hash - snippet = _snippet() - account = _account() - monkeypatch.setattr(service, "get_draft_workflow", Mock(return_value=workflow)) - - result = service.sync_draft_workflow( - snippet=snippet, - graph={"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []}, - unique_hash=unique_hash, - account=account, - input_fields=[{"variable": "query"}], - ) - - assert result is workflow - assert workflow.graph_dict["nodes"][0]["id"] == "llm-1" - assert workflow.type == WorkflowType.WORKFLOW - assert workflow.kind == WorkflowKind.SNIPPET - assert workflow.updated_by == account.id - assert workflow.environment_variables == [] - assert workflow.conversation_variables == [] - assert json.loads(snippet.input_fields) == [{"variable": "query"}] - sqlite_session.expire_all() - assert sqlite_session.get(Workflow, workflow.id) is not None - assert sqlite_session.get(CustomizedSnippet, snippet.id) is not None - - def test_update_workflow_updates_marked_fields(sqlite_session: Session) -> None: service = SnippetService.__new__(SnippetService) workflow = _create_workflow( @@ -344,7 +240,9 @@ def test_update_workflow_updates_marked_fields(sqlite_session: Session) -> None: data={"marked_name": "v1", "marked_comment": "first version", "ignored": "value"}, ) - assert result is workflow + assert result is not None + assert result.id == workflow.id + assert result.marked_name == "v1" assert workflow.marked_name == "v1" assert workflow.marked_comment == "first version" assert workflow.updated_by == "account-1" @@ -368,62 +266,34 @@ def test_update_workflow_returns_none_when_missing(sqlite_session: Session) -> N assert result is None -def test_delete_workflow_removes_published_version(sqlite_session: Session) -> None: - service = SnippetService.__new__(SnippetService) - workflow = _create_workflow( - workflow_id="workflow-1", - version="2026-01-01 00:00:00", - graph={"nodes": []}, - features={}, - ) - snippet = _snippet(workflow_id="workflow-2") - sqlite_session.add_all([snippet, workflow]) - sqlite_session.flush() - - result = service.delete_workflow(session=sqlite_session, snippet=snippet, workflow_id="workflow-1") - - assert result is True - sqlite_session.flush() - assert sqlite_session.get(Workflow, "workflow-1") is None - - -def test_delete_workflow_raises_when_missing(sqlite_session: Session) -> None: - service = SnippetService.__new__(SnippetService) - - with pytest.raises(ValueError, match="not found"): - service.delete_workflow(session=sqlite_session, snippet=_snippet(), workflow_id="missing-workflow") - - -def test_delete_workflow_raises_for_draft_version(sqlite_session: Session) -> None: - service = SnippetService.__new__(SnippetService) - workflow = _create_workflow( - workflow_id="workflow-1", - version=Workflow.VERSION_DRAFT, - graph={"nodes": []}, - features={}, +@pytest.mark.parametrize("mismatch", ["tenant_id", "app_id", "kind", "version"]) +def test_update_workflow_preserves_snippet_owner_kind_and_published_constraints( + sqlite_session: Session, mismatch: str +) -> None: + workflow = _create_workflow(workflow_id="workflow-1", version="published", graph={"nodes": []}, features={}) + setattr( + workflow, + mismatch, + { + "tenant_id": "another-tenant", + "app_id": "another-snippet", + "kind": WorkflowKind.STANDARD.value, + "version": Workflow.VERSION_DRAFT, + }[mismatch], ) - snippet = _snippet() - sqlite_session.add_all([snippet, workflow]) + sqlite_session.add(workflow) sqlite_session.flush() + before = (workflow.marked_name, workflow.marked_comment, workflow.updated_by, workflow.updated_at) - with pytest.raises(DraftWorkflowDeletionError): - service.delete_workflow(session=sqlite_session, snippet=snippet, workflow_id="workflow-1") - - -def test_delete_workflow_raises_when_currently_active(sqlite_session: Session) -> None: - service = SnippetService.__new__(SnippetService) - workflow = _create_workflow( - workflow_id="workflow-1", - version="2026-01-01 00:00:00", - graph={"nodes": []}, - features={}, + result = SnippetService.__new__(SnippetService).update_workflow( + session=sqlite_session, + snippet=_snippet(), + workflow_id=workflow.id, + account=_account(), + data={"marked_name": "changed", "marked_comment": "changed"}, ) - snippet = _snippet(workflow_id="workflow-1") - sqlite_session.add_all([snippet, workflow]) - sqlite_session.flush() - - with pytest.raises(WorkflowInUseError): - service.delete_workflow(session=sqlite_session, snippet=snippet, workflow_id="workflow-1") + assert result is None + assert (workflow.marked_name, workflow.marked_comment, workflow.updated_by, workflow.updated_at) == before def test_get_default_block_configs_skips_empty_defaults(monkeypatch: pytest.MonkeyPatch) -> None: @@ -468,119 +338,6 @@ def test_get_default_block_config_returns_none_for_empty_default(monkeypatch: py assert service.get_default_block_config("llm") is None -def test_restore_published_snippet_workflow_to_draft_copies_source_snapshot( - monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], - sqlite_session: Session, -) -> None: - snippet = _snippet() - account = _account("account-2") - source_graph = {"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []} - source_features = {"opening_statement": "hello"} - source_workflow = _create_workflow( - workflow_id="published-workflow", - version="2026-04-28 00:00:00", - graph=source_graph, - features=source_features, - ) - draft_workflow = _create_workflow( - workflow_id="draft-workflow", - version=Workflow.VERSION_DRAFT, - graph={"nodes": [], "edges": []}, - features={}, - ) - service = SnippetService(session_maker=sqlite_session_factory) - - monkeypatch.setattr(service, "get_published_workflow_by_id", Mock(return_value=source_workflow)) - monkeypatch.setattr(service, "get_draft_workflow", Mock(return_value=draft_workflow)) - monkeypatch.setattr( - "services.agent.legacy_workflow_publish_service.WorkflowAgentPublishService.restore_agent_node_bindings_to_draft", - Mock(return_value={"retired-agent"}), - ) - retire_unowned = Mock() - monkeypatch.setattr( - "services.snippet_service.WorkflowAgentRetirementService.retire_unowned", - retire_unowned, - ) - - result = service.restore_published_workflow_to_draft( - snippet=snippet, - workflow_id=source_workflow.id, - account=account, - ) - - assert result is draft_workflow - assert draft_workflow.graph_dict == source_graph - assert draft_workflow.features_dict == source_features - assert draft_workflow.updated_by == account.id - sqlite_session.expire_all() - stored = sqlite_session.get(Workflow, draft_workflow.id) - assert stored is not None - assert stored.graph_dict == source_graph - retire_unowned.assert_called_once_with( - tenant_id=snippet.tenant_id, - agent_ids={"retired-agent"}, - account_id=account.id, - ) - - -def test_restore_published_snippet_workflow_to_draft_raises_when_source_missing( - monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], -) -> None: - snippet = _snippet() - account = _account("account-2") - service = SnippetService(session_maker=sqlite_session_factory) - - monkeypatch.setattr(service, "get_published_workflow_by_id", Mock(return_value=None)) - - with pytest.raises(WorkflowNotFoundError): - service.restore_published_workflow_to_draft( - snippet=snippet, - workflow_id="missing-workflow", - account=account, - ) - - -def test_restore_published_snippet_workflow_to_draft_adds_new_draft( - monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], - sqlite_session: Session, -) -> None: - snippet = _snippet() - account = _account("account-2") - source_workflow = _create_workflow( - workflow_id="published-workflow", - version="2026-04-28 00:00:00", - graph={"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []}, - features={}, - ) - new_draft_workflow = _create_workflow( - workflow_id="draft-workflow", - version=Workflow.VERSION_DRAFT, - graph={"nodes": [], "edges": []}, - features={}, - ) - service = SnippetService(session_maker=sqlite_session_factory) - - monkeypatch.setattr(service, "get_published_workflow_by_id", Mock(return_value=source_workflow)) - monkeypatch.setattr(service, "get_draft_workflow", Mock(return_value=None)) - monkeypatch.setattr( - "services.snippet_service.apply_published_workflow_snapshot_to_draft", - Mock(return_value=(new_draft_workflow, True)), - ) - - result = service.restore_published_workflow_to_draft( - snippet=snippet, - workflow_id=source_workflow.id, - account=account, - ) - - assert result is new_draft_workflow - sqlite_session.expire_all() - assert sqlite_session.get(Workflow, new_draft_workflow.id) is not None - - def test_get_published_workflow_returns_none_without_workflow_id() -> None: service = SnippetService.__new__(SnippetService) @@ -597,7 +354,7 @@ def test_get_published_workflow_by_id_raises_for_draft( ) sqlite_session.add(draft_workflow) sqlite_session.commit() - service = SnippetService(session_maker=sqlite_session_factory) + service = build_snippet_service(session_maker=sqlite_session_factory) with pytest.raises(IsDraftWorkflowError): service.get_published_workflow_by_id( @@ -606,12 +363,11 @@ def test_get_published_workflow_by_id_raises_for_draft( ) -def test_publish_workflow_raises_when_draft_missing(sqlite_session: Session) -> None: - service = SnippetService.__new__(SnippetService) +def test_publish_workflow_raises_when_draft_missing(sqlite_session_factory: sessionmaker[Session]) -> None: + service = build_snippet_service(session_maker=sqlite_session_factory) with pytest.raises(ValueError, match="No valid workflow found"): service.publish_workflow( - session=sqlite_session, snippet=_snippet(), account=_account(), ) @@ -620,7 +376,7 @@ def test_publish_workflow_raises_when_draft_missing(sqlite_session: Session) -> def test_publish_workflow_creates_snapshot_and_updates_snippet( monkeypatch: pytest.MonkeyPatch, sqlite_session: Session ) -> None: - service = SnippetService.__new__(SnippetService) + service = build_snippet_service(session=sqlite_session) draft_workflow = _create_workflow( workflow_id="draft-workflow", version=Workflow.VERSION_DRAFT, @@ -641,25 +397,25 @@ def test_publish_workflow_creates_snapshot_and_updates_snippet( ) snippet = _snippet() sqlite_session.add_all([draft_workflow, snippet]) - sqlite_session.flush() + sqlite_session.commit() monkeypatch.setattr( - "services.agent.legacy_workflow_publish_service.WorkflowAgentPublishService.copy_agent_node_bindings_to_published", + "services.agent.workflow_publish_service.WorkflowAgentPublishService.copy_agent_node_bindings_to_published", Mock(return_value=set()), ) result = service.publish_workflow( - session=sqlite_session, snippet=snippet, account=_account(), ) + sqlite_session.refresh(snippet) assert result.kind == WorkflowKind.SNIPPET assert snippet.version == 2 assert snippet.is_published is True assert snippet.workflow_id == result.id assert snippet.updated_by == "account-1" sqlite_session.flush() - assert sqlite_session.get(Workflow, result.id) is result + assert sqlite_session.get(Workflow, result.id) is not None assert sqlite_session.get(CustomizedSnippet, snippet.id).workflow_id == result.id @@ -780,11 +536,11 @@ def test_delete_snippet_releases_last_owner_and_retries_archived_agent_cleanup_a cleanup_delay = Mock(side_effect=lambda **_kwargs: events.append("cleanup-hidden-app")) enqueue_collection = Mock(side_effect=lambda **_kwargs: events.append("enqueue-agent-purge")) monkeypatch.setattr( - "services.agent.legacy_retirement_service.remove_app_and_related_data_task.delay", + "services.agent.retirement_service.remove_app_and_related_data_task.delay", cleanup_delay, ) monkeypatch.setattr( - "services.agent.legacy_retirement_service.enqueue_agent_resource_collection", + "services.agent.retirement_service.enqueue_agent_resource_collection", enqueue_collection, ) @@ -878,7 +634,7 @@ def test_delete_snippet_keeps_agent_with_persisted_external_owner( sqlite_session.flush() cleanup_delay = Mock() monkeypatch.setattr( - "services.agent.legacy_retirement_service.remove_app_and_related_data_task.delay", + "services.agent.retirement_service.remove_app_and_related_data_task.delay", cleanup_delay, ) @@ -961,7 +717,7 @@ def test_delete_archived_workflow_run_files_removes_prefixed_objects(monkeypatch archive_storage.delete_object.assert_called_once_with("tenant-1/app_id=snippet-1/run.json") -def test_workflow_run_queries_delegate_to_repositories(monkeypatch: pytest.MonkeyPatch) -> None: +def test_workflow_run_queries_delegate_to_repositories(monkeypatch: pytest.MonkeyPatch, *, tool_providers) -> None: service = SnippetService.__new__(SnippetService) workflow_run_repo = SimpleNamespace( get_paginated_workflow_runs=Mock(return_value=SimpleNamespace(data=[])), @@ -982,7 +738,10 @@ def test_workflow_run_queries_delegate_to_repositories(monkeypatch: pytest.Monke assert service.get_snippet_workflow_runs(snippet=snippet, args={"limit": "5", "last_id": "run-0"}).data == [] assert service.get_snippet_workflow_run(snippet=snippet, run_id="run-1").id == "run-1" - assert service.get_snippet_workflow_run_node_executions(snippet=snippet, run_id="run-1") == expected_traces + assert ( + service.get_snippet_workflow_run_node_executions(snippet=snippet, run_id="run-1", tool_providers=tool_providers) + == expected_traces + ) assert ( service.get_snippet_node_last_run( snippet=snippet, @@ -1005,7 +764,10 @@ def test_workflow_run_queries_delegate_to_repositories(monkeypatch: pytest.Monke workflow_run_id="run-1", ) mock_assemble.assert_called_once_with( - node_execution_repo.get_executions_by_workflow_run.return_value, node_execution_repo, session=ANY + node_execution_repo.get_executions_by_workflow_run.return_value, + node_execution_repo, + session=ANY, + tool_providers=tool_providers, ) node_execution_repo.get_node_last_execution.assert_called_once_with( tenant_id="tenant-1", @@ -1015,14 +777,13 @@ def test_workflow_run_queries_delegate_to_repositories(monkeypatch: pytest.Monke ) -def test_workflow_run_node_executions_returns_empty_when_run_missing() -> None: +def test_workflow_run_node_executions_returns_empty_when_run_missing(*, tool_providers) -> None: service = SnippetService.__new__(SnippetService) service._node_execution_service_repo = SimpleNamespace(get_executions_by_workflow_run=Mock()) service.get_snippet_workflow_run = Mock(return_value=None) result = service.get_snippet_workflow_run_node_executions( - snippet=_snippet(), - run_id="missing-run", + snippet=_snippet(), run_id="missing-run", tool_providers=tool_providers ) assert result == [] @@ -1040,3 +801,174 @@ def test_increment_use_count_adds_updated_snippet(sqlite_session: Session) -> No assert snippet.use_count == 3 sqlite_session.flush() assert sqlite_session.get(CustomizedSnippet, snippet.id).use_count == 3 + + +@pytest.mark.parametrize("concurrent_edit", [None, "graph", "binding", "agent"]) +def test_snippet_skill_validation_releases_connections_and_rechecks_state( + sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, concurrent_edit: str | None +) -> None: + import io + import zipfile + + from sqlalchemy import Select, func + from sqlalchemy.dialects import postgresql + from sqlalchemy.engine import Engine + from sqlalchemy.orm import ORMExecuteState + from sqlalchemy.pool import QueuePool + + from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource + from models.agent_config_entities import AgentSoulConfig, AgentSoulModelConfig, AgentSoulPromptConfig + from models.skill import AgentSkillBindingSnapshot, Skill, SkillVersion, SkillVersionManifest + from models.tools import ToolFile + + soul = AgentSoulConfig( + model=AgentSoulModelConfig(plugin_id="langgenius/openai", model_provider="openai", model="test"), + prompt=AgentSoulPromptConfig(system_prompt="Use [§skill:archived-skill§]"), + ) + with sqlite_session_factory.begin() as session: + draft = _create_workflow(workflow_id="draft", version="draft", graph={}, features={}) + session.add_all([draft, _snippet()]) + draft.graph = json.dumps( + { + "nodes": [{"id": "agent", "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}], + "edges": [], + } + ) + archive = ToolFile( + tenant_id="tenant-1", + user_id="account-1", + conversation_id=None, + file_key="skills/published.zip", + mimetype="application/zip", + ) + session.add_all( + [ + archive, + Agent( + id="agent", + tenant_id="tenant-1", + name="Agent", + scope=AgentScope.ROSTER, + source=AgentSource.ROSTER, + active_config_snapshot_id="soul", + ), + AgentConfigSnapshot(id="soul", tenant_id="tenant-1", agent_id="agent", version=1, config_snapshot=soul), + AgentConfigSnapshot( + id="new-soul", tenant_id="tenant-1", agent_id="agent", version=2, config_snapshot=soul + ), + WorkflowAgentNodeBinding( + id="binding", + tenant_id="tenant-1", + app_id="snippet-1", + workflow_id="draft", + workflow_version="draft", + node_id="agent", + binding_type=WorkflowAgentBindingType.ROSTER_AGENT, + agent_id="agent", + current_snapshot_id="soul", + node_job_config={}, + created_by="account-1", + ), + Skill( + id="skill", + tenant_id="tenant-1", + name="draft-name", + display_name="Skill", + latest_published_version_id="version", + ), + SkillVersion( + id="version", + skill_id="skill", + version_number=1, + manifest=SkillVersionManifest(files=[]), + archive_tool_file_id=archive.id, + hash_code="hash", + archive_size=123, + ), + AgentSkillBindingSnapshot( + tenant_id="tenant-1", agent_id="agent", config_snapshot_id="soul", skill_id="skill", priority=0 + ), + ] + ) + archive_bytes = io.BytesIO() + with zipfile.ZipFile(archive_bytes, "w") as zip_file: + zip_file.writestr("SKILL.md", "---\nname: archived-skill\ndescription: Published skill\n---\nInstructions") + sessions: list[Session] = [] + statements: list[str] = [] + downloads: list[str] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def query(execution: ORMExecuteState) -> None: + if isinstance(execution.statement, Select): + statements.append(str(execution.statement.compile(dialect=postgresql.dialect()))) + + def load(key: str) -> bytes: + assert all(not session.in_transaction() for session in sessions) + engine = sqlite_session_factory.kw["bind"] + assert isinstance(engine, Engine) + assert isinstance(engine.pool, QueuePool) + assert engine.pool.checkedout() == 0 + assert not any("FOR UPDATE" in statement for statement in statements) + downloads.append(key) + if concurrent_edit: + with sqlite_session_factory.begin() as session: + if concurrent_edit == "graph": + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = '{"nodes": [], "edges": []}' + elif concurrent_edit == "binding": + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + else: + agent = session.get(Agent, "agent") + assert agent is not None + agent.active_config_snapshot_id = "new-soul" + return archive_bytes.getvalue() + + monkeypatch.setattr("services.skill_management_service.storage.load_once", load) + service = build_snippet_service(session_maker=sqlite_session_factory) + event.listen(sqlite_session_factory, "after_begin", opened) + event.listen(sqlite_session_factory, "do_orm_execute", query) + try: + if concurrent_edit: + with pytest.raises(WorkflowHashNotEqualError): + service.publish_workflow(snippet=_snippet(), account=_account()) + else: + service.publish_workflow(snippet=_snippet(), account=_account()) + finally: + event.remove(sqlite_session_factory, "after_begin", opened) + event.remove(sqlite_session_factory, "do_orm_execute", query) + assert downloads == ["skills/published.zip"] + with sqlite_session_factory() as session: + snippet = session.get(CustomizedSnippet, "snippet-1") + assert snippet is not None + assert snippet.is_published is (concurrent_edit is None) + versions = session.scalar(select(func.count()).select_from(Workflow).where(Workflow.version != "draft")) + assert versions == (0 if concurrent_edit else 1) + + +def test_snippet_publication_rolls_back_version_and_pointer_on_binding_write_failure( + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + from sqlalchemy import func + + with sqlite_session_factory.begin() as session: + session.add_all([_snippet(), _create_workflow(workflow_id="draft", version="draft", graph={}, features={})]) + monkeypatch.setattr( + "services.agent.workflow_publish_service.WorkflowAgentPublishService.copy_agent_node_bindings_to_published", + Mock(side_effect=ValueError("binding write failed")), + ) + service = build_snippet_service(session_maker=sqlite_session_factory) + with pytest.raises(ValueError, match="binding write failed"): + service.publish_workflow(snippet=_snippet(), account=_account()) + with sqlite_session_factory() as session: + snippet = session.get(CustomizedSnippet, "snippet-1") + assert snippet is not None + assert snippet.workflow_id is None + assert snippet.is_published is False + assert snippet.version == 1 + assert session.scalar(select(func.count()).select_from(Workflow).where(Workflow.version != "draft")) == 0 diff --git a/api/tests/unit_tests/services/test_trial_app_generation_adapters.py b/api/tests/unit_tests/services/test_trial_app_generation_adapters.py index 826bcc35de07b1..8be7abd003c1bb 100644 --- a/api/tests/unit_tests/services/test_trial_app_generation_adapters.py +++ b/api/tests/unit_tests/services/test_trial_app_generation_adapters.py @@ -8,13 +8,14 @@ from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import QueuePool -from core.app.apps.chat.app_generator import ChatAppGenerator from core.app.entities.app_invoke_entities import InvokeFrom from models import Account, App, AppMode, Conversation from models.enums import ConversationFromSource from models.model import AccountTrialAppRecord from repositories.trial_app_repository import TrialAppRepository from services.account_errors import AccountNotFoundError +from services.app.generation.adapters.chat import ChatAppGenerator +from services.app.generation.response import convert_to_event_stream from services.app_definition_query_service import AppDefinitionUnavailableError from services.app_generate_service import AppGenerateService from services.errors.app_model_config import AppModelConfigBrokenError @@ -22,6 +23,8 @@ from services.trial_app_access_service import TrialAppRef from services.trial_app_generation_adapters import AppGenerateServiceRuntime from services.trial_app_generation_service import GenerationResponse, TrialAppGenerationService +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables _ARGS: dict[str, object] = {"inputs": {"count": 0}, "query": "hello", "auto_generate_name": False} @@ -36,7 +39,13 @@ class _Harness: @pytest.fixture -def harness(sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session]) -> _Harness: +def harness( + sqlite_engine: Engine, + sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, +) -> _Harness: with sqlite_session_factory.begin() as session: app = App(tenant_id=str(uuid4()), name="Trial app", mode=AppMode.CHAT, enable_site=True, enable_api=False) account = Account(name="Viewer", email="viewer@example.com") @@ -64,7 +73,9 @@ def close(self) -> None: factory = sessionmaker(bind=sqlite_engine, class_=TrackedSession, expire_on_commit=True) return _Harness( - runtime=AppGenerateServiceRuntime(session_factory=cast(sessionmaker[Session], factory)), + runtime=AppGenerateServiceRuntime( + session_factory=cast(sessionmaker[Session], factory), variables=workflow_variables, runtime=workflow_runtime + ), app=TrialAppRef(app_id=app.id, tenant_id=app.tenant_id, app_mode=app.mode), account_id=account.id, closed_sessions=closed_sessions, @@ -103,8 +114,10 @@ def _patch_generation( expected_streaming = streaming expected_args = args - def legacy_generate( + def boundary_generate( *, + variables: WorkflowExecutionVariables, + runtime: WorkflowRuntime, session: Session, app_model: App, user: Account, @@ -112,6 +125,8 @@ def legacy_generate( invoke_from: InvokeFrom, streaming: bool, ) -> GenerationResponse: + assert runtime is harness.runtime._runtime + assert variables is harness.runtime._variables assert len(harness.closed_sessions) == 1 read_session = harness.closed_sessions[0] assert not read_session.in_transaction() @@ -131,7 +146,7 @@ def legacy_generate( assert args == expected_args return generate(session) - monkeypatch.setattr(AppGenerateService, "generate", legacy_generate) + monkeypatch.setattr(AppGenerateService, "generate", boundary_generate) def test_mapping_result_is_unchanged_and_generation_writes_are_committed( @@ -351,8 +366,10 @@ def _add_conversation(harness: _Harness, session_factory: sessionmaker[Session]) def _patch_chat_generation(monkeypatch: pytest.MonkeyPatch, harness: _Harness) -> None: - def legacy_generate( + def boundary_generate( *, + variables: WorkflowExecutionVariables, + runtime: WorkflowRuntime, session: Session, app_model: App, user: Account, @@ -360,16 +377,22 @@ def legacy_generate( invoke_from: InvokeFrom, streaming: bool, ) -> GenerationResponse: + assert runtime is harness.runtime._runtime + assert variables is harness.runtime._variables assert len(harness.closed_sessions) == 1 assert not harness.closed_sessions[0].in_transaction() assert session is not harness.closed_sessions[0] assert not session.in_transaction() - response = ChatAppGenerator().generate( + response = ChatAppGenerator( + retrieval=runtime.retrieval, + annotations=runtime.annotation_replies, + records=runtime.chat_records, + ).generate( app_model=app_model, user=user, args=args, invoke_from=invoke_from, streaming=streaming, session=session ) - return cast(GenerationResponse, ChatAppGenerator.convert_to_event_stream(response)) + return cast(GenerationResponse, convert_to_event_stream(response)) - monkeypatch.setattr(AppGenerateService, "generate", legacy_generate) + monkeypatch.setattr(AppGenerateService, "generate", boundary_generate) @pytest.mark.parametrize("mismatch", ["missing", "app", "account", "source", "end-user", "deleted"]) diff --git a/api/tests/unit_tests/services/test_webhook_service_additional.py b/api/tests/unit_tests/services/test_webhook_service_additional.py index 9bf7f9d50b6515..54eaf56d702028 100644 --- a/api/tests/unit_tests/services/test_webhook_service_additional.py +++ b/api/tests/unit_tests/services/test_webhook_service_additional.py @@ -5,7 +5,6 @@ import pytest from flask import Flask -from werkzeug.exceptions import RequestEntityTooLarge from core.workflow.nodes.trigger_webhook.entities import ( ContentType, @@ -16,6 +15,7 @@ from graphon.variables.types import SegmentType from models.trigger import WorkflowWebhookTrigger from services.trigger import webhook_service as service_module +from services.trigger.errors import WebhookBodyTooLargeError from services.trigger.webhook_service import WebhookService @@ -67,7 +67,7 @@ def test_extract_webhook_data_should_raise_for_request_too_large( apply_config_overrides(monkeypatch, WEBHOOK_REQUEST_BODY_MAX_SIZE=1) with flask_app.test_request_context("/webhook", method="POST", data="ab"): - with pytest.raises(RequestEntityTooLarge): + with pytest.raises(WebhookBodyTooLargeError): WebhookService.extract_webhook_data(_workflow_trigger()) def test_extract_octet_stream_body_should_return_none_when_empty_payload(self, flask_app: Flask) -> None: @@ -276,7 +276,9 @@ def test_generate_webhook_response_should_fallback_when_response_body_is_not_jso assert "message" in body def test_generate_webhook_id_should_return_24_character_identifier(self) -> None: - webhook_id = WebhookService.generate_webhook_id() + from repositories.trigger.workflow_repository import WorkflowTriggerRepository + + webhook_id = WorkflowTriggerRepository.generate_webhook_id() assert isinstance(webhook_id, str) assert len(webhook_id) == 24 diff --git a/api/tests/unit_tests/services/test_webhook_service_relationships.py b/api/tests/unit_tests/services/test_webhook_service_relationships.py index ed22c91624a7d9..4870189a4adbc1 100644 --- a/api/tests/unit_tests/services/test_webhook_service_relationships.py +++ b/api/tests/unit_tests/services/test_webhook_service_relationships.py @@ -1,94 +1,69 @@ -from types import SimpleNamespace -from typing import cast -from unittest.mock import MagicMock +"""Webhook draft sync preserves stable URLs using the injected database.""" -import pytest -from sqlalchemy import Engine, Table, select +from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker -from core.trigger.constants import TRIGGER_WEBHOOK_NODE_TYPE -from models.base import TypeBase -from models.model import App from models.trigger import WorkflowWebhookTrigger -from models.workflow import Workflow -from services.trigger import webhook_service from services.trigger.webhook_service import WebhookService +from tests.unit_tests.model_factories import make_app, make_workflow def test_draft_sync_preserves_webhook_id_when_deleted_node_is_restored( - monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine + sqlite_session_factory: sessionmaker[Session], ) -> None: - webhook_trigger_table = cast(Table, WorkflowWebhookTrigger.__table__) - TypeBase.metadata.create_all(sqlite_engine, tables=[webhook_trigger_table]) - session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False) - app = cast( - App, - SimpleNamespace(id="00000000-0000-0000-0000-000000000001", tenant_id="tenant-1", created_by="user-1"), - ) - webhook_trigger = WorkflowWebhookTrigger( + app = make_app(created_by="account-1") + trigger = WorkflowWebhookTrigger( app_id=app.id, tenant_id=app.tenant_id, node_id="webhook-node", webhook_id="stable-webhook-id", created_by=app.created_by, ) - with session_maker.begin() as session: - session.add(webhook_trigger) - original_record_id = webhook_trigger.id - - redis = MagicMock() - redis.get.return_value = None - redis.lock.return_value.acquire.return_value = True - monkeypatch.setattr(webhook_service, "db", SimpleNamespace(engine=sqlite_engine)) - monkeypatch.setattr(webhook_service, "redis_client", redis) - - deleted_draft = cast(Workflow, SimpleNamespace(walk_nodes=lambda _node_type: iter(()))) - WebhookService.sync_webhook_relationships(app, deleted_draft, remove_stale=False) - - restored_draft = cast( - Workflow, - SimpleNamespace(walk_nodes=lambda _node_type: iter([("webhook-node", {"type": TRIGGER_WEBHOOK_NODE_TYPE})])), + with sqlite_session_factory.begin() as session: + session.add_all([app, trigger]) + original_id = trigger.id + WebhookService.sync_webhook_relationships( + app, + make_workflow(), + remove_stale=False, + session_factory=sqlite_session_factory, ) - WebhookService.sync_webhook_relationships(app, restored_draft, remove_stale=False) - - with Session(sqlite_engine) as session: - records = session.scalars(select(WorkflowWebhookTrigger)).all() - - assert len(records) == 1 - assert records[0].id == original_record_id - assert records[0].webhook_id == "stable-webhook-id" - - -def test_published_sync_removes_stale_webhook_relationships( - monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine -) -> None: - webhook_trigger_table = cast(Table, WorkflowWebhookTrigger.__table__) - TypeBase.metadata.create_all(sqlite_engine, tables=[webhook_trigger_table]) - session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False) - app = cast( - App, - SimpleNamespace(id="00000000-0000-0000-0000-000000000001", tenant_id="tenant-1", created_by="user-1"), + restored = make_workflow( + graph={"nodes": [{"id": "webhook-node", "data": {"type": "trigger-webhook"}}], "edges": []} ) - webhook_trigger = WorkflowWebhookTrigger( - app_id=app.id, - tenant_id=app.tenant_id, - node_id="deleted-webhook-node", - webhook_id="deleted-webhook-id", - created_by=app.created_by, + WebhookService.sync_webhook_relationships( + app, + restored, + remove_stale=False, + session_factory=sqlite_session_factory, ) - with session_maker.begin() as session: - session.add(webhook_trigger) - - redis = MagicMock() - redis.get.return_value = None - redis.lock.return_value.acquire.return_value = True - monkeypatch.setattr(webhook_service, "db", SimpleNamespace(engine=sqlite_engine)) - monkeypatch.setattr(webhook_service, "redis_client", redis) + with sqlite_session_factory() as session: + records = session.scalars(select(WorkflowWebhookTrigger)).all() + assert len(records) == 1 + assert records[0].id == original_id + assert records[0].webhook_id == "stable-webhook-id" - published_workflow = cast(Workflow, SimpleNamespace(walk_nodes=lambda _node_type: iter(()))) - WebhookService.sync_webhook_relationships(app, published_workflow, remove_stale=True) - with session_maker() as session: - records = session.scalars(select(WorkflowWebhookTrigger)).all() - assert records == [] - redis.delete.assert_called_once_with(f"{WebhookService.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:deleted-webhook-node") +def test_publication_sync_removes_stale_webhooks(sqlite_session_factory: sessionmaker[Session]) -> None: + app = make_app(created_by="account-1") + with sqlite_session_factory.begin() as session: + session.add_all( + [ + app, + WorkflowWebhookTrigger( + app_id=app.id, + tenant_id=app.tenant_id, + node_id="deleted", + webhook_id="deleted-webhook", + created_by=app.created_by, + ), + ] + ) + WebhookService.sync_webhook_relationships( + app, + make_workflow(), + remove_stale=True, + session_factory=sqlite_session_factory, + ) + with sqlite_session_factory() as session: + assert session.scalars(select(WorkflowWebhookTrigger)).all() == [] diff --git a/api/tests/unit_tests/services/test_workflow_app_log_query_service.py b/api/tests/unit_tests/services/test_workflow_app_log_query_service.py index f5a3c61492e3d1..085180def073d8 100644 --- a/api/tests/unit_tests/services/test_workflow_app_log_query_service.py +++ b/api/tests/unit_tests/services/test_workflow_app_log_query_service.py @@ -7,8 +7,8 @@ import pytest from models.enums import AppTriggerType -from repositories.workflow_app_log_query_repository import WorkflowAppLogQueryRepository -from services.workflow_app_log_query_service import ( +from repositories.workflow.app_log_repository import WorkflowAppLogRepository +from services.workflow.app_log_query_service import ( WorkflowAppLogItem, WorkflowAppLogPage, WorkflowAppLogQueryService, @@ -60,7 +60,7 @@ def test_list_logs_enriches_plugin_icons() -> None: service = _service_with_metadata(json.dumps(metadata)) with patch( - "services.workflow_app_log_query_service.PluginService.get_plugin_icon_url", + "services.workflow.app_log_query_service.PluginService.get_plugin_icon_url", side_effect=["https://cdn/light.png", "https://cdn/dark.png"], ) as get_icon_url: result = service.list_logs(tenant_id="tenant-1", app_id="app-1", detail=True) @@ -76,7 +76,7 @@ def test_list_logs_enriches_plugin_icons() -> None: def test_list_logs_does_not_fetch_icons_for_non_plugin_metadata() -> None: service = _service_with_metadata(json.dumps({"type": AppTriggerType.TRIGGER_WEBHOOK.value})) - with patch("services.workflow_app_log_query_service.PluginService.get_plugin_icon_url") as get_icon_url: + with patch("services.workflow.app_log_query_service.PluginService.get_plugin_icon_url") as get_icon_url: result = service.list_logs(tenant_id="tenant-1", app_id="app-1", detail=True) assert result.data[0].details == {"trigger_metadata": {"type": AppTriggerType.TRIGGER_WEBHOOK.value}} @@ -106,14 +106,14 @@ def test_safe_json_loads(value: object, expected: object) -> None: def test_safe_parse_uuid_rejects_short_and_invalid_values() -> None: - assert WorkflowAppLogQueryRepository._safe_parse_uuid("short") is None - assert WorkflowAppLogQueryRepository._safe_parse_uuid("x" * 40) is None + assert WorkflowAppLogRepository._safe_parse_uuid("short") is None + assert WorkflowAppLogRepository._safe_parse_uuid("x" * 40) is None def test_safe_parse_uuid_accepts_uuid() -> None: raw = str(uuid.uuid4()) - result = WorkflowAppLogQueryRepository._safe_parse_uuid(raw) + result = WorkflowAppLogRepository._safe_parse_uuid(raw) assert result is not None assert str(result) == raw diff --git a/api/tests/unit_tests/services/test_workflow_node_execution_trace_service.py b/api/tests/unit_tests/services/test_workflow_node_execution_trace_service.py index eb2cc1077f666c..e4555d4d4139e8 100644 --- a/api/tests/unit_tests/services/test_workflow_node_execution_trace_service.py +++ b/api/tests/unit_tests/services/test_workflow_node_execution_trace_service.py @@ -77,7 +77,7 @@ def _repository(full_process_data: dict[str, object] | None) -> DifyAPIWorkflowN return cast(DifyAPIWorkflowNodeExecutionRepository, repository) -def test_assemble_expands_retry_history_before_terminal_trace(sqlite_session: Session) -> None: +def test_assemble_expands_retry_history_before_terminal_trace(sqlite_session: Session, *, tool_providers) -> None: process_data = { "request": "successful-attempt", RETRY_HISTORY_PROCESS_DATA_KEY: [_retry_attempt(2), _retry_attempt(1)], @@ -85,7 +85,9 @@ def test_assemble_expands_retry_history_before_terminal_trace(sqlite_session: Se execution = _execution(process_data) repository = _repository(process_data) - traces = assemble_workflow_node_execution_traces([execution], repository, session=sqlite_session) + traces = assemble_workflow_node_execution_traces( + [execution], repository, session=sqlite_session, tool_providers=tool_providers + ) assert [trace.id for trace in traces] == ["exec-1:retry:1", "exec-1:retry:2", "exec-1"] assert [trace.status for trace in traces] == ["retry", "retry", "succeeded"] @@ -104,18 +106,20 @@ def test_assemble_expands_retry_history_before_terminal_trace(sqlite_session: Se repository.load_full_process_data.assert_called_once_with(execution) -def test_assemble_keeps_old_execution_without_retry_history(sqlite_session: Session) -> None: +def test_assemble_keeps_old_execution_without_retry_history(sqlite_session: Session, *, tool_providers) -> None: process_data = {"request": "terminal"} execution = _execution(process_data) - traces = assemble_workflow_node_execution_traces([execution], _repository(process_data), session=sqlite_session) + traces = assemble_workflow_node_execution_traces( + [execution], _repository(process_data), session=sqlite_session, tool_providers=tool_providers + ) assert len(traces) == 1 assert traces[0].id == "exec-1" assert traces[0].process_data == process_data -def test_assemble_skips_malformed_and_duplicate_retry_attempts(sqlite_session: Session) -> None: +def test_assemble_skips_malformed_and_duplicate_retry_attempts(sqlite_session: Session, *, tool_providers) -> None: process_data = { RETRY_HISTORY_PROCESS_DATA_KEY: [ _retry_attempt(2), @@ -126,39 +130,43 @@ def test_assemble_skips_malformed_and_duplicate_retry_attempts(sqlite_session: S } traces = assemble_workflow_node_execution_traces( - [_execution(process_data)], _repository(process_data), session=sqlite_session + [_execution(process_data)], _repository(process_data), session=sqlite_session, tool_providers=tool_providers ) assert [trace.retry_index for trace in traces[:-1]] == [1, 2] assert traces[0].error == "attempt 1 failed" -def test_assemble_truncates_retry_attempt_fields(sqlite_session: Session) -> None: +def test_assemble_truncates_retry_attempt_fields(sqlite_session: Session, *, tool_providers) -> None: process_data = {RETRY_HISTORY_PROCESS_DATA_KEY: [_retry_attempt(1, outputs={"body": "x" * 2_000_000})]} traces = assemble_workflow_node_execution_traces( - [_execution(process_data)], _repository(process_data), session=sqlite_session + [_execution(process_data)], _repository(process_data), session=sqlite_session, tool_providers=tool_providers ) assert traces[0].outputs_truncated is True assert traces[-1].id == "exec-1" -def test_assemble_falls_back_to_inline_process_data_when_loader_fails(sqlite_session: Session) -> None: +def test_assemble_falls_back_to_inline_process_data_when_loader_fails( + sqlite_session: Session, *, tool_providers +) -> None: process_data = {RETRY_HISTORY_PROCESS_DATA_KEY: [_retry_attempt(1)]} execution = _execution(process_data) repository = _repository(None) repository.load_full_process_data.side_effect = OSError("storage unavailable") - traces = assemble_workflow_node_execution_traces([execution], repository, session=sqlite_session) + traces = assemble_workflow_node_execution_traces( + [execution], repository, session=sqlite_session, tool_providers=tool_providers + ) assert [trace.id for trace in traces] == ["exec-1:retry:1", "exec-1"] -def test_virtual_trace_validates_through_node_execution_response(sqlite_session: Session) -> None: +def test_virtual_trace_validates_through_node_execution_response(sqlite_session: Session, *, tool_providers) -> None: process_data = {RETRY_HISTORY_PROCESS_DATA_KEY: [_retry_attempt(1)]} traces = assemble_workflow_node_execution_traces( - [_execution(process_data)], _repository(process_data), session=sqlite_session + [_execution(process_data)], _repository(process_data), session=sqlite_session, tool_providers=tool_providers ) response = WorkflowRunNodeExecutionListResponse.model_validate({"data": traces}, from_attributes=True) diff --git a/api/tests/unit_tests/services/test_workflow_run_service.py b/api/tests/unit_tests/services/test_workflow_run_service.py index 646266f826d4cd..4d3b1b771018a3 100644 --- a/api/tests/unit_tests/services/test_workflow_run_service.py +++ b/api/tests/unit_tests/services/test_workflow_run_service.py @@ -19,11 +19,10 @@ def service_dependencies() -> tuple[MagicMock, MagicMock]: return MagicMock(), MagicMock() -def _service(dependencies: tuple[MagicMock, MagicMock]) -> WorkflowRunService: +def _service(dependencies: tuple[MagicMock, MagicMock], *, tool_providers) -> WorkflowRunService: node_executions, workflow_runs = dependencies return WorkflowRunService( - workflow_runs=workflow_runs, - node_executions=node_executions, + workflow_runs=workflow_runs, node_executions=node_executions, tool_providers=tool_providers ) @@ -57,11 +56,10 @@ def _workflow_run( class TestWorkflowRunServiceQueries: def test_get_paginate_workflow_runs_should_forward_filters_and_parse_limit( - self, - service_dependencies: tuple[MagicMock, MagicMock], + self, service_dependencies: tuple[MagicMock, MagicMock], *, tool_providers ) -> None: _, workflow_runs = service_dependencies - service = _service(service_dependencies) + service = _service(service_dependencies, tool_providers=tool_providers) expected = MagicMock(name="pagination") workflow_runs.get_paginated_workflow_runs.return_value = expected args = {"limit": "7", "last_id": "last-1", "status": "succeeded"} @@ -84,12 +82,10 @@ def test_get_paginate_workflow_runs_should_forward_filters_and_parse_limit( ) def test_get_paginate_advanced_chat_workflow_runs_should_attach_message_fields_when_message_exists( - self, - service_dependencies: tuple[MagicMock, MagicMock], - monkeypatch: pytest.MonkeyPatch, + self, service_dependencies: tuple[MagicMock, MagicMock], monkeypatch: pytest.MonkeyPatch, *, tool_providers ) -> None: _, workflow_runs = service_dependencies - service = _service(service_dependencies) + service = _service(service_dependencies, tool_providers=tool_providers) run_with_message = _workflow_run(status=WorkflowExecutionStatus.RUNNING) run_without_message = _workflow_run(run_id="run-2") pagination = SimpleNamespace(data=[run_with_message, run_without_message]) @@ -117,11 +113,10 @@ def test_get_paginate_advanced_chat_workflow_runs_should_attach_message_fields_w ) def test_get_workflow_run_should_delegate_to_repository_by_tenant_and_app( - self, - service_dependencies: tuple[MagicMock, MagicMock], + self, service_dependencies: tuple[MagicMock, MagicMock], *, tool_providers ) -> None: _, workflow_runs = service_dependencies - service = _service(service_dependencies) + service = _service(service_dependencies, tool_providers=tool_providers) expected = _workflow_run() workflow_runs.get_workflow_run_by_id.return_value = expected @@ -135,11 +130,10 @@ def test_get_workflow_run_should_delegate_to_repository_by_tenant_and_app( ) def test_get_workflow_runs_count_should_forward_optional_filters( - self, - service_dependencies: tuple[MagicMock, MagicMock], + self, service_dependencies: tuple[MagicMock, MagicMock], *, tool_providers ) -> None: _, workflow_runs = service_dependencies - service = _service(service_dependencies) + service = _service(service_dependencies, tool_providers=tool_providers) expected = {"total": 3, "succeeded": 2} workflow_runs.get_workflow_runs_count.return_value = expected @@ -161,11 +155,9 @@ def test_get_workflow_runs_count_should_forward_optional_filters( ) def test_get_workflow_run_node_executions_should_return_empty_list_when_run_not_found( - self, - service_dependencies: tuple[MagicMock, MagicMock], - monkeypatch: pytest.MonkeyPatch, + self, service_dependencies: tuple[MagicMock, MagicMock], monkeypatch: pytest.MonkeyPatch, *, tool_providers ) -> None: - service = _service(service_dependencies) + service = _service(service_dependencies, tool_providers=tool_providers) monkeypatch.setattr(service, "get_workflow_run", MagicMock(return_value=None)) result = service.get_workflow_run_node_executions( @@ -177,12 +169,10 @@ def test_get_workflow_run_node_executions_should_return_empty_list_when_run_not_ assert result == [] def test_get_workflow_run_node_executions_should_use_request_workspace( - self, - service_dependencies: tuple[MagicMock, MagicMock], - monkeypatch: pytest.MonkeyPatch, + self, service_dependencies: tuple[MagicMock, MagicMock], monkeypatch: pytest.MonkeyPatch, *, tool_providers ) -> None: node_executions, _ = service_dependencies - service = _service(service_dependencies) + service = _service(service_dependencies, tool_providers=tool_providers) monkeypatch.setattr(service, "get_workflow_run", MagicMock(return_value=_workflow_run())) expected_executions = [SimpleNamespace(id="exec-1"), SimpleNamespace(id="exec-2")] expected_traces = [SimpleNamespace(id="exec-1:retry:1"), SimpleNamespace(id="exec-1")] @@ -202,4 +192,6 @@ def test_get_workflow_run_node_executions_should_use_request_workspace( app_id="app-1", workflow_run_id="run-1", ) - mock_assemble.assert_called_once_with(expected_executions, node_executions, session=ANY) + mock_assemble.assert_called_once_with( + expected_executions, node_executions, session=ANY, tool_providers=tool_providers + ) diff --git a/api/tests/unit_tests/services/test_workflow_run_service_pause.py b/api/tests/unit_tests/services/test_workflow_run_service_pause.py index 3552cbd7c9df7f..236f3451de3d20 100644 --- a/api/tests/unit_tests/services/test_workflow_run_service_pause.py +++ b/api/tests/unit_tests/services/test_workflow_run_service_pause.py @@ -22,11 +22,8 @@ def workflow_runs() -> MagicMock: return MagicMock() -def _service(workflow_runs: MagicMock) -> WorkflowRunService: - return WorkflowRunService( - workflow_runs=workflow_runs, - node_executions=MagicMock(), - ) +def _service(workflow_runs: MagicMock, *, tool_providers) -> WorkflowRunService: + return WorkflowRunService(workflow_runs=workflow_runs, node_executions=MagicMock(), tool_providers=tool_providers) def _request_context(*, workspace_id: str = "tenant-1") -> RequestContext: @@ -38,10 +35,12 @@ def _request_context(*, workspace_id: str = "tenant-1") -> RequestContext: ) -def test_get_pause_details_returns_none_when_run_is_not_found(workflow_runs: MagicMock) -> None: +def test_get_pause_details_returns_none_when_run_is_not_found(workflow_runs: MagicMock, *, tool_providers) -> None: workflow_runs.get_pause_record.return_value = None - result = _service(workflow_runs).get_pause_details(_request_context(), workflow_run_id="run-1") + result = _service(workflow_runs, tool_providers=tool_providers).get_pause_details( + _request_context(), workflow_run_id="run-1" + ) assert result is None workflow_runs.get_pause_record.assert_called_once_with( @@ -50,7 +49,9 @@ def test_get_pause_details_returns_none_when_run_is_not_found(workflow_runs: Mag ) -def test_get_pause_details_returns_empty_details_for_non_paused_run(workflow_runs: MagicMock) -> None: +def test_get_pause_details_returns_empty_details_for_non_paused_run( + workflow_runs: MagicMock, *, tool_providers +) -> None: workflow_runs.get_pause_record.return_value = WorkflowRunPauseRecord( status=WorkflowExecutionStatus.SUCCEEDED, paused_at=None, @@ -58,12 +59,14 @@ def test_get_pause_details_returns_empty_details_for_non_paused_run(workflow_run form_tokens={}, ) - result = _service(workflow_runs).get_pause_details(_request_context(), workflow_run_id="run-1") + result = _service(workflow_runs, tool_providers=tool_providers).get_pause_details( + _request_context(), workflow_run_id="run-1" + ) assert result == WorkflowRunPauseDetails(paused_at=None, paused_nodes=()) -def test_get_pause_details_maps_human_input_and_token(workflow_runs: MagicMock) -> None: +def test_get_pause_details_maps_human_input_and_token(workflow_runs: MagicMock, *, tool_providers) -> None: reason = HumanInputRequired( form_id="form-1", form_content="Approve?", @@ -78,7 +81,7 @@ def test_get_pause_details_maps_human_input_and_token(workflow_runs: MagicMock) form_tokens={"form-1": "form-token"}, ) - result = _service(workflow_runs).get_pause_details( + result = _service(workflow_runs, tool_providers=tool_providers).get_pause_details( _request_context(workspace_id="tenant-context"), workflow_run_id="run-1", ) @@ -100,7 +103,7 @@ def test_get_pause_details_maps_human_input_and_token(workflow_runs: MagicMock) ) -def test_get_pause_details_rejects_unsupported_pause_reason(workflow_runs: MagicMock) -> None: +def test_get_pause_details_rejects_unsupported_pause_reason(workflow_runs: MagicMock, *, tool_providers) -> None: workflow_runs.get_pause_record.return_value = WorkflowRunPauseRecord( status=WorkflowExecutionStatus.PAUSED, paused_at=None, @@ -109,4 +112,6 @@ def test_get_pause_details_rejects_unsupported_pause_reason(workflow_runs: Magic ) with pytest.raises(NotImplementedError, match="Pause details do not support SchedulingPause"): - _service(workflow_runs).get_pause_details(_request_context(), workflow_run_id="run-1") + _service(workflow_runs, tool_providers=tool_providers).get_pause_details( + _request_context(), workflow_run_id="run-1" + ) diff --git a/api/tests/unit_tests/services/test_workflow_service.py b/api/tests/unit_tests/services/test_workflow_service.py index 4ac18277a5f8d1..cfe23b12c736ef 100644 --- a/api/tests/unit_tests/services/test_workflow_service.py +++ b/api/tests/unit_tests/services/test_workflow_service.py @@ -12,20 +12,16 @@ import json import uuid from collections.abc import Callable -from datetime import datetime, timedelta -from types import SimpleNamespace from typing import Any, cast -from unittest.mock import ANY, MagicMock, patch, sentinel +from unittest.mock import ANY, MagicMock, patch import pytest -from sqlalchemy import event, select from sqlalchemy.dialects import postgresql from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, sessionmaker from core.workflow.llm_environment_variable import LLMEnvironmentVariable from enums import DeploymentEdition -from enums.human_input import RecipientType from graphon.enums import ( BuiltinNodeTypes, ErrorStrategy, @@ -37,29 +33,14 @@ from graphon.model_runtime.entities.model_entities import ModelType from graphon.node_events import NodeRunResult from graphon.nodes.http_request import HTTP_REQUEST_CONFIG_FILTER_KEY, HttpRequestNode, HttpRequestNodeConfig -from graphon.variables import StringVariable from graphon.variables.input_entities import VariableEntityType from libs.datetime_utils import naive_utc_now from models.account import Account -from models.agent import ( - Agent, - AgentConfigSnapshot, - AgentKind, - AgentScope, - AgentSource, - AgentStatus, - WorkflowAgentBindingType, - WorkflowAgentNodeBinding, -) -from models.agent_config_entities import AgentSoulConfig -from models.human_input import HumanInputFormRecipient from models.model import App, AppMode -from models.tools import BuiltinToolProvider, WorkflowToolProvider +from models.tools import BuiltinToolProvider from models.workflow import Workflow, WorkflowType -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService -from services.errors.app import IsDraftWorkflowError, TriggerNodeLimitExceededError, WorkflowHashNotEqualError -from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError -from services.workflow_ref_service import WorkflowRef +from repositories.credentials.query_repository import CredentialQueryRepository +from services.errors.app import IsDraftWorkflowError from services.workflow_service import ( WorkflowService, _rebuild_file_for_user_inputs_in_start_node, @@ -237,52 +218,9 @@ def workflow_service(self, sqlite_engine: Engine) -> WorkflowService: """Create a WorkflowService whose repositories use the test SQLite engine.""" return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) - def test_get_tenant_app_maintainers_scopes_requested_apps( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - sqlite_session.add_all( - [ - TestWorkflowAssociatedDataFactory.create_app( - app_id="app-1", tenant_id="tenant-1", maintainer="owner-1" - ), - TestWorkflowAssociatedDataFactory.create_app(app_id="app-2", tenant_id="tenant-1"), - TestWorkflowAssociatedDataFactory.create_app( - app_id="app-3", tenant_id="tenant-2", maintainer="owner-2" - ), - ] - ) - sqlite_session.commit() - - assert workflow_service.get_tenant_app_maintainers( - ["app-1", "app-2", "app-3", "missing"], "tenant-1", session=sqlite_session - ) == {"app-1": "owner-1", "app-2": None} - # ==================== Workflow Existence Tests ==================== # These tests verify the service can check if a draft workflow exists - def test_is_workflow_exist_returns_true(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test is_workflow_exist returns True when draft workflow exists. - - Verifies that the service correctly identifies when an app has a draft workflow. - This is used to determine whether to create or update a workflow. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - sqlite_session.add(TestWorkflowAssociatedDataFactory.create_workflow()) - sqlite_session.commit() - - result = workflow_service.is_workflow_exist(app, session=sqlite_session) - - assert result is True - - def test_is_workflow_exist_returns_false(self, workflow_service: WorkflowService, sqlite_session: Session): - """Test is_workflow_exist returns False when no draft workflow exists.""" - app = TestWorkflowAssociatedDataFactory.create_app() - - result = workflow_service.is_workflow_exist(app, session=sqlite_session) - - assert result is False - # ==================== Get Draft Workflow Tests ==================== # These tests verify retrieval of draft workflows (version="draft") @@ -336,18 +274,14 @@ def test_get_draft_workflow_with_workflow_id(self, workflow_service: WorkflowSer def test_get_draft_workflow_with_workflow_id_reuses_provided_session( self, workflow_service: WorkflowService, sqlite_session: Session ): - """Test get_draft_workflow passes an injected session to published workflow lookup.""" + """A version lookup sees the caller's uncommitted rows through the shared repository.""" app = TestWorkflowAssociatedDataFactory.create_app() workflow_id = "workflow-123" - mock_workflow = TestWorkflowAssociatedDataFactory.create_workflow(version="v1") - - with patch.object( - workflow_service, "get_published_workflow_by_id", return_value=mock_workflow - ) as mock_get_published: - result = workflow_service.get_draft_workflow(app, workflow_id=workflow_id, session=sqlite_session) - - assert result == mock_workflow - mock_get_published.assert_called_once_with(app, workflow_id, session=sqlite_session) + workflow = TestWorkflowAssociatedDataFactory.create_workflow( + workflow_id=workflow_id, app_id=app.id, tenant_id=app.tenant_id, version="v1" + ) + sqlite_session.add(workflow) + assert workflow_service.get_draft_workflow(app, workflow_id=workflow_id, session=sqlite_session) is workflow # ==================== Get Published Workflow Tests ==================== # These tests verify retrieval of published workflows (versioned snapshots) @@ -460,531 +394,6 @@ def test_get_published_workflow_returns_none_when_no_workflow_id( assert result is None - # ==================== Sync Draft Workflow Tests ==================== - # These tests verify creating and updating draft workflows with validation - - def test_sync_draft_workflow_creates_new_draft(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test sync_draft_workflow creates new draft workflow when none exists. - - When a user first creates a workflow app, this creates the initial draft. - The draft is validated before creation to ensure graph and features are valid. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - features = {"file_upload": {"enabled": False}} - - with ( - patch("services.workflow_service.app_draft_workflow_was_synced"), - patch( - "services.agent.legacy_workflow_publish_service.WorkflowAgentPublishService.sync_agent_bindings_for_draft", - return_value={"retired-agent"}, - ), - patch("services.workflow_service.WorkflowAgentRetirementService.retire_unowned") as retire_unowned, - ): - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=graph, - features=features, - unique_hash=None, - account=account, - environment_variables=[], - conversation_variables=[], - session=sqlite_session, - ) - - persisted_workflow = sqlite_session.scalar(select(Workflow).where(Workflow.id == result.id)) - assert persisted_workflow is result - assert result.graph_dict == graph - assert result.features_dict == features - retire_unowned.assert_called_once_with( - tenant_id=app.tenant_id, - agent_ids={"retired-agent"}, - account_id=account.id, - ) - - def test_sync_draft_workflow_updates_existing_draft( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test sync_draft_workflow updates existing draft workflow. - - When users edit their workflow, this updates the existing draft. - The unique_hash is used for optimistic locking to prevent conflicts. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - features = {"file_upload": {"enabled": False}} - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - sqlite_session.add(workflow) - sqlite_session.commit() - unique_hash = workflow.unique_hash - - with patch("services.workflow_service.app_draft_workflow_was_synced"): - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=graph, - features=features, - unique_hash=unique_hash, - account=account, - environment_variables=[], - conversation_variables=[], - session=sqlite_session, - ) - - sqlite_session.refresh(workflow) - assert result is workflow - assert workflow.graph_dict == graph - assert workflow.features_dict == features - assert workflow.updated_by == account.id - - def test_sync_draft_workflow_collaborative_save_preserves_environment_variables_and_locks_row( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """A collaborative graph-only save locks the draft and keeps server environment values.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - remote_variable = StringVariable( - id="env-shared", - name="shared", - value="remote-value", - selector=["env", "shared"], - ) - stale_variable = remote_variable.model_copy(update={"value": "stale-client-value"}) - workflow.environment_variables = [remote_variable] - sqlite_session.add(workflow) - sqlite_session.commit() - unique_hash = workflow.unique_hash - statements = [] - commits = [] - - @event.listens_for(sqlite_session, "do_orm_execute") - def capture_statement(execute_state): - statements.append(execute_state.statement) - - @event.listens_for(sqlite_session, "before_commit") - def capture_commit(_session): - commits.append(True) - - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=TestWorkflowAssociatedDataFactory.create_valid_workflow_graph(), - features={}, - unique_hash=unique_hash, - account=account, - environment_variables=[stale_variable], - conversation_variables=[], - session=sqlite_session, - preserve_environment_variables=True, - commit=False, - sync_agent_bindings=False, - ) - - assert "FOR UPDATE" in str(statements[0].compile(dialect=postgresql.dialect())) - assert result.environment_variables == [remote_variable] - assert commits == [] - - def test_sync_draft_workflow_merges_environment_patch_with_graph_update( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """Graph changes and a per-ID environment patch commit without replacing untouched aliases.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - remote_variable = StringVariable(id="env-a", name="a", value="remote-a", selector=["env", "a"]) - replaced_variable = StringVariable(id="env-b", name="b", value="old-b", selector=["env", "b"]) - workflow.environment_variables = [remote_variable, replaced_variable] - sqlite_session.add(workflow) - sqlite_session.commit() - unique_hash = workflow.unique_hash - next_graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - next_graph["viewport"] = {"x": 10, "y": 20, "zoom": 1} - - with patch("services.workflow_service.app_draft_workflow_was_synced"): - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=next_graph, - features={}, - unique_hash=unique_hash, - account=account, - environment_variables=[ - remote_variable.model_copy(update={"value": "stale-a"}), - replaced_variable, - ], - conversation_variables=[], - session=sqlite_session, - environment_variable_upserts=[replaced_variable.model_copy(update={"value": "new-b"})], - deleted_environment_variable_ids=[], - preserve_environment_variables=True, - ) - - assert result.graph_dict == next_graph - assert [(variable.id, variable.value) for variable in result.environment_variables] == [ - ("env-a", "remote-a"), - ("env-b", "new-b"), - ] - - def test_sync_draft_workflow_noncollaborative_save_replaces_environment_variables( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """Legacy full-draft saves retain their full environment replacement contract.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - workflow.environment_variables = [ - StringVariable(id="env-old", name="old", value="old", selector=["env", "old"]) - ] - sqlite_session.add(workflow) - sqlite_session.commit() - replacement = StringVariable(id="env-new", name="new", value="new", selector=["env", "new"]) - - with patch("services.workflow_service.app_draft_workflow_was_synced"): - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=TestWorkflowAssociatedDataFactory.create_valid_workflow_graph(), - features={}, - unique_hash=workflow.unique_hash, - account=account, - environment_variables=[replacement], - conversation_variables=[], - session=sqlite_session, - ) - - assert result.environment_variables == [replacement] - - def test_sync_draft_workflow_graph_only_preserves_independently_updated_draft_fields( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - existing_features = {"file_upload": {"enabled": True}} - existing_environment_variables = [ - StringVariable(id="env-new", name="region", value="new", selector=["env", "region"]) - ] - existing_conversation_variables = [ - StringVariable( - id="conversation-new", - name="topic", - value="new", - selector=["conversation", "topic"], - ) - ] - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - features=existing_features, - environment_variables=existing_environment_variables, - conversation_variables=existing_conversation_variables, - ) - sqlite_session.add(workflow) - sqlite_session.commit() - unique_hash = workflow.unique_hash - updated_graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - updated_graph["nodes"][0]["data"]["title"] = "Updated graph" - - with patch("services.workflow_service.app_draft_workflow_was_synced"): - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=updated_graph, - features={"file_upload": {"enabled": False}}, - unique_hash=unique_hash, - account=account, - environment_variables=[ - StringVariable(id="env-old", name="region", value="old", selector=["env", "region"]) - ], - conversation_variables=[ - StringVariable( - id="conversation-old", - name="topic", - value="old", - selector=["conversation", "topic"], - ) - ], - session=sqlite_session, - graph_only=True, - ) - - sqlite_session.refresh(workflow) - assert result is workflow - assert workflow.graph_dict == updated_graph - assert workflow.features_dict == existing_features - assert workflow.environment_variables == existing_environment_variables - assert workflow.conversation_variables == existing_conversation_variables - - def test_sync_draft_workflow_graph_only_creates_complete_initial_draft( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - features = {"file_upload": {"enabled": True}} - environment_variables = [StringVariable(id="env-id", name="region", value="new", selector=["env", "region"])] - conversation_variables = [ - StringVariable( - id="conversation-id", - name="topic", - value="new", - selector=["conversation", "topic"], - ) - ] - - with patch("services.workflow_service.app_draft_workflow_was_synced"): - workflow = workflow_service.sync_draft_workflow( - app_model=app, - graph=graph, - features=features, - unique_hash=None, - account=account, - environment_variables=environment_variables, - conversation_variables=conversation_variables, - session=sqlite_session, - graph_only=True, - ) - - assert workflow.graph_dict == graph - assert workflow.features_dict == features - assert workflow.environment_variables == environment_variables - assert workflow.conversation_variables == conversation_variables - - def test_sync_draft_workflow_raises_hash_not_equal_error( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test sync_draft_workflow raises error when hash doesn't match. - - This implements optimistic locking: if the workflow was modified by another - user/session since it was loaded, the hash won't match and the update fails. - This prevents overwriting concurrent changes. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - features = {} - - sqlite_session.add(TestWorkflowAssociatedDataFactory.create_workflow()) - sqlite_session.commit() - - with pytest.raises(WorkflowHashNotEqualError): - workflow_service.sync_draft_workflow( - app_model=app, - graph=graph, - features=features, - unique_hash="new-hash", - account=account, - environment_variables=[], - conversation_variables=[], - session=sqlite_session, - ) - - def test_restore_published_workflow_to_draft_keeps_source_features_unmodified( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - legacy_features = { - "file_upload": { - "image": { - "enabled": True, - "number_limits": 6, - "transfer_methods": ["remote_url", "local_file"], - } - }, - "opening_statement": "", - "retriever_resource": {"enabled": True}, - "sensitive_word_avoidance": {"enabled": False}, - "speech_to_text": {"enabled": False}, - "suggested_questions": [], - "suggested_questions_after_answer": {"enabled": False}, - "text_to_speech": {"enabled": False, "language": "", "voice": ""}, - } - source_workflow = Workflow( - id="published-workflow-id", - tenant_id=app.tenant_id, - app_id=app.id, - type=WorkflowType.WORKFLOW, - version="2026-03-19T00:00:00", - graph=json.dumps(TestWorkflowAssociatedDataFactory.create_valid_workflow_graph()), - features=json.dumps(legacy_features), - created_by=account.id, - environment_variables=[], - conversation_variables=[], - rag_pipeline_variables=[], - ) - draft_workflow = Workflow( - id="draft-workflow-id", - tenant_id=app.tenant_id, - app_id=app.id, - type=WorkflowType.WORKFLOW, - version=Workflow.VERSION_DRAFT, - graph=json.dumps({"nodes": [], "edges": []}), - features=json.dumps({}), - created_by=account.id, - environment_variables=[], - conversation_variables=[], - rag_pipeline_variables=[], - ) - sqlite_session.add_all([source_workflow, draft_workflow]) - sqlite_session.commit() - - with ( - patch("services.workflow_service.app_draft_workflow_was_synced"), - patch.object( - workflow_service, - "get_published_workflow_by_id", - wraps=workflow_service.get_published_workflow_by_id, - ) as get_published_workflow_by_id, - patch( - "services.agent.legacy_workflow_publish_service.WorkflowAgentPublishService.restore_agent_node_bindings_to_draft", - return_value={"retired-agent"}, - ), - patch("services.workflow_service.WorkflowAgentRetirementService.retire_unowned") as retire_unowned, - ): - result = workflow_service.restore_published_workflow_to_draft( - app_model=app, - workflow_id=source_workflow.id, - account=account, - session=sqlite_session, - ) - - assert result is draft_workflow - assert source_workflow.serialized_features == json.dumps(legacy_features) - assert draft_workflow.serialized_features == json.dumps(legacy_features) - sqlite_session.refresh(draft_workflow) - assert draft_workflow.serialized_features == json.dumps(legacy_features) - get_published_workflow_by_id.assert_called_once_with( - app_model=app, - workflow_id=source_workflow.id, - session=sqlite_session, - for_update=True, - ) - retire_unowned.assert_called_once_with( - tenant_id=app.tenant_id, - agent_ids={"retired-agent"}, - account_id=account.id, - ) - - def test_restore_historical_inline_agent_after_current_pointer_moves_uses_real_clone( - self, - workflow_service: WorkflowService, - sqlite_session: Session, - ) -> None: - app = TestWorkflowAssociatedDataFactory.create_app(workflow_id=None) - account = TestWorkflowAssociatedDataFactory.create_account() - graph = { - "nodes": [ - { - "id": "agent-node", - "data": { - "type": "agent", - "version": "2", - "agent_node_kind": "dify_agent", - }, - } - ], - "edges": [], - } - historical = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="historical-workflow", - version="historical-version", - graph=graph, - ) - current = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="current-workflow", - version="current-version", - graph=graph, - ) - draft = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="draft-workflow", - version=Workflow.VERSION_DRAFT, - ) - source_agent = Agent( - id="historical-agent", - tenant_id=app.tenant_id, - name="Historical inline Agent", - description="", - role="", - agent_kind=AgentKind.DIFY_AGENT, - scope=AgentScope.WORKFLOW_ONLY, - source=AgentSource.WORKFLOW, - app_id=app.id, - workflow_id=historical.id, - workflow_node_id="agent-node", - active_config_snapshot_id="historical-snapshot", - active_config_has_model=False, - active_config_is_published=True, - status=AgentStatus.ACTIVE, - created_by=account.id, - updated_by=account.id, - ) - source_snapshot = AgentConfigSnapshot( - id="historical-snapshot", - tenant_id=app.tenant_id, - agent_id=source_agent.id, - version=1, - config_snapshot=AgentSoulConfig(config_note="historical soul"), - created_by=account.id, - ) - historical_binding = WorkflowAgentNodeBinding( - id="historical-binding", - tenant_id=app.tenant_id, - app_id=app.id, - workflow_id=historical.id, - workflow_version=historical.version, - node_id="agent-node", - binding_type=WorkflowAgentBindingType.INLINE_AGENT, - agent_id=source_agent.id, - current_snapshot_id=source_snapshot.id, - node_job_config={}, - created_by=account.id, - ) - sqlite_session.add_all([app, historical, current, draft, source_agent, source_snapshot, historical_binding]) - app.workflow_id = historical.id - sqlite_session.commit() - - app.workflow_id = current.id - sqlite_session.commit() - - WorkflowAgentRetirementService.retire_unowned( - tenant_id=app.tenant_id, - agent_ids=[source_agent.id], - account_id=account.id, - ) - sqlite_session.expire_all() - retained_agent = sqlite_session.get(Agent, source_agent.id) - assert retained_agent is not None - assert retained_agent.status is AgentStatus.ACTIVE - - with patch("services.workflow_service.app_draft_workflow_was_synced"): - restored_draft = workflow_service.restore_published_workflow_to_draft( - app_model=app, - workflow_id=historical.id, - account=account, - session=sqlite_session, - ) - - restored_binding = sqlite_session.scalar( - select(WorkflowAgentNodeBinding).where( - WorkflowAgentNodeBinding.workflow_id == draft.id, - WorkflowAgentNodeBinding.workflow_version == Workflow.VERSION_DRAFT, - WorkflowAgentNodeBinding.node_id == "agent-node", - ) - ) - assert restored_draft is draft - assert app.workflow_id == current.id - assert sqlite_session.get(Workflow, historical.id) is historical - assert sqlite_session.get(WorkflowAgentNodeBinding, historical_binding.id) is historical_binding - assert restored_binding is not None - assert restored_binding.agent_id not in (None, source_agent.id) - assert restored_binding.current_snapshot_id not in (None, source_snapshot.id) - restored_agent = sqlite_session.get(Agent, restored_binding.agent_id) - restored_snapshot = sqlite_session.get(AgentConfigSnapshot, restored_binding.current_snapshot_id) - assert restored_agent is not None - assert restored_agent.workflow_id == draft.id - assert restored_agent.workflow_node_id == "agent-node" - assert restored_snapshot is not None - assert restored_snapshot.config_snapshot_dict == source_snapshot.config_snapshot_dict - # ==================== Workflow Validation Tests ==================== # These tests verify graph structure and feature configuration validation @@ -1034,865 +443,52 @@ def test_validate_graph_structure_start_and_trigger_coexist_raises_error(self, w "edges": [], } - with pytest.raises(ValueError, match="Start node and trigger nodes cannot coexist"): - workflow_service.validate_graph_structure(graph) - - def test_validate_features_structure_workflow_mode(self, workflow_service: WorkflowService): - """ - Test validate_features_structure for workflow mode. - - Different app modes have different feature configurations. - This ensures the features match the expected schema for workflow apps. - """ - app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.WORKFLOW) - features = {"file_upload": {"enabled": False}} - - with patch("services.workflow_service.WorkflowAppConfigManager.config_validate") as mock_validate: - workflow_service.validate_features_structure(app, features) - mock_validate.assert_called_once_with( - tenant_id=app.tenant_id, config=features, only_structure_validate=True - ) - - def test_validate_features_structure_advanced_chat_mode(self, workflow_service: WorkflowService): - """Test validate_features_structure for advanced chat mode.""" - app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.ADVANCED_CHAT) - features = {"opening_statement": "Hello"} - - with patch("services.workflow_service.AdvancedChatAppConfigManager.config_validate") as mock_validate: - workflow_service.validate_features_structure(app, features) - mock_validate.assert_called_once_with( - tenant_id=app.tenant_id, config=features, only_structure_validate=True - ) - - def test_validate_features_structure_invalid_mode_raises_error(self, workflow_service: WorkflowService): - """Test validate_features_structure raises error for invalid mode.""" - app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.COMPLETION) - features = {} - - with pytest.raises(ValueError, match="Invalid app mode"): - workflow_service.validate_features_structure(app, features) - - # ==================== Draft Workflow Variable Update Tests ==================== - # These tests verify updating draft workflow environment/conversation variables - - def test_update_draft_workflow_environment_variables_updates_workflow( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test update_draft_workflow_environment_variables updates draft fields.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - sqlite_session.add(workflow) - sqlite_session.commit() - variables = [StringVariable(id="env-id", name="region", value="us-east-1", selector=["env", "region"])] - - workflow_service.update_draft_workflow_environment_variables( - app_model=app, - environment_variables=variables, - account=account, - session=sqlite_session, - ) - - assert workflow.environment_variables == variables - assert workflow.updated_by == account.id - sqlite_session.expire_all() - persisted_workflow = sqlite_session.get(Workflow, workflow.id) - assert persisted_workflow is not None - assert persisted_workflow.environment_variables == variables - - def test_update_draft_workflow_environment_variables_raises_when_missing( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test update_draft_workflow_environment_variables raises when draft missing.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - - with pytest.raises(ValueError, match="No draft workflow found."): - workflow_service.update_draft_workflow_environment_variables( - app_model=app, - environment_variables=[], - account=account, - session=sqlite_session, - ) - - def test_patch_draft_workflow_environment_variables_merges_by_id( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """A patch preserves untouched variables while applying ordered upserts and deletions.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - workflow.environment_variables = [ - StringVariable(id="env-a", name="a", value="remote-a", selector=["env", "a"]), - StringVariable(id="env-b", name="b", value="old-b", selector=["env", "b"]), - StringVariable(id="env-c", name="c", value="remote-c", selector=["env", "c"]), - ] - sqlite_session.add(workflow) - sqlite_session.commit() - - workflow_service.patch_draft_workflow_environment_variables( - app_model=app, - environment_variables=[ - StringVariable(id="env-b", name="b", value="new-b", selector=["env", "b"]), - StringVariable(id="env-d", name="d", value="new-d", selector=["env", "d"]), - ], - deleted_environment_variable_ids=["env-a"], - account=account, - session=sqlite_session, - ) - - sqlite_session.expire_all() - persisted_workflow = sqlite_session.get(Workflow, workflow.id) - assert persisted_workflow is not None - assert [(variable.id, variable.value) for variable in persisted_workflow.environment_variables] == [ - ("env-b", "new-b"), - ("env-c", "remote-c"), - ("env-d", "new-d"), - ] - assert persisted_workflow.updated_by == account.id - - def test_patch_draft_workflow_environment_variables_locks_draft_row( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """The merge reads the draft with a row lock before applying a partial update.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - sqlite_session.add(workflow) - sqlite_session.commit() - statements = [] - - @event.listens_for(sqlite_session, "do_orm_execute") - def capture_statement(execute_state): - statements.append(execute_state.statement) - - workflow_service.patch_draft_workflow_environment_variables( - app_model=app, - environment_variables=[], - deleted_environment_variable_ids=[], - account=account, - session=sqlite_session, - ) - - compiled_statement = str(statements[0].compile(dialect=postgresql.dialect())) - assert "FOR UPDATE" in compiled_statement - assert sqlite_session.get(Workflow, workflow.id) is workflow - - def test_patch_draft_workflow_environment_variables_rejects_conflicting_ids( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """A patch cannot both upsert and delete the same variable ID.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - variable = StringVariable(id="env-a", name="a", value="a", selector=["env", "a"]) - sqlite_session.add(TestWorkflowAssociatedDataFactory.create_workflow()) - sqlite_session.commit() - - with pytest.raises(ValueError, match="cannot be upserted and deleted"): - workflow_service.patch_draft_workflow_environment_variables( - app_model=app, - environment_variables=[variable], - deleted_environment_variable_ids=["env-a"], - account=account, - session=sqlite_session, - ) - - def test_patch_draft_workflow_environment_variables_raises_when_missing( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """A patch fails when the app has no draft workflow to lock.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - - with pytest.raises(ValueError, match="No draft workflow found."): - workflow_service.patch_draft_workflow_environment_variables( - app_model=app, - environment_variables=[], - deleted_environment_variable_ids=[], - account=account, - session=sqlite_session, - ) - - def test_update_draft_workflow_conversation_variables_updates_workflow( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test update_draft_workflow_conversation_variables updates draft fields.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - sqlite_session.add(workflow) - sqlite_session.commit() - variables = [ - StringVariable(id="conversation-id", name="topic", value="sqlite", selector=["conversation", "topic"]) - ] - - workflow_service.update_draft_workflow_conversation_variables( - app_model=app, - conversation_variables=variables, - account=account, - session=sqlite_session, - ) - - assert workflow.conversation_variables == variables - assert workflow.updated_by == account.id - sqlite_session.expire_all() - persisted_workflow = sqlite_session.get(Workflow, workflow.id) - assert persisted_workflow is not None - assert persisted_workflow.conversation_variables == variables - - def test_update_draft_workflow_conversation_variables_raises_when_missing( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test update_draft_workflow_conversation_variables raises when draft missing.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - - with pytest.raises(ValueError, match="No draft workflow found."): - workflow_service.update_draft_workflow_conversation_variables( - app_model=app, - conversation_variables=[], - account=account, - session=sqlite_session, - ) - - # ==================== Publish Workflow Tests ==================== - # These tests verify creating published versions from draft workflows - - def test_publish_workflow_success(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test publish_workflow creates new published version. - - Publishing creates a timestamped snapshot of the draft workflow. - This allows users to: - - Roll back to previous versions - - Use stable versions in production - - Continue editing draft without affecting published version - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - - draft = TestWorkflowAssociatedDataFactory.create_workflow(version=Workflow.VERSION_DRAFT, graph=graph) - sqlite_session.add(draft) - sqlite_session.commit() - - with ( - patch("services.workflow_service.app_published_workflow_was_updated"), - patch( - "services.workflow_service.register_new_agent_beta_workflow_publish_after_commit" - ) as register_workflow_publish, - ): - result = workflow_service.publish_workflow( - session=sqlite_session, - app_model=app, - account=account, - marked_name="Version 1", - marked_comment="Initial release", - ) - - sqlite_session.flush() - assert result in sqlite_session - assert result.version != Workflow.VERSION_DRAFT - assert result.marked_name == "Version 1" - assert result.marked_comment == "Initial release" - register_workflow_publish.assert_not_called() - - def test_publish_workflow_registers_inline_agent_after_commit( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - draft = TestWorkflowAssociatedDataFactory.create_workflow( - version=Workflow.VERSION_DRAFT, - graph=TestWorkflowAssociatedDataFactory.create_valid_workflow_graph(), - ) - sqlite_session.add(draft) - sqlite_session.commit() - - with ( - patch("services.workflow_service.app_published_workflow_was_updated"), - patch( - "services.agent.legacy_workflow_publish_service.WorkflowAgentPublishService.copy_agent_node_bindings_to_published", - return_value=True, - ), - patch( - "services.workflow_service.register_new_agent_beta_workflow_publish_after_commit" - ) as register_workflow_publish, - ): - published = workflow_service.publish_workflow( - session=sqlite_session, - app_model=app, - account=account, - ) - - register_workflow_publish.assert_called_once_with( - session=sqlite_session, - published_workflow_id=published.id, - published_at=published.created_at, - ) - - def test_publish_workflow_numbers_versions_from_one( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test publish_workflow assigns an app-scoped version number starting at #1. - - The number is what users see when a version carries no name, so it has to be - stable and monotonic per app rather than derived from list position. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - - draft = TestWorkflowAssociatedDataFactory.create_workflow(version=Workflow.VERSION_DRAFT, graph=graph) - sqlite_session.add(draft) - sqlite_session.commit() - - with ( - patch("services.workflow_service.app_published_workflow_was_updated"), - ): - first = workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - second = workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - - assert first.version_number == 1 - assert second.version_number == 2 - - def test_publish_workflow_does_not_reuse_a_deleted_version_number( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test version numbers are never handed out twice, even after a version is deleted. - - Deployment records and audit logs refer to versions by number, so reusing one - would make two different workflows share an identity. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - - draft = TestWorkflowAssociatedDataFactory.create_workflow(version=Workflow.VERSION_DRAFT, graph=graph) - sqlite_session.add(draft) - sqlite_session.commit() - - with ( - patch("services.workflow_service.app_published_workflow_was_updated"), - ): - published = workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - sqlite_session.flush() - sqlite_session.delete(published) - sqlite_session.flush() - - republished = workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - - assert republished.version_number == 2 - - def test_publish_workflow_numbers_each_app_independently( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test the counter is scoped per app rather than global. - - Every app starts its own sequence at #1; a busy neighbour must not advance it. - """ - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - - published: list[Workflow] = [] - for app_id in ("app-first", "app-second"): - app = TestWorkflowAssociatedDataFactory.create_app(app_id=app_id) - draft = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=f"draft-{app_id}", - app_id=app_id, - version=Workflow.VERSION_DRAFT, - graph=graph, - ) - sqlite_session.add(draft) - sqlite_session.commit() - - with ( - patch("services.workflow_service.app_published_workflow_was_updated"), - ): - workflow = workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - published.append(workflow) - - assert [workflow.version_number for workflow in published] == [1, 1] - - def test_publish_workflow_no_draft_raises_error(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test publish_workflow raises error when no draft exists. - - Cannot publish if there's no draft to publish from. - Users must create and save a draft before publishing. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - - with pytest.raises(ValueError, match="No valid workflow found"): - workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - - @pytest.mark.parametrize( - ("environment_variables", "error"), - [ - ([], "was not found or is not an LLM variable"), - ( - [ - LLMEnvironmentVariable( - name="shared_model", - value={"provider": "provider", "name": "model", "mode": "completion"}, - ) - ], - "uses mode 'completion'.*uses mode 'chat'", - ), - ], - ) - def test_publish_workflow_rejects_invalid_llm_environment_reference_without_credential_validation( - self, - workflow_service: WorkflowService, - sqlite_session: Session, - environment_variables: list[LLMEnvironmentVariable], - error: str, - ) -> None: - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - llm_node = next(node for node in graph["nodes"] if node["data"]["type"] == BuiltinNodeTypes.LLM) - llm_node["data"]["model"] = { - "provider": "provider", - "name": "model", - "mode": "chat", - "completion_params": {}, - } - llm_node["data"]["model_selector"] = ["env", "shared_model"] - draft = TestWorkflowAssociatedDataFactory.create_workflow(graph=graph) - draft.environment_variables = environment_variables - sqlite_session.add(draft) - sqlite_session.commit() - - with ( - patch( - "services.system_feature_service.SystemFeatureService.is_plugin_manager_enabled", - return_value=False, - ), - pytest.raises(ValueError, match=error), - ): - workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - - def test_publish_workflow_trigger_limit_exceeded( - self, - workflow_service: WorkflowService, - sqlite_session: Session, - config_overrides: Callable[..., None], - ): - config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) - """ - Test publish_workflow raises error when trigger node limit exceeded in SANDBOX plan. - - Free/sandbox tier users have limits on the number of trigger nodes. - This prevents resource abuse while allowing users to test the feature. - The limit is enforced at publish time, not during draft editing. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - - # Create graph with 3 trigger nodes (exceeds SANDBOX limit of 2) - # Trigger nodes enable event-driven automation which consumes resources - graph = { - "nodes": [ - {"id": "trigger-1", "data": {"type": "trigger-webhook"}}, - {"id": "trigger-2", "data": {"type": "trigger-schedule"}}, - {"id": "trigger-3", "data": {"type": "trigger-plugin"}}, - ], - "edges": [], - } - draft = TestWorkflowAssociatedDataFactory.create_workflow(version=Workflow.VERSION_DRAFT, graph=graph) - sqlite_session.add(draft) - sqlite_session.commit() - - with ( - patch("services.workflow_service.BillingService") as MockBillingService, - ): - MockBillingService.get_info.return_value = {"subscription": {"plan": "sandbox"}} - - with pytest.raises(TriggerNodeLimitExceededError): - workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - - # ==================== Version Management Tests ==================== - # These tests verify listing and managing published workflow versions - - def test_get_all_published_workflow_with_pagination( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test get_all_published_workflow returns paginated results. - - Apps can have many published versions over time. - Pagination prevents loading all versions at once, improving performance. - """ - app = TestWorkflowAssociatedDataFactory.create_app(workflow_id="workflow-123") - - sqlite_session.add_all( - [ - TestWorkflowAssociatedDataFactory.create_workflow(workflow_id=f"workflow-{i}", version=f"v{i}") - for i in range(5) - ] - ) - sqlite_session.commit() - - workflows, has_more = workflow_service.get_all_published_workflow( - session=sqlite_session, app_model=app, page=1, limit=10, user_id=None - ) - - assert len(workflows) == 5 - assert has_more is False - - def test_get_all_published_workflow_lists_the_draft_first( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test the draft heads the version list no matter how old it is. - - A draft is created together with its app and its `created_at` is never refreshed, - so ordering purely by publish time would put it last — off the first page entirely - once the app has accumulated enough published versions. - """ - app = TestWorkflowAssociatedDataFactory.create_app(workflow_id="workflow-3") - app_created_at = datetime(2026, 1, 1) - - sqlite_session.add( - TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="workflow-draft", - version=Workflow.VERSION_DRAFT, - created_at=app_created_at, - ) - ) - sqlite_session.add_all( - [ - TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=f"workflow-{i}", - version=f"2026-02-0{i} 00:00:00", - created_at=app_created_at + timedelta(days=i), - ) - for i in range(1, 4) - ] - ) - sqlite_session.commit() - - workflows, _ = workflow_service.get_all_published_workflow( - session=sqlite_session, app_model=app, page=1, limit=2, user_id=None - ) - - assert [workflow.id for workflow in workflows] == ["workflow-draft", "workflow-3"] - - def test_get_all_published_workflow_has_more(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test get_all_published_workflow indicates has_more when results exceed limit. - - The has_more flag tells the UI whether to show a "Load More" button. - This is determined by fetching limit+1 records and checking if we got that many. - """ - app = TestWorkflowAssociatedDataFactory.create_app(workflow_id="workflow-123") - - sqlite_session.add_all( - [ - TestWorkflowAssociatedDataFactory.create_workflow(workflow_id=f"workflow-{i}", version=f"v{i}") - for i in range(11) - ] - ) - sqlite_session.commit() - - workflows, has_more = workflow_service.get_all_published_workflow( - session=sqlite_session, app_model=app, page=1, limit=10, user_id=None - ) - - assert len(workflows) == 10 - assert has_more is True - - def test_get_all_published_workflow_no_workflow_id( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test get_all_published_workflow returns empty when app has no workflow_id.""" - app = TestWorkflowAssociatedDataFactory.create_app(workflow_id=None) - - workflows, has_more = workflow_service.get_all_published_workflow( - session=sqlite_session, app_model=app, page=1, limit=10, user_id=None - ) - - assert workflows == [] - assert has_more is False - - # ==================== Update Workflow Tests ==================== - # These tests verify updating workflow metadata (name, comments, etc.) - - def test_update_workflow_success(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test update_workflow updates workflow attributes. - - Allows updating metadata like marked_name and marked_comment - without creating a new version. Only specific fields are allowed - to prevent accidental modification of workflow logic. - """ - workflow_id = "workflow-123" - tenant_id = "tenant-456" - app_id = "app-789" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id=app_id, workflow_id=workflow_id) - account_id = "user-123" - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, tenant_id=tenant_id, app_id=app_id - ) - sqlite_session.add(workflow) - sqlite_session.commit() - - result = workflow_service.update_workflow( - session=sqlite_session, - account_id=account_id, - data={"marked_name": "Updated Name", "marked_comment": "Updated Comment"}, - workflow_ref=workflow_ref, - ) - - sqlite_session.flush() - assert result is workflow - assert workflow.marked_name == "Updated Name" - assert workflow.marked_comment == "Updated Comment" - assert workflow.updated_by == account_id - - def test_update_workflow_not_found(self, workflow_service: WorkflowService, sqlite_session: Session): - """Test update_workflow returns None when workflow not found.""" - result = workflow_service.update_workflow( - session=sqlite_session, - account_id="user-123", - data={"marked_name": "Test"}, - workflow_ref=WorkflowRef(tenant_id="tenant-456", owner_id="app-789", workflow_id="nonexistent"), - ) - - assert result is None - - def test_update_workflow_with_ref_scopes_lookup_to_app( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test update_workflow includes the trusted app owner in the lookup.""" - workflow_id = "workflow-123" - tenant_id = "tenant-456" - app_id = "app-789" - account_id = "user-123" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id="other-app", workflow_id=workflow_id) - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, tenant_id=tenant_id, app_id=app_id - ) - sqlite_session.add(workflow) - sqlite_session.commit() - - result = workflow_service.update_workflow( - session=sqlite_session, - account_id=account_id, - data={"marked_name": "Updated Name"}, - workflow_ref=workflow_ref, - ) - - assert result is None - assert workflow.marked_name == "" - - # ==================== Delete Workflow Tests ==================== - # These tests verify workflow deletion with safety checks - - def test_delete_workflow_success(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test delete_workflow successfully deletes a published workflow. - - Users can delete old published versions they no longer need. - This helps manage storage and keeps the version list clean. - """ - workflow_id = "workflow-123" - tenant_id = "tenant-456" - app_id = "app-789" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id=app_id, workflow_id=workflow_id) - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, tenant_id=tenant_id, app_id=app_id, version="v1" - ) - inline_binding = WorkflowAgentNodeBinding( - id="inline-binding", - tenant_id=tenant_id, - app_id=app_id, - workflow_id=workflow_id, - workflow_version="v1", - node_id="inline-node", - binding_type=WorkflowAgentBindingType.INLINE_AGENT, - agent_id="inline-agent", - current_snapshot_id="snapshot-1", - node_job_config={}, - ) - roster_binding = WorkflowAgentNodeBinding( - id="roster-binding", - tenant_id=tenant_id, - app_id=app_id, - workflow_id=workflow_id, - workflow_version="v1", - node_id="roster-node", - binding_type=WorkflowAgentBindingType.ROSTER_AGENT, - agent_id="roster-agent", - current_snapshot_id="snapshot-2", - node_job_config={}, - ) - non_target_bindings = [ - WorkflowAgentNodeBinding( - id=f"non-target-{key}", - tenant_id="other-tenant" if key == "tenant" else tenant_id, - app_id="other-app" if key == "app" else app_id, - workflow_id="other-workflow" if key == "workflow" else workflow_id, - workflow_version="other-version" if key == "version" else workflow.version, - node_id=f"{key}-node", - binding_type=WorkflowAgentBindingType.INLINE_AGENT, - agent_id=f"{key}-inline-agent", - current_snapshot_id=f"{key}-snapshot", - node_job_config={}, - ) - for key in ("tenant", "app", "workflow", "version") - ] - sqlite_session.add_all([workflow, inline_binding, roster_binding, *non_target_bindings]) - sqlite_session.commit() - - result = workflow_service.delete_workflow(session=sqlite_session, workflow_ref=workflow_ref) - sqlite_session.flush() - - assert result == ["inline-agent"] - assert sqlite_session.get(Workflow, workflow_id) is None - assert sqlite_session.get(WorkflowAgentNodeBinding, inline_binding.id) is None - assert sqlite_session.get(WorkflowAgentNodeBinding, roster_binding.id) is None - for binding in non_target_bindings: - assert sqlite_session.get(WorkflowAgentNodeBinding, binding.id) is binding - - def test_delete_workflow_locks_source_until_caller_commits(self, workflow_service: WorkflowService): - workflow = TestWorkflowAssociatedDataFactory.create_workflow(version="v1") - workflow_ref = WorkflowRef( - tenant_id=workflow.tenant_id, - owner_id=workflow.app_id, - workflow_id=workflow.id, - ) - session = MagicMock(spec=Session) - session.scalar.side_effect = [workflow, None, None] - session.scalars.return_value.all.return_value = [] - - result = workflow_service.delete_workflow(session=session, workflow_ref=workflow_ref) - - stmt = session.scalar.call_args_list[0].args[0] - sql = str(stmt.compile(dialect=postgresql.dialect())) - assert result == [] - assert "FOR UPDATE" in sql - session.delete.assert_called_once_with(workflow) - - def test_delete_workflow_with_ref_scopes_lookup_to_app( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test delete_workflow includes the trusted app owner in the lookup.""" - workflow_id = "workflow-123" - tenant_id = "tenant-456" - app_id = "app-789" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id="other-app", workflow_id=workflow_id) - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, tenant_id=tenant_id, app_id=app_id, version="v1" - ) - sqlite_session.add(workflow) - sqlite_session.commit() - - with pytest.raises(ValueError, match="not found"): - workflow_service.delete_workflow(session=sqlite_session, workflow_ref=workflow_ref) - - assert sqlite_session.get(Workflow, workflow_id) is workflow - - def test_delete_workflow_draft_raises_error(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test delete_workflow raises error when trying to delete draft. - - Draft workflows cannot be deleted - they're the working copy. - Users can only delete published versions to clean up old snapshots. - """ - workflow_id = "workflow-123" - tenant_id = "tenant-456" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id="app-789", workflow_id=workflow_id) - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, - tenant_id=tenant_id, - app_id=workflow_ref.owner_id, - version=Workflow.VERSION_DRAFT, - ) - sqlite_session.add(workflow) - sqlite_session.commit() - - with pytest.raises(DraftWorkflowDeletionError, match="Cannot delete draft workflow"): - workflow_service.delete_workflow(session=sqlite_session, workflow_ref=workflow_ref) + with pytest.raises(ValueError, match="Start node and trigger nodes cannot coexist"): + workflow_service.validate_graph_structure(graph) - def test_delete_workflow_in_use_by_app_raises_error( - self, workflow_service: WorkflowService, sqlite_session: Session - ): + def test_validate_features_structure_workflow_mode(self, workflow_service: WorkflowService): """ - Test delete_workflow raises error when workflow is in use by app. + Test validate_features_structure for workflow mode. - Cannot delete a workflow version that's currently published/active. - This would break the app for users. Must publish a different version first. + Different app modes have different feature configurations. + This ensures the features match the expected schema for workflow apps. """ - workflow_id = "workflow-123" - tenant_id = "tenant-456" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id="app-789", workflow_id=workflow_id) - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, tenant_id=tenant_id, app_id=workflow_ref.owner_id, version="v1" - ) - app = App( - id="active-app", - tenant_id=tenant_id, - name="Active App", - description="", - mode=AppMode.WORKFLOW, - workflow_id=workflow_id, - enable_site=True, - enable_api=True, - max_active_requests=0, - ) - sqlite_session.add_all([workflow, app]) - sqlite_session.commit() + app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.WORKFLOW) + features = {"file_upload": {"enabled": False}} - with pytest.raises(WorkflowInUseError, match="currently in use by app"): - workflow_service.delete_workflow(session=sqlite_session, workflow_ref=workflow_ref) + with patch("services.workflow_service.WorkflowAppConfigManager.config_validate") as mock_validate: + workflow_service.validate_features_structure(app, features) + mock_validate.assert_called_once_with( + tenant_id=app.tenant_id, config=features, only_structure_validate=True + ) - def test_delete_workflow_published_as_tool_raises_error( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test delete_workflow raises error when workflow is published as tool. + def test_validate_features_structure_advanced_chat_mode(self, workflow_service: WorkflowService): + """Test validate_features_structure for advanced chat mode.""" + app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.ADVANCED_CHAT) + features = {"opening_statement": "Hello"} - Workflows can be published as reusable tools for other workflows. - Cannot delete a version that's being used as a tool, as this would - break other workflows that depend on it. - """ - workflow_id = "workflow-123" - tenant_id = "tenant-456" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id="app-789", workflow_id=workflow_id) - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, tenant_id=tenant_id, app_id=workflow_ref.owner_id, version="v1" - ) - tool_provider = WorkflowToolProvider( - name="workflow-tool", - label="Workflow Tool", - icon="icon.svg", - app_id=workflow.app_id, - version=workflow.version, - user_id="user-123", - tenant_id=workflow.tenant_id, - description="Test provider", - parameter_configuration="[]", - ) - sqlite_session.add_all([workflow, tool_provider]) - sqlite_session.commit() + with patch("services.workflow_service.AdvancedChatAppConfigManager.config_validate") as mock_validate: + workflow_service.validate_features_structure(app, features) + mock_validate.assert_called_once_with( + tenant_id=app.tenant_id, config=features, only_structure_validate=True + ) + + def test_validate_features_structure_invalid_mode_raises_error(self, workflow_service: WorkflowService): + """Test validate_features_structure raises error for invalid mode.""" + app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.COMPLETION) + features = {} + + with pytest.raises(ValueError, match="Invalid app mode"): + workflow_service.validate_features_structure(app, features) - with pytest.raises(WorkflowInUseError, match="published as a tool"): - workflow_service.delete_workflow(session=sqlite_session, workflow_ref=workflow_ref) + # ==================== Publish Workflow Tests ==================== + # These tests verify creating published versions from draft workflows - def test_delete_workflow_not_found_raises_error(self, workflow_service: WorkflowService, sqlite_session: Session): - """Test delete_workflow raises error when workflow not found.""" - workflow_id = "nonexistent" - tenant_id = "tenant-456" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id="app-789", workflow_id=workflow_id) + # ==================== Version Management Tests ==================== + # These tests verify listing and managing published workflow versions - with pytest.raises(ValueError, match="not found"): - workflow_service.delete_workflow(session=sqlite_session, workflow_ref=workflow_ref) + # ==================== Update Workflow Tests ==================== + # These tests verify updating workflow metadata (name, comments, etc.) # ==================== Get Default Block Config Tests ==================== # These tests verify retrieval of default node configurations @@ -2068,73 +664,6 @@ def test_get_default_block_config_http_request_malformed_config_raises_type_erro filters={HTTP_REQUEST_CONFIG_FILTER_KEY: "invalid"}, ) - # ==================== Workflow Conversion Tests ==================== - # These tests verify converting basic apps to workflow apps - - def test_convert_to_workflow_from_chat_app(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test convert_to_workflow converts chat app to workflow. - - Allows users to migrate from simple chat apps to advanced workflow apps. - The conversion creates equivalent workflow nodes from the chat configuration, - giving users more control and customization options. - """ - app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.CHAT) - account = TestWorkflowAssociatedDataFactory.create_account() - args = { - "name": "Converted Workflow", - "icon_type": "emoji", - "icon": "🤖", - "icon_background": "#FFEAD5", - } - - with patch("services.workflow_service.WorkflowConverter") as MockConverter: - mock_converter = MockConverter.return_value - mock_new_app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.WORKFLOW) - mock_converter.convert_to_workflow.return_value = mock_new_app - - result = workflow_service.convert_to_workflow(app, account, args, session=sqlite_session) - - assert result == mock_new_app - mock_converter.convert_to_workflow.assert_called_once() - - def test_convert_to_workflow_from_completion_app(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test convert_to_workflow converts completion app to workflow. - - Similar to chat conversion, but for completion-style apps. - Completion apps are simpler (single prompt-response), so the - conversion creates a basic workflow with fewer nodes. - """ - app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.COMPLETION) - account = TestWorkflowAssociatedDataFactory.create_account() - args = {"name": "Converted Workflow"} - - with patch("services.workflow_service.WorkflowConverter") as MockConverter: - mock_converter = MockConverter.return_value - mock_new_app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.WORKFLOW) - mock_converter.convert_to_workflow.return_value = mock_new_app - - result = workflow_service.convert_to_workflow(app, account, args, session=sqlite_session) - - assert result == mock_new_app - - def test_convert_to_workflow_invalid_mode_raises_error( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test convert_to_workflow raises error for invalid app mode. - - Only chat and completion apps can be converted to workflows. - Apps that are already workflows or have other modes cannot be converted. - """ - app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.WORKFLOW) - account = TestWorkflowAssociatedDataFactory.create_account() - args = {} - - with pytest.raises(ValueError, match="not supported convert to workflow"): - workflow_service.convert_to_workflow(app, account, args, session=sqlite_session) - # =========================================================================== # TestWorkflowServiceCredentialValidation @@ -2157,6 +686,10 @@ class TestWorkflowServiceCredentialValidation: def service(self, sqlite_engine: Engine) -> WorkflowService: return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) + @pytest.fixture(autouse=True) + def credential_repository(self, sqlite_session_factory: sessionmaker[Session]) -> None: + self.credentials = CredentialQueryRepository(session_factory=sqlite_session_factory) + @staticmethod def _make_workflow(nodes: list[dict]) -> Workflow: return TestWorkflowAssociatedDataFactory.create_workflow( @@ -2186,7 +719,7 @@ def test_validate_workflow_credentials_should_check_tool_credential_when_credent # Act + Assert with patch("core.helper.credential_utils.check_credential_policy_compliance") as mock_check: # Should not raise; mock allows the call - service._validate_workflow_credentials(workflow, session=sqlite_session) + service._validate_workflow_credentials(workflow, credentials=self.credentials) mock_check.assert_called_once() def test_validate_workflow_credentials_should_check_default_credential_when_no_credential_id( @@ -2207,10 +740,10 @@ def test_validate_workflow_credentials_should_check_default_credential_when_no_c # Act with patch.object(service, "_check_default_tool_credential") as mock_default: - service._validate_workflow_credentials(workflow, session=sqlite_session) + service._validate_workflow_credentials(workflow, credentials=self.credentials) # Assert - mock_default.assert_called_once_with("tenant-1", "my-provider", session=sqlite_session) + mock_default.assert_called_once_with("tenant-1", "my-provider", credentials=self.credentials) def test_validate_workflow_credentials_should_skip_tool_node_without_provider( self, service: WorkflowService, sqlite_session: Session @@ -2222,7 +755,7 @@ def test_validate_workflow_credentials_should_skip_tool_node_without_provider( # Act + Assert (no error raised) with patch.object(service, "_check_default_tool_credential") as mock_default: - service._validate_workflow_credentials(workflow, session=sqlite_session) + service._validate_workflow_credentials(workflow, credentials=self.credentials) mock_default.assert_not_called() def test_validate_workflow_credentials_should_validate_llm_node_with_model_config( @@ -2245,7 +778,7 @@ def test_validate_workflow_credentials_should_validate_llm_node_with_model_confi patch.object(service, "_validate_llm_model_config") as mock_llm, patch.object(service, "_validate_load_balancing_credentials"), ): - service._validate_workflow_credentials(workflow, session=sqlite_session) + service._validate_workflow_credentials(workflow, credentials=self.credentials) # Assert mock_llm.assert_called_once_with("tenant-1", "openai", "gpt-4") @@ -2286,7 +819,7 @@ def test_validate_workflow_credentials_should_use_llm_environment_variable_model patch.object(service, "_validate_llm_model_config") as validate_model, patch.object(service, "_validate_load_balancing_credentials") as validate_load_balancing, ): - service._validate_workflow_credentials(workflow, session=sqlite_session) + service._validate_workflow_credentials(workflow, credentials=self.credentials) validate_model.assert_called_once_with("tenant-1", "new-provider", "new-model") validated_node_data = validate_load_balancing.call_args.args[1] @@ -2320,7 +853,7 @@ def test_validate_workflow_credentials_should_reject_llm_environment_variable_mo ] with pytest.raises(ValueError, match="uses mode 'completion'.*uses mode 'chat'"): - service._validate_workflow_credentials(workflow, session=sqlite_session) + service._validate_workflow_credentials(workflow, credentials=self.credentials) def test_validate_workflow_credentials_should_raise_for_llm_node_missing_model( self, service: WorkflowService, sqlite_session: Session @@ -2337,7 +870,7 @@ def test_validate_workflow_credentials_should_raise_for_llm_node_missing_model( # Act + Assert with pytest.raises(ValueError, match="Missing provider or model configuration"): - service._validate_workflow_credentials(workflow, session=sqlite_session) + service._validate_workflow_credentials(workflow, credentials=self.credentials) def test_validate_workflow_credentials_should_wrap_unexpected_exception_in_value_error( self, service: WorkflowService, sqlite_session: Session @@ -2358,7 +891,7 @@ def test_validate_workflow_credentials_should_wrap_unexpected_exception_in_value # Act + Assert with patch.object(service, "_validate_llm_model_config", side_effect=RuntimeError("boom")): with pytest.raises(ValueError, match="boom"): - service._validate_workflow_credentials(workflow, session=sqlite_session) + service._validate_workflow_credentials(workflow, credentials=self.credentials) def test_validate_workflow_credentials_should_validate_agent_node_model( self, service: WorkflowService, sqlite_session: Session @@ -2383,7 +916,7 @@ def test_validate_workflow_credentials_should_validate_agent_node_model( patch.object(service, "_validate_llm_model_config") as mock_llm, patch.object(service, "_validate_load_balancing_credentials"), ): - service._validate_workflow_credentials(workflow, session=sqlite_session) + service._validate_workflow_credentials(workflow, credentials=self.credentials) # Assert mock_llm.assert_called_once_with("tenant-1", "openai", "gpt-4") @@ -2417,11 +950,11 @@ def test_validate_workflow_credentials_should_validate_agent_tools( patch("core.helper.credential_utils.check_credential_policy_compliance") as mock_check, patch.object(service, "_check_default_tool_credential") as mock_default, ): - service._validate_workflow_credentials(workflow, session=sqlite_session) + service._validate_workflow_credentials(workflow, credentials=self.credentials) # Assert mock_check.assert_called_once() # provider-a has credential_id - mock_default.assert_called_once_with("tenant-1", "provider-b", session=sqlite_session) + mock_default.assert_called_once_with("tenant-1", "provider-b", credentials=self.credentials) # --- _validate_llm_model_config --- @@ -2478,7 +1011,7 @@ def test_check_default_tool_credential_should_silently_pass_when_no_provider_fou self, service: WorkflowService, sqlite_session: Session ) -> None: """Missing BuiltinToolProvider → plugin requires no credentials → no error.""" - service._check_default_tool_credential("tenant-1", "some-provider", session=sqlite_session) + service._check_default_tool_credential("tenant-1", "some-provider", credentials=self.credentials) def test_check_default_tool_credential_should_raise_when_compliance_fails( self, service: WorkflowService, sqlite_session: Session @@ -2494,7 +1027,7 @@ def test_check_default_tool_credential_should_raise_when_compliance_fails( with patch("core.helper.credential_utils.check_credential_policy_compliance", side_effect=Exception("denied")): with pytest.raises(ValueError, match="Failed to validate default credential"): - service._check_default_tool_credential("tenant-1", "some-provider", session=sqlite_session) + service._check_default_tool_credential("tenant-1", "some-provider", credentials=self.credentials) # --- _is_load_balancing_enabled --- @@ -2540,43 +1073,6 @@ def test_is_load_balancing_enabled_should_return_false_on_exception(self, servic # Assert assert result is False - # --- _get_load_balancing_configs --- - - def test_get_load_balancing_configs_should_return_empty_list_on_exception( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - """Any exception during LB config retrieval should return an empty list.""" - # Arrange - with patch( - "services.model_load_balancing_service.ModelLoadBalancingService.get_load_balancing_configs", - side_effect=RuntimeError("fail"), - ): - # Act - result = service._get_load_balancing_configs("tenant-1", "openai", "gpt-4", session=sqlite_session) - - # Assert - assert result == [] - - def test_get_load_balancing_configs_should_merge_predefined_and_custom( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - # Arrange - predefined = [{"credential_id": "cred-a"}, {"credential_id": None}] - custom = [{"credential_id": "cred-b"}] - with patch( - "services.model_load_balancing_service.ModelLoadBalancingService.get_load_balancing_configs", - side_effect=[ - (None, predefined), # first call: predefined-model - (None, custom), # second call: custom-model - ], - ): - # Act - result = service._get_load_balancing_configs("tenant-1", "openai", "gpt-4", session=sqlite_session) - - # Assert — only entries with a credential_id should be returned - assert len(result) == 2 - assert all(c["credential_id"] for c in result) - # --- _validate_load_balancing_credentials --- def test_validate_load_balancing_credentials_should_skip_when_no_model_config( @@ -2588,7 +1084,7 @@ def test_validate_load_balancing_credentials_should_skip_when_no_model_config( node_data: dict[str, Any] = {} # no model key # Act + Assert (no error expected) - service._validate_load_balancing_credentials(workflow, node_data, "node-1", session=sqlite_session) + service._validate_load_balancing_credentials(workflow, node_data, "node-1", credentials=self.credentials) def test_validate_load_balancing_credentials_should_skip_when_lb_not_enabled( self, service: WorkflowService, sqlite_session: Session @@ -2599,7 +1095,7 @@ def test_validate_load_balancing_credentials_should_skip_when_lb_not_enabled( # Act + Assert (no error expected) with patch.object(service, "_is_load_balancing_enabled", return_value=False): - service._validate_load_balancing_credentials(workflow, node_data, "node-1", session=sqlite_session) + service._validate_load_balancing_credentials(workflow, node_data, "node-1", credentials=self.credentials) def test_validate_load_balancing_credentials_should_raise_when_compliance_fails( self, service: WorkflowService, sqlite_session: Session @@ -2607,19 +1103,21 @@ def test_validate_load_balancing_credentials_should_raise_when_compliance_fails( # Arrange workflow = self._make_workflow([]) node_data = {"model": {"provider": "openai", "name": "gpt-4"}} - lb_configs = [{"credential_id": "cred-lb-1"}] + lb_configs = ["cred-lb-1"] # Act + Assert with ( patch.object(service, "_is_load_balancing_enabled", return_value=True), - patch.object(service, "_get_load_balancing_configs", return_value=lb_configs), + patch.object(self.credentials, "list_model_load_balancing_credential_ids", return_value=lb_configs), patch( "core.helper.credential_utils.check_credential_policy_compliance", side_effect=Exception("policy violation"), ), ): with pytest.raises(ValueError, match="Invalid load balancing credentials"): - service._validate_load_balancing_credentials(workflow, node_data, "node-1", session=sqlite_session) + service._validate_load_balancing_credentials( + workflow, node_data, "node-1", credentials=self.credentials + ) # =========================================================================== @@ -3186,430 +1684,12 @@ def test_rebuild_should_skip_variable_not_in_inputs( assert "attachment" not in result -class TestWorkflowServiceResolveDeliveryMethod: - """ - Tests for the static helper `_resolve_human_input_delivery_method`. - """ - - def _make_method(self, method_id) -> MagicMock: - m = MagicMock() - m.id = method_id - return m - - def test_resolve_delivery_method_should_return_method_when_id_matches(self) -> None: - # Arrange - method_a = self._make_method("method-1") - method_b = self._make_method("method-2") - - # Act - with patch("services.workflow_service.parse_human_input_delivery_methods", return_value=[method_a, method_b]): - result = WorkflowService._resolve_human_input_delivery_method( - node_data=MagicMock(), delivery_method_id="method-2" - ) - - # Assert - assert result is method_b - - def test_resolve_delivery_method_should_return_none_when_no_match(self) -> None: - # Arrange - method_a = self._make_method("method-1") - - # Act - with patch("services.workflow_service.parse_human_input_delivery_methods", return_value=[method_a]): - result = WorkflowService._resolve_human_input_delivery_method( - node_data=MagicMock(), delivery_method_id="does-not-exist" - ) - - # Assert - assert result is None - - def test_resolve_delivery_method_should_return_none_for_empty_methods(self) -> None: - # Act - with patch("services.workflow_service.parse_human_input_delivery_methods", return_value=[]): - result = WorkflowService._resolve_human_input_delivery_method( - node_data=MagicMock(), delivery_method_id="method-1" - ) - - # Assert - assert result is None - - -# =========================================================================== -# TestWorkflowServiceDraftExecution -# Tests for run_draft_workflow_node -# =========================================================================== - - -class TestWorkflowServiceDraftExecution: - @pytest.fixture - def service(self, sqlite_engine: Engine) -> WorkflowService: - return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) - - def test_run_draft_workflow_node_should_execute_start_node_successfully( - self, service: WorkflowService, sqlite_engine: Engine - ) -> None: - # Arrange - app = TestWorkflowAssociatedDataFactory.create_app(app_id="app-1", tenant_id="tenant-1") - account = TestWorkflowAssociatedDataFactory.create_account(account_id="user-1") - node_id = "start-node" - draft_workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="wf-1", - tenant_id="tenant-1", - app_id="app-1", - graph={ - "nodes": [ - { - "id": node_id, - "data": {"type": BuiltinNodeTypes.START, "title": "Start", "variables": []}, - } - ], - "edges": [], - }, - ) - - # Mocking complex dependencies - with ( - patch("services.workflow_service.db", SimpleNamespace(engine=sqlite_engine)), - patch("services.workflow_service.WorkflowDraftVariableService"), - patch("services.workflow_service.StartNodeData") as mock_start_data, - patch( - "services.workflow_service._rebuild_file_for_user_inputs_in_start_node", - side_effect=lambda **kwargs: kwargs["user_inputs"], - ), - patch("services.workflow_service._setup_variable_pool"), - patch("services.workflow_service.DraftVarLoader"), - patch("services.workflow_service.WorkflowEntry.single_step_run") as mock_run, - patch("services.workflow_service.DifyCoreRepositoryFactory") as mock_repo_factory, - patch("services.workflow_service.DraftVariableSaver") as mock_saver_cls, - patch("services.workflow_service.storage"), - ): - mock_node = MagicMock() - mock_node.node_type = BuiltinNodeTypes.START - mock_node.title = "Start Node" - mock_run_result = NodeRunResult( - status=WorkflowNodeExecutionStatus.SUCCEEDED, inputs={}, outputs={"result": "ok"} - ) - mock_event = NodeRunSucceededEvent( - id=str(uuid.uuid4()), - node_id="start-node", - node_type=BuiltinNodeTypes.START, - node_run_result=mock_run_result, - start_at=naive_utc_now(), - ) - mock_run.return_value = (mock_node, [mock_event]) - - mock_repo = MagicMock() - mock_repo_factory.create_workflow_node_execution_repository.return_value = mock_repo - - service._node_execution_service_repo = MagicMock() - mock_execution_record = MagicMock() - mock_execution_record.node_type = "start" - mock_execution_record.node_id = "start-node" - mock_execution_record.load_full_outputs.return_value = {} - service._node_execution_service_repo.get_execution_by_id.return_value = mock_execution_record - - # Act - result = service.run_draft_workflow_node( - app_model=app, - draft_workflow=draft_workflow, - account=account, - node_id=node_id, - user_inputs={"key": "val"}, - query="hi", - files=[], - ) - - # Assert - assert result is not None - mock_run.assert_called_once() - mock_repo.save.assert_called_once() - mock_saver_cls.return_value.save.assert_called_once() - - def test_run_draft_workflow_node_should_execute_non_start_node_successfully( - self, service: WorkflowService, sqlite_engine: Engine - ) -> None: - # Arrange - app = TestWorkflowAssociatedDataFactory.create_app(app_id="app-1", tenant_id="tenant-1") - account = TestWorkflowAssociatedDataFactory.create_account(account_id="user-1") - node_id = "llm-node" - draft_workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="wf-1", - tenant_id="tenant-1", - app_id="app-1", - graph={ - "nodes": [ - { - "id": node_id, - "data": { - "type": BuiltinNodeTypes.LLM, - "title": "LLM", - "model": {"provider": "openai", "name": "gpt-4"}, - }, - } - ], - "edges": [], - }, - ) - - with ( - patch("services.workflow_service.db", SimpleNamespace(engine=sqlite_engine)), - patch("services.workflow_service.WorkflowDraftVariableService"), - patch("services.workflow_service.VariablePool") as mock_pool_cls, - patch("services.workflow_service.default_system_variables") as mock_default_system_variables, - patch("services.workflow_service.build_bootstrap_variables") as mock_build_bootstrap_variables, - patch("services.workflow_service.add_variables_to_pool") as mock_add_variables_to_pool, - patch("services.workflow_service.DraftVarLoader"), - patch("services.workflow_service.WorkflowEntry.single_step_run") as mock_run, - patch("services.workflow_service.DifyCoreRepositoryFactory"), - patch("services.workflow_service.DraftVariableSaver"), - patch("services.workflow_service.storage"), - ): - mock_node = MagicMock() - mock_node.node_type = BuiltinNodeTypes.LLM - mock_node.title = "LLM Node" - mock_run_result = NodeRunResult( - status=WorkflowNodeExecutionStatus.SUCCEEDED, inputs={}, outputs={"result": "ok"} - ) - mock_event = NodeRunSucceededEvent( - id=str(uuid.uuid4()), - node_id="llm-node", - node_type=BuiltinNodeTypes.LLM, - node_run_result=mock_run_result, - start_at=naive_utc_now(), - ) - mock_run.return_value = (mock_node, [mock_event]) - - service._node_execution_service_repo = MagicMock() - mock_execution_record = MagicMock() - mock_execution_record.node_type = "llm" - mock_execution_record.node_id = "llm-node" - mock_execution_record.load_full_outputs.return_value = {"answer": "hello"} - service._node_execution_service_repo.get_execution_by_id.return_value = mock_execution_record - - # Act - service.run_draft_workflow_node( - app_model=app, - draft_workflow=draft_workflow, - account=account, - node_id=node_id, - user_inputs={}, - query="", - files=None, - ) - - # Assert - # For non-start nodes, bootstrap variables should be loaded into an empty pool. - mock_pool_cls.assert_called_once_with() - mock_default_system_variables.assert_called_once() - mock_build_bootstrap_variables.assert_called_once_with( - system_variables=mock_default_system_variables.return_value, - environment_variables=draft_workflow.environment_variables, - ) - mock_add_variables_to_pool.assert_called_once_with( - mock_pool_cls.return_value, mock_build_bootstrap_variables.return_value - ) - - # =========================================================================== # TestWorkflowServiceHumanInputOperations # Tests for Human Input related methods # =========================================================================== -class TestWorkflowServiceHumanInputOperations: - @pytest.fixture - def service(self, sqlite_engine: Engine) -> WorkflowService: - return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) - - @staticmethod - def _create_human_input_workflow() -> Workflow: - return TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="wf-1", - tenant_id="tenant-1", - app_id="app-1", - graph={ - "nodes": [ - { - "id": "node-1", - "data": {"type": BuiltinNodeTypes.HUMAN_INPUT, "title": "Human Input"}, - } - ], - "edges": [], - }, - ) - - def test_get_human_input_form_preview_should_raise_if_workflow_not_init( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - service.get_draft_workflow = MagicMock(return_value=None) - with pytest.raises(ValueError, match="Workflow not initialized"): - service.get_human_input_form_preview( - app_model=TestWorkflowAssociatedDataFactory.create_app(), - account=TestWorkflowAssociatedDataFactory.create_account(), - node_id="node-1", - session=sqlite_session, - ) - - def test_get_human_input_form_preview_should_raise_if_wrong_node_type( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - draft = TestWorkflowAssociatedDataFactory.create_workflow( - graph={"nodes": [{"id": "node-1", "data": {"type": "llm", "title": "LLM"}}], "edges": []} - ) - service.get_draft_workflow = MagicMock(return_value=draft) - with pytest.raises(ValueError, match="Node type must be human-input"): - service.get_human_input_form_preview( - app_model=TestWorkflowAssociatedDataFactory.create_app(), - account=TestWorkflowAssociatedDataFactory.create_account(), - node_id="node-1", - session=sqlite_session, - ) - - def test_get_human_input_form_preview_success(self, service: WorkflowService, sqlite_session: Session) -> None: - app_model = TestWorkflowAssociatedDataFactory.create_app(app_id="app-1", tenant_id="tenant-1") - account = TestWorkflowAssociatedDataFactory.create_account(account_id="user-1") - draft = self._create_human_input_workflow() - service.get_draft_workflow = MagicMock(return_value=draft) - - mock_node = MagicMock() - mock_node.render_form_content_before_submission.return_value = "rendered" - mock_node.resolve_default_values.return_value = {"def": 1} - mock_node.title = "Form Title" - mock_node.node_data = MagicMock() - - with ( - patch.object(service, "_build_human_input_variable_pool"), - patch("services.workflow_service.HumanInputNode", return_value=mock_node), - patch("services.workflow_service.HumanInputRequired") as mock_required_cls, - ): - service.get_human_input_form_preview( - app_model=app_model, account=account, node_id="node-1", session=sqlite_session - ) - mock_node.render_form_content_before_submission.assert_called_once() - mock_required_cls.return_value.model_dump.assert_called_once() - - def test_submit_human_input_form_preview_success(self, service: WorkflowService, sqlite_session: Session) -> None: - app_model = TestWorkflowAssociatedDataFactory.create_app(app_id="app-1", tenant_id="tenant-1") - account = TestWorkflowAssociatedDataFactory.create_account(account_id="user-1") - draft = self._create_human_input_workflow() - service.get_draft_workflow = MagicMock(return_value=draft) - - mock_node = MagicMock() - mock_node.node_data = MagicMock() - mock_node.node_data.user_actions = [ - SimpleNamespace(id="submit", title="card_visa_enterprise_001"), - ] - mock_node.node_data.outputs_field_names.return_value = ["field1"] - mock_node.node_data.inputs = [] - mock_node.render_form_content_before_submission.return_value = "Ticket: {{#$output.field1#}}" - mock_node.render_form_content_with_outputs.return_value = "Ticket: val1" - - with ( - patch("services.workflow_service.db"), - patch.object(service, "_build_human_input_variable_pool"), - patch("services.workflow_service.HumanInputNode", return_value=mock_node), - patch( - "services.workflow_service.HumanInputService.validate_and_normalize_submission", - return_value={"field1": "val1"}, - ) as mock_validate, - patch("services.workflow_service.DraftVariableSaver") as mock_saver_cls, - ): - result = service.submit_human_input_form_preview( - app_model=app_model, - account=account, - node_id="node-1", - form_inputs={"field1": "val1"}, - action="submit", - session=sqlite_session, - ) - assert result["__action_id"] == "submit" - mock_validate.assert_called_once() - assert result["__action_value"] == "card_visa_enterprise_001" - assert result["__rendered_content"] == "Ticket: val1" - mock_saver_cls.return_value.save.assert_called_once() - - def test_test_human_input_delivery_success(self, service: WorkflowService, sqlite_session: Session) -> None: - draft = self._create_human_input_workflow() - service.get_draft_workflow = MagicMock(return_value=draft) - - with ( - patch("services.workflow_service.HumanInputNodeData.model_validate"), - patch.object(service, "_resolve_human_input_delivery_method") as mock_resolve, - patch("services.workflow_service.apply_dify_debug_email_recipient"), - patch.object(service, "_build_human_input_variable_pool"), - patch.object(service, "_build_human_input_node_for_debugging"), - patch.object(service, "_create_human_input_delivery_test_form", return_value=("form-1", [])), - patch("services.workflow_service.HumanInputDeliveryTestService") as mock_test_srv, - ): - mock_resolve.return_value = MagicMock() - service.test_human_input_delivery( - app_model=TestWorkflowAssociatedDataFactory.create_app(), - account=TestWorkflowAssociatedDataFactory.create_account(), - node_id="node-1", - delivery_method_id="method-1", - session=sqlite_session, - ) - mock_test_srv.return_value.send_test.assert_called_once() - - def test_test_human_input_delivery_failure_cases(self, service: WorkflowService, sqlite_session: Session) -> None: - draft = self._create_human_input_workflow() - service.get_draft_workflow = MagicMock(return_value=draft) - - with ( - patch("services.workflow_service.HumanInputNodeData.model_validate"), - patch.object(service, "_resolve_human_input_delivery_method", return_value=None), - ): - with pytest.raises(ValueError, match="Delivery method not found"): - service.test_human_input_delivery( - app_model=TestWorkflowAssociatedDataFactory.create_app(), - account=TestWorkflowAssociatedDataFactory.create_account(), - node_id="node-1", - delivery_method_id="none", - session=sqlite_session, - ) - - def test_load_email_recipients_parsing_failure(self, service: WorkflowService, sqlite_session: Session) -> None: - """Malformed persisted recipient payloads are skipped instead of aborting delivery tests.""" - recipient = HumanInputFormRecipient( - form_id="form-1", - delivery_id="delivery-1", - recipient_payload="invalid json", - recipient_type=RecipientType.EMAIL_MEMBER, - access_token="recipient-token", - ) - sqlite_session.add(recipient) - sqlite_session.commit() - - with patch("services.workflow_service.db") as mock_db: - mock_db.engine = sqlite_session.get_bind() - result = WorkflowService._load_email_recipients("form-1") - - assert result == [] - - def test_build_human_input_variable_pool(self, service: WorkflowService, sqlite_engine: Engine) -> None: - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - node_data = MagicMock() - node_data.extract_variable_selector_to_variable_mapping.return_value = {} - - with ( - patch("services.workflow_service.db", SimpleNamespace(engine=sqlite_engine)), - patch("services.workflow_service.WorkflowDraftVariableService"), - patch("services.workflow_service.VariablePool") as mock_pool_cls, - patch("services.workflow_service.DraftVarLoader"), - patch("services.workflow_service.HumanInputNodeData.model_validate", return_value=node_data), - patch("services.workflow_service.load_into_variable_pool"), - patch("services.workflow_service.WorkflowEntry.mapping_user_inputs_to_variable_pool"), - ): - service._build_human_input_variable_pool( - app_model=TestWorkflowAssociatedDataFactory.create_app(), - workflow=workflow, - node_config={"id": "node-1", "data": {}}, - manual_inputs={}, - user_id="user-1", - ) - mock_pool_cls.assert_called_once() - - # =========================================================================== # TestWorkflowServiceFreeNodeExecution # Tests for run_free_workflow_node and handle_single_step_result @@ -3662,28 +1742,3 @@ def test_rebuild_single_file_unreachable(self) -> None: # Test line 1523 (unreachable) with pytest.raises(Exception, match="unreachable"): _rebuild_single_file("tenant-1", {}, cast(Any, "invalid_type")) - - def test_build_human_input_node_for_debugging(self, service: WorkflowService) -> None: - """Cover _build_human_input_node_for_debugging.""" - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - account = TestWorkflowAssociatedDataFactory.create_account() - node_config = {"id": "n-1", "data": {"type": BuiltinNodeTypes.HUMAN_INPUT, "title": "Human Input"}} - variable_pool = MagicMock() - node_data = MagicMock() - node_data.title = "Human Input" - - with ( - patch( - "services.workflow_service.adapt_human_input_node_data_for_graph", - return_value=sentinel.adapted_node_data, - ) as mock_adapt_node_data, - patch("services.workflow_service.HumanInputNodeData.model_validate", return_value=node_data), - ): - node = service._build_human_input_node_for_debugging( - workflow=workflow, account=account, node_config=node_config, variable_pool=variable_pool - ) - mock_adapt_node_data.assert_called_once_with(node_config["data"]) - assert node.node_id == "n-1" - assert node.title == "Human Input" - assert node.node_data is node_data - assert node.variable_pool is variable_pool diff --git a/api/tests/unit_tests/services/tools/test_api_provider_runtime.py b/api/tests/unit_tests/services/tools/test_api_provider_runtime.py new file mode 100644 index 00000000000000..cefbf81d3da6f4 --- /dev/null +++ b/api/tests/unit_tests/services/tools/test_api_provider_runtime.py @@ -0,0 +1,155 @@ +"""Exercise Agent API-tool invocation through real provider and credentials reads.""" + +import json + +import httpx +import pytest +from sqlalchemy import create_engine, event +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import QueuePool + +from core.db import session_factory +from core.tools.entities.tool_bundle import ApiToolBundle +from core.tools.entities.tool_entities import ApiProviderSchemaType +from core.tools.errors import ToolInvokeError +from extensions.application_services.workflow import build_workflow_execution_dependencies +from extensions.ext_database import db +from models.account import Account +from models.base import TypeBase +from models.model import App +from models.tools import ApiToolProvider +from services.entities.agent_tool_inner import AgentToolInvokeRequest +from services.errors.agent_tool_inner import AgentToolInnerServiceError +from tests.unit_tests.model_factories import make_account +from tests.unit_tests.services.agent.test_tool_invocation_service import ( + TENANT_ID, + USER_ID, + _persist_app, + _request, + _service, +) + + +@pytest.fixture +def databases(monkeypatch): + engines = [create_engine("sqlite://", poolclass=QueuePool) for _ in range(2)] + for engine in engines: + TypeBase.metadata.create_all(engine, tables=[model.__table__ for model in (App, Account, ApiToolProvider)]) + sessions, global_sessions = [sessionmaker(engine, expire_on_commit=False) for engine in engines] + with global_sessions() as global_session: + monkeypatch.setattr(db, "session", global_session) + monkeypatch.setattr(type(db), "engine", property(lambda _db: engines[1])) + monkeypatch.setattr(session_factory, "create_session", global_sessions) + + def reject_global(*_args): + pytest.fail("Tool runtime queried the global database") + + event.listen(engines[1], "before_cursor_execute", reject_global) + try: + yield sessions, global_session + finally: + global_session.close() + for engine in engines: + engine.dispose() + + +@pytest.mark.parametrize("http_failure", [False, True]) +def test_api_tool_provider_read_finishes_before_decryption_and_http(databases, monkeypatch, http_failure): + sessions, global_session = databases + with sessions() as session: + _persist_app(session) + session.add(make_account(account_id=USER_ID, name="Injected author")) + provider = ApiToolProvider( + name="search", + icon="icon.svg", + schema="{}", + schema_type_str=ApiProviderSchemaType.OPENAPI, + tenant_id=TENANT_ID, + user_id=USER_ID, + description="Search", + credentials_str=json.dumps({"auth_type": "api_key_header", "api_key_value": "encrypted"}), + tools_str=json.dumps( + [ + ApiToolBundle( + server_url="https://api.example.com/search", + method="GET", + operation_id="search", + summary="Search", + author="Injected author", + parameters=[], + openapi={"parameters": []}, + ).model_dump(mode="json") + ] + ), + ) + session.add(provider) + session.commit() + engine = sessions.kw["bind"] + calls = [] + + def released(): + assert engine.pool.checkedout() == 0 + assert not global_session.in_transaction() + + class Decrypter: + def decrypt(self, credentials): + released() + assert credentials["api_key_value"] == "encrypted" + calls.append("decrypt") + return {**credentials, "api_key_value": "clear-secret"} + + def encryption(*, tenant_id, controller): + released() + assert tenant_id == TENANT_ID + assert controller.entity.identity.author == "Injected author" + return Decrypter(), None + + def http_get(url, **kwargs): + released() + assert url == "https://api.example.com/search" + assert kwargs["headers"]["Authorization"] == "clear-secret" + calls.append("http") + if http_failure: + raise ToolInvokeError("remote unavailable") + return httpx.Response(200, text="ok", request=httpx.Request("GET", url)) + + monkeypatch.setattr("services.tools.tool_manager.create_tool_provider_encrypter", encryption) + monkeypatch.setattr("core.tools.custom_tool.tool.ssrf_proxy.get", http_get) + payload = _request().model_dump() + payload["tool"].update(provider_type="api", provider_id=provider.id, tool_parameters={}, runtime_parameters={}) + request = AgentToolInvokeRequest.model_validate(payload) + runtime = build_workflow_execution_dependencies(sessions) + with sessions() as session: + service = _service(session, runtime) + if http_failure: + with pytest.raises(AgentToolInnerServiceError) as exc: + service.invoke(request) + assert exc.value.error_code == "agent_tool_invoke_failed" + else: + assert service.invoke(request).observation == "ok" + assert calls == ["decrypt", "http"] + released() + + +def test_api_provider_lookup_is_tenant_scoped(databases): + sessions, _ = databases + with sessions.begin() as session: + provider = ApiToolProvider( + name="private", + icon="", + schema="{}", + schema_type_str=ApiProviderSchemaType.OPENAPI, + tenant_id=TENANT_ID, + user_id=USER_ID, + description="", + credentials_str="{}", + tools_str="[]", + ) + session.add(provider) + runtime = build_workflow_execution_dependencies(sessions) + assert runtime.tool_providers.get(tenant_id="other-tenant", provider_id=provider.id) is None + record = runtime.tool_providers.get(tenant_id=TENANT_ID, provider_id=provider.id) + assert record is not None + assert record.author == "" + assert record.tools == [] + assert sessions.kw["bind"].pool.checkedout() == 0 diff --git a/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py b/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py new file mode 100644 index 00000000000000..e9fd8e55002c9a --- /dev/null +++ b/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py @@ -0,0 +1,289 @@ +"""Exercise selected builtin credentials through real repositories and real tool construction.""" + +import json +from uuid import uuid4 + +import pytest +from sqlalchemy import create_engine, event +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import QueuePool + +from core.db import session_factory +from core.plugin.entities.plugin_daemon import CredentialType, PluginOAuthCredentialsResponse +from core.tools.entities.tool_entities import ToolProviderEntityWithPlugin, ToolProviderType +from core.tools.errors import ToolProviderCredentialValidationError, ToolProviderNotFoundError +from core.tools.plugin_tool.provider import PluginToolProviderController +from extensions.application_services.workflow import build_workflow_execution_dependencies +from extensions.ext_database import db +from models.account import Account +from models.base import TypeBase +from models.tools import BuiltinToolProvider, MCPToolProvider, ToolOAuthSystemClient, ToolOAuthTenantClient +from services.entities.agent_tool_inner import AgentToolInvokeRequest +from services.errors.agent_tool_inner import AgentToolInnerServiceError +from services.tools.agent_invocation_gateway import AgentToolInvocationGateway +from services.tools.builtin import credentials as credentials_module +from services.tools.tool_manager import ToolManager + +TENANT = "tenant" +PROVIDER = "vendor/search/search" + + +@pytest.fixture +def databases(monkeypatch): + engines = [create_engine("sqlite://", poolclass=QueuePool) for _ in range(2)] + for engine in engines: + TypeBase.metadata.create_all( + engine, + tables=[ + model.__table__ + for model in ( + Account, + BuiltinToolProvider, + ToolOAuthTenantClient, + ToolOAuthSystemClient, + MCPToolProvider, + ) + ], + ) + sessions, global_sessions = [sessionmaker(engine, expire_on_commit=False) for engine in engines] + with global_sessions() as global_session: + monkeypatch.setattr(db, "session", global_session) + monkeypatch.setattr(type(db), "engine", property(lambda _db: engines[1])) + monkeypatch.setattr(session_factory, "create_session", global_sessions) + + def reject_global(*_args): + pytest.fail("Tool runtime read the global database") + + event.listen(engines[1], "before_cursor_execute", reject_global) + try: + yield sessions, global_session + finally: + global_session.close() + for engine in engines: + engine.dispose() + + +def seed(sessions, *, expires_at=1, oauth_client="tenant"): + with sessions.begin() as session: + credential = BuiltinToolProvider( + tenant_id=TENANT, + user_id="owner", + provider=PROVIDER, + name="Selected", + encrypted_credentials=json.dumps({"token": "encrypted-old"}), + credential_type=CredentialType.OAUTH2, + expires_at=expires_at, + ) + session.add(credential) + if oauth_client == "tenant": + client = ToolOAuthTenantClient(tenant_id=TENANT, plugin_id="vendor/search", provider="search") + client.encrypted_oauth_params = json.dumps({"client": "encrypted-client"}) + session.add(client) + elif oauth_client == "system": + session.add( + ToolOAuthSystemClient(plugin_id="vendor/search", provider="search", encrypted_oauth_params="system") + ) + return credential.id + + +def controller(): + entity = ToolProviderEntityWithPlugin.model_validate( + { + "identity": { + "author": "author", + "name": PROVIDER, + "description": {"en_US": "Search"}, + "icon": "icon.svg", + "label": {"en_US": "Search"}, + }, + "credentials_schema": [ + {"name": "token", "label": {"en_US": "Token"}, "type": "secret-input", "required": True} + ], + "oauth_schema": {"client_schema": [], "credentials_schema": []}, + "tools": [ + {"identity": {"author": "author", "name": "search", "provider": PROVIDER, "label": {"en_US": "Search"}}} + ], + } + ) + return PluginToolProviderController( + entity=entity, plugin_id="vendor/search", plugin_unique_identifier="vendor/search:1", tenant_id=TENANT + ) + + +@pytest.mark.parametrize("client", ["tenant", "system"]) +@pytest.mark.parametrize("outcome", ["success", "refresh-error", "concurrent-edit", "concurrent-delete"]) +def test_selected_oauth_credential_releases_connections_across_full_invocation(databases, monkeypatch, client, outcome): + sessions, global_session = databases + credential_id = seed(sessions, oauth_client=client) + runtime = build_workflow_execution_dependencies(sessions) + calls = [] + + def released(): + assert sessions.kw["bind"].pool.checkedout() == 0 + assert not global_session.in_transaction() + + class Codec: + def decrypt(self, values): + released() + calls.append("decrypt") + return {k: v.removeprefix("encrypted-") for k, v in values.items()} + + def encrypt(self, values): + released() + calls.append("encrypt") + return {k: "encrypted-" + v for k, v in values.items()} + + class Cache: + def delete(self): + released() + calls.append("invalidate") + + def encrypter(**_kwargs): + released() + return Codec(), Cache() + + def policy(**kwargs): + released() + assert kwargs["check_existence"] is False + assert kwargs["credential_id"] == credential_id + calls.append("policy") + + def refresh(_self, **kwargs): + released() + calls.append("refresh") + assert kwargs["credentials"] == {"token": "old"} + assert kwargs["system_credentials"] == {"client": "client"} + if outcome == "refresh-error": + raise ValueError("broker unavailable") + if outcome in {"concurrent-edit", "concurrent-delete"}: + with sessions.begin() as session: + record = session.get(BuiltinToolProvider, credential_id) + assert record is not None + if outcome == "concurrent-delete": + session.delete(record) + else: + record.encrypted_credentials = '{"token":"manual-edit"}' + return PluginOAuthCredentialsResponse(credentials={"token": "new"}, expires_at=4_000_000_000) + + def invoke(_self, **kwargs): + released() + calls.append("invoke") + assert kwargs["credentials"] == {"token": "new"} + return iter(()) + + def verified(*_args): + released() + calls.append("verified") + return True + + def decrypt_system(params): + released() + assert params == "system" + return {"client": "client"} + + monkeypatch.setattr(ToolManager, "get_builtin_provider", lambda *_args: controller()) + monkeypatch.setattr(credentials_module, "create_provider_encrypter", encrypter) + monkeypatch.setattr(credentials_module, "runtime_check_credential_policy_compliance", policy) + monkeypatch.setattr(credentials_module.OAuthHandler, "refresh_credentials", refresh) + monkeypatch.setattr(credentials_module.PluginService, "is_plugin_verified", verified) + monkeypatch.setattr(credentials_module, "decrypt_system_params", decrypt_system) + monkeypatch.setattr("core.plugin.impl.tool.PluginToolManager.invoke", invoke) + + class Variables: + def saver_factory(self, *_args, **_kwargs): + raise AssertionError("No workflow variables for this tool") + + request = AgentToolInvokeRequest.model_validate( + { + "caller": { + "tenant_id": TENANT, + "user_id": "caller", + "user_from": "account", + "app_id": "app", + "invoke_from": "debugger", + }, + "tool": { + "provider_type": "builtin", + "provider_id": PROVIDER, + "tool_name": "search", + "credential_id": credential_id, + }, + } + ) + gateway = AgentToolInvocationGateway(variables=Variables(), runtime=runtime) + if outcome == "success": + gateway.invoke(request) + assert calls[-1] == "invoke" + else: + with pytest.raises(AgentToolInnerServiceError) as exc: + gateway.invoke(request) + assert exc.value.error_code == "agent_tool_credential_invalid" + assert "invoke" not in calls + released() + with sessions() as session: + persisted = session.get(BuiltinToolProvider, credential_id) + if outcome == "concurrent-delete": + assert persisted is None + else: + assert persisted is not None + expected = {"success": "encrypted-new", "refresh-error": "encrypted-old", "concurrent-edit": "manual-edit"}[ + outcome + ] + assert persisted.credentials == {"token": expected} + assert persisted.expires_at == (4_000_000_000 if outcome == "success" else 1) + + +@pytest.mark.parametrize("wrong_scope", ["tenant", "provider"]) +def test_selected_credential_cannot_cross_owner_scope(databases, monkeypatch, wrong_scope): + sessions, _ = databases + credential_id = seed(sessions, expires_at=-1) + with sessions.begin() as session: + record = session.get(BuiltinToolProvider, credential_id) + if wrong_scope == "tenant": + record.tenant_id = "other" + else: + record.provider = "other/provider" + monkeypatch.setattr(ToolManager, "get_builtin_provider", lambda *_args: controller()) + runtime = build_workflow_execution_dependencies(sessions) + with pytest.raises(ToolProviderCredentialValidationError, match="has been deleted"): + ToolManager.get_tool_runtime( + provider_type=ToolProviderType.BUILT_IN, + provider_id=PROVIDER, + tool_name="search", + tenant_id=TENANT, + credential_id=credential_id, + tool_providers=runtime.tool_providers, + workflow_queries=runtime.tools, + ) + + +@pytest.mark.parametrize("reference", ["id", "identifier", "uuid-identifier", "missing", "other-tenant"]) +def test_mcp_runtime_resolves_injected_provider_before_decryption(databases, monkeypatch, reference): + sessions, _ = databases + with sessions.begin() as session: + record = MCPToolProvider( + name="server", + server_identifier=str(uuid4()) if reference == "uuid-identifier" else "server", + server_url="encrypted-url", + server_url_hash="hash", + icon="icon.svg", + tenant_id=TENANT, + user_id="owner", + ) + session.add(record) + + def build(provider): + assert sessions.kw["bind"].pool.checkedout() == 0 + assert provider.id == record.id + return "controller" + + monkeypatch.setattr("services.tools.tool_manager.MCPToolProviderController.from_db", build) + key = record.id if reference == "id" else "missing" if reference == "missing" else record.server_identifier + providers = build_workflow_execution_dependencies(sessions).tool_providers + if reference in {"missing", "other-tenant"}: + with pytest.raises(ToolProviderNotFoundError): + ToolManager.get_mcp_provider_controller( + "other" if reference == "other-tenant" else TENANT, key, tool_providers=providers + ) + else: + assert ToolManager.get_mcp_provider_controller(TENANT, key, tool_providers=providers) == "controller" diff --git a/api/tests/unit_tests/services/workflow/test_conversion_service.py b/api/tests/unit_tests/services/workflow/test_conversion_service.py new file mode 100644 index 00000000000000..ab1e667185ca81 --- /dev/null +++ b/api/tests/unit_tests/services/workflow/test_conversion_service.py @@ -0,0 +1,262 @@ +"""Real conversion, isolated persistence, and failures at the atomic write boundary.""" + +import json + +import pytest +from sqlalchemy import event, func, select +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session, SessionTransaction, sessionmaker + +from machinery.context import RequestContext +from models.account import TenantAccountJoin, TenantAccountRole +from models.api_based_extension import APIBasedExtension +from models.model import App, AppMode, AppModelConfig, InstalledApp, Site +from models.workflow import Workflow +from repositories.app.console_repository import ConsoleAppRepository +from services.app.console_service import ConsoleAppNotFoundError +from services.entities.app_entities import AppEvent +from services.errors.workflow_service import WorkflowConversionError +from services.workflow.conversion_service import WorkflowConversionService +from tests.unit_tests.model_factories import make_account, make_app, make_tenant + +CONTEXT = RequestContext("conversion", None, "account-1", "tenant-1") + + +@pytest.fixture +def sessions(sqlite_engine: Engine) -> sessionmaker[Session]: + class ConversionSession(Session): + pass + + factory = sessionmaker(sqlite_engine, class_=ConversionSession, expire_on_commit=False) + extension = APIBasedExtension( + tenant_id="tenant-1", name="Weather", api_endpoint="https://example.com/weather", api_key="encrypted" + ) + extension.id = "extension-1" + with factory.begin() as session: + session.add_all( + [ + make_account(), + make_tenant(), + TenantAccountJoin( + tenant_id=CONTEXT.active_workspace_id, + account_id=CONTEXT.account_id, + role=TenantAccountRole.OWNER, + current=True, + ), + make_app(name="Source", app_model_config_id="config-1", api_rpm=10, api_rph=100), + ( + config := AppModelConfig( + app_id="app-1", + model=json.dumps( + { + "provider": "openai", + "name": "gpt-4", + "mode": "chat", + "completion_params": {"stop": ["stop"]}, + } + ), + pre_prompt="Hello {{name}}: {{weather}}", + user_input_form=json.dumps([{"text-input": {"variable": "name", "label": "Name"}}]), + external_data_tools=json.dumps( + [ + { + "enabled": True, + "variable": "weather", + "type": "api", + "config": {"api_based_extension_id": "extension-1"}, + } + ] + ), + dataset_configs=json.dumps( + { + "retrieval_model": "multiple", + "top_k": 3, + "datasets": {"datasets": [{"dataset": {"enabled": True, "id": "dataset-1"}}]}, + } + ), + dataset_query_variable="name", + opening_statement="Welcome", + ) + ), + extension, + ] + ) + config.id = "config-1" + return factory + + +@pytest.mark.parametrize("mode", [AppMode.CHAT, AppMode.COMPLETION]) +@pytest.mark.parametrize("agent_enabled", [False, True]) +def test_conversion_releases_reads_and_commits_target_once( + sessions: sessionmaker[Session], mode: AppMode, agent_enabled: bool +) -> None: + with sessions.begin() as session: + source = session.get(App, "app-1") + config = session.get(AppModelConfig, "config-1") + assert source is not None + assert config is not None + source.mode = mode + config.agent_mode = json.dumps({"enabled": agent_enabled, "strategy": "react"}) + original_config = config.to_dict(annotation_reply={"enabled": False}) + active: set[Session] = set() + commits = [] + event.listen(sessions.class_, "after_begin", lambda session, *_: active.add(session)) + + def end_transaction(session: Session, transaction: SessionTransaction) -> None: + if transaction.parent is None: + active.discard(session) + + event.listen(sessions.class_, "after_transaction_end", end_transaction) + event.listen(sessions.class_, "after_commit", lambda session: commits.append(session)) + notifications = [] + + def decrypt(tenant_id: str, token: str) -> str: + assert not active + assert (tenant_id, token) == ("tenant-1", "encrypted") + return "decrypted" + + def created(*, event: AppEvent, account_id: str, backing_agent_id: str | None) -> None: + assert not active + assert len(commits) == 1 + assert (account_id, backing_agent_id) == (CONTEXT.account_id, None) + with sessions() as session: + assert session.scalar(select(Workflow).where(Workflow.app_id == event.id)) is not None + assert session.scalar(select(Site).where(Site.app_id == event.id)) is not None + assert session.scalar(select(InstalledApp).where(InstalledApp.app_id == event.id)) is not None + notifications.append(event) + + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), decrypt_token=decrypt, notify_created=created + ) + new_id = service.convert(CONTEXT, "app-1", {"name": "", "icon_type": "", "icon": "", "icon_background": ""}) + assert len(notifications) == 1 + assert notifications[0].id == new_id + with sessions() as session: + target = session.get(App, new_id) + source = session.get(App, "app-1") + config = session.get(AppModelConfig, "config-1") + assert target is not None + assert source is not None + assert config is not None + assert target.name == "Source(workflow)" + assert target.mode == (AppMode.WORKFLOW if mode == AppMode.COMPLETION else AppMode.ADVANCED_CHAT) + assert (target.icon_type, target.icon, target.icon_background) == ( + source.icon_type, + source.icon, + source.icon_background, + ) + assert (target.api_rpm, target.api_rph, target.enable_site, target.enable_api) == (10, 100, True, True) + assert (target.created_by, target.maintainer, target.updated_by) == (CONTEXT.account_id,) * 3 + assert source.mode == mode + assert config.to_dict(annotation_reply={"enabled": False}) == original_config + assert session.scalar(select(Workflow.id).where(Workflow.app_id == source.id)) is None + workflow = session.scalar(select(Workflow).where(Workflow.app_id == new_id)) + assert workflow is not None + assert workflow.version == Workflow.VERSION_DRAFT + nodes = json.loads(workflow.graph)["nodes"] + assert [node["id"] for node in nodes] == [ + "start", + "http_request_1", + "code_1", + "knowledge_retrieval", + "llm", + "end" if mode == AppMode.COMPLETION else "answer", + ] + http = nodes[1]["data"] + assert http["authorization"]["config"]["api_key"] == "decrypted" + params = json.loads(http["body"]["data"])["params"] + assert params["app_id"] == source.id + assert params["query"] == ("{{#sys.query#}}" if mode == AppMode.CHAT else "") + assert nodes[3]["data"]["query_variable_selector"] == ( + ["sys", "query"] if mode == AppMode.CHAT else ["start", "name"] + ) + assert "{{#code_1.result#}}" in json.dumps(nodes[4]) + features = json.loads(workflow.features) + if mode == AppMode.CHAT: + assert features["opening_statement"] == "Welcome" + assert "retriever_resource" in features + else: + assert set(features) == {"text_to_speech", "file_upload", "sensitive_word_avoidance"} + + +@pytest.mark.parametrize("failed_record", [InstalledApp, Workflow]) +def test_failed_conversion_rolls_back_entire_target(sessions: sessionmaker[Session], failed_record: type) -> None: + def fail_write(session: Session, *_: object) -> None: + if any(isinstance(row, failed_record) for row in session.new): + raise RuntimeError("write failed") + + event.listen(sessions.class_, "before_flush", fail_write) + notifications = [] + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), + decrypt_token=lambda *_: "key", + notify_created=lambda **kwargs: notifications.append(kwargs), + ) + with pytest.raises(RuntimeError, match="write failed"): + service.convert(CONTEXT, "app-1", {}) + assert notifications == [] + with sessions() as session: + assert session.scalar(select(func.count()).select_from(App)) == 1 + for model in (Workflow, Site, InstalledApp): + assert session.scalar(select(func.count()).select_from(model)) == 0 + + +@pytest.mark.parametrize( + "invalid", ["mode", "missing_config", "foreign_config", "foreign_extension", "missing_extension"] +) +def test_conversion_rejects_invalid_owned_data_before_creating_target( + sessions: sessionmaker[Session], invalid: str +) -> None: + with sessions.begin() as session: + source = session.get(App, "app-1") + config = session.get(AppModelConfig, "config-1") + extension = session.get(APIBasedExtension, "extension-1") + assert source is not None + assert config is not None + assert extension is not None + if invalid == "mode": + source.mode = AppMode.WORKFLOW + elif invalid == "missing_config": + source.app_model_config_id = None + elif invalid == "foreign_config": + config.app_id = "another-app" + elif invalid == "foreign_extension": + extension.tenant_id = "another-tenant" + else: + session.delete(extension) + notifications = [] + decryptions = [] + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), + decrypt_token=lambda *args: decryptions.append(args) or "key", + notify_created=lambda **kwargs: notifications.append(kwargs), + ) + with pytest.raises(WorkflowConversionError): + service.convert(CONTEXT, "app-1", {}) + assert notifications == [] + assert decryptions == [] + with sessions() as session: + assert session.scalar(select(func.count()).select_from(App)) == 1 + assert session.scalar(select(func.count()).select_from(Workflow)) == 0 + + +def test_conversion_rejects_other_tenant(sessions: sessionmaker[Session]) -> None: + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), decrypt_token=lambda *_: "key", notify_created=lambda **_: None + ) + with pytest.raises(ConsoleAppNotFoundError): + service.convert(CONTEXT._replace(active_workspace_id="other-tenant"), "app-1", {}) + + +def test_decryption_failure_creates_no_target(sessions: sessionmaker[Session]) -> None: + def fail(*_: str) -> str: + raise RuntimeError("key service unavailable") + + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), decrypt_token=fail, notify_created=lambda **_: None + ) + with pytest.raises(RuntimeError, match="key service unavailable"): + service.convert(CONTEXT, "app-1", {}) + with sessions() as session: + assert session.scalar(select(func.count()).select_from(App)) == 1 + assert session.scalar(select(func.count()).select_from(Workflow)) == 0 diff --git a/api/tests/unit_tests/services/workflow/test_draft_var_loader_simple.py b/api/tests/unit_tests/services/workflow/test_draft_var_loader_simple.py index 2fe76f7becccd9..c695e76a7a42f9 100644 --- a/api/tests/unit_tests/services/workflow/test_draft_var_loader_simple.py +++ b/api/tests/unit_tests/services/workflow/test_draft_var_loader_simple.py @@ -6,7 +6,7 @@ import pytest from sqlalchemy import Engine -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from core.workflow.file_reference import build_file_reference from extensions.storage.storage_type import StorageType @@ -16,7 +16,8 @@ from models.enums import CreatorUserRole from models.model import UploadFile from models.workflow import WorkflowDraftVariable, WorkflowDraftVariableFile -from services.workflow_draft_variable_service import DraftVarLoader +from repositories.workflow.draft_variable_repository import WorkflowDraftVariableRepository +from services.workflow.draft_variable_service import DraftVarLoader def _persist_offloaded_variable( @@ -67,11 +68,13 @@ class TestDraftVarLoaderSimple: def draft_var_loader(self, sqlite_engine: Engine): """Create DraftVarLoader instance for testing.""" return DraftVarLoader( - engine=sqlite_engine, + load_file=Mock(), + repository=WorkflowDraftVariableRepository( + sessions=sessionmaker(bind=sqlite_engine, expire_on_commit=False) + ), app_id="test-app-id", tenant_id="test-tenant-id", user_id="test-user-id", - fallback_variables=[], ) def test_load_offloaded_variable_object_type_unit(self, draft_var_loader): @@ -114,8 +117,8 @@ def test_load_offloaded_variable_object_type_unit(self, draft_var_loader): test_object = {"key1": "value1", "key2": 42} test_json_content = json.dumps(test_object, ensure_ascii=False, separators=(",", ":")) - with patch("services.workflow_draft_variable_service.storage") as mock_storage: - mock_storage.load.return_value = test_json_content.encode() + with patch.object(draft_var_loader, "_load_file") as load_file: + load_file.return_value = test_json_content.encode() # Execute the method selector_tuple, variable = draft_var_loader._load_offloaded_variable(draft_var) @@ -128,7 +131,7 @@ def test_load_offloaded_variable_object_type_unit(self, draft_var_loader): assert variable.value == test_object # Verify method calls - mock_storage.load.assert_called_once_with("storage/key/test.json") + load_file.assert_called_once_with("storage/key/test.json") def test_load_offloaded_variable_missing_variable_file_unit(self, draft_var_loader): """Test that assertion error is raised when variable_file is None.""" @@ -210,8 +213,8 @@ def test_load_offloaded_variable_array_type_unit(self, draft_var_loader): test_array = ["item1", 2, True] test_json_content = json.dumps(test_array) - with patch("services.workflow_draft_variable_service.storage") as mock_storage: - mock_storage.load.return_value = test_json_content.encode() + with patch.object(draft_var_loader, "_load_file") as load_file: + load_file.return_value = test_json_content.encode() # Execute the method selector_tuple, variable = draft_var_loader._load_offloaded_variable(draft_var) @@ -222,7 +225,7 @@ def test_load_offloaded_variable_array_type_unit(self, draft_var_loader): assert variable.description == "test array description" # Verify method calls - mock_storage.load.assert_called_once_with("storage/key/test_array.json") + load_file.assert_called_once_with("storage/key/test_array.json") def test_load_offloaded_variable_file_type_rebuilds_storage_backed_payload(self, draft_var_loader): upload_file = UploadFile( @@ -288,11 +291,11 @@ def test_load_offloaded_variable_file_type_rebuilds_storage_backed_payload(self, } with ( - patch("services.workflow_draft_variable_service.storage") as mock_storage, + patch.object(draft_var_loader, "_load_file") as load_file, patch("models.workflow._resolve_workflow_app_tenant_id", return_value="tenant-1"), patch("models.workflow.build_file_from_stored_mapping", return_value=rebuilt_file) as rebuild_file, ): - mock_storage.load.return_value = json.dumps(raw_file).encode() + load_file.return_value = json.dumps(raw_file).encode() selector_tuple, variable = draft_var_loader._load_offloaded_variable(draft_var) @@ -345,13 +348,13 @@ def test_load_variables_with_offloaded_variables_unit( offloaded_variable.selector = ["node2", "offloaded_var"] with ( - patch("services.workflow_draft_variable_service.StorageKeyLoader"), + patch("repositories.workflow.draft_variable_repository.StorageKeyLoader"), patch.object( draft_var_loader, "_load_offloaded_variable", return_value=(("node2", "offloaded_var"), offloaded_variable), ) as load_offloaded, - patch("services.workflow_draft_variable_service.ThreadPoolExecutor") as executor_cls, + patch("services.workflow.draft_variable_service.ThreadPoolExecutor") as executor_cls, ): executor = executor_cls.return_value.__enter__.return_value executor.map.side_effect = lambda function, values: [function(value) for value in values] @@ -392,8 +395,8 @@ def test_load_variables_all_offloaded_variables_unit( sqlite_session.commit() with ( - patch("services.workflow_draft_variable_service.StorageKeyLoader"), - patch("services.workflow_draft_variable_service.ThreadPoolExecutor") as executor_cls, + patch("repositories.workflow.draft_variable_repository.StorageKeyLoader"), + patch("services.workflow.draft_variable_service.ThreadPoolExecutor") as executor_cls, ): executor = executor_cls.return_value.__enter__.return_value executor.map.return_value = [ diff --git a/api/tests/unit_tests/services/workflow/test_generation_read_boundary.py b/api/tests/unit_tests/services/workflow/test_generation_read_boundary.py new file mode 100644 index 00000000000000..1da40fb58a5eab --- /dev/null +++ b/api/tests/unit_tests/services/workflow/test_generation_read_boundary.py @@ -0,0 +1,130 @@ +"""Generation must release request reads before waiting for a worker's response.""" + +import json +from unittest.mock import Mock + +import pytest +from flask import Flask +from sqlalchemy import inspect +from sqlalchemy.orm import Session + +from core.app.apps import base_app_queue_manager +from core.app.entities.app_invoke_entities import InvokeFrom +from core.ops.ops_trace_manager import TraceQueueManager +from models.enums import EndUserType +from models.model import App, AppMode, EndUser +from models.workflow import Workflow, WorkflowType +from services.app_generate_service import AppGenerateService +from services.workflow.execution.adapters.chatflow import app_generator as chatflow_module +from services.workflow.execution.adapters.workflow import app_generator as workflow_module +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables +from tests.unit_tests.services.test_app_generate_service import _DummyRateLimit + + +@pytest.mark.parametrize("mode", [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) +@pytest.mark.parametrize("fail_while_waiting", [False, True]) +def test_blocking_generation_detaches_request_inputs_before_waiting( + app: Flask, + sqlite_session: Session, + workflow_runtime: WorkflowRuntime, + workflow_variables: WorkflowExecutionVariables, + monkeypatch: pytest.MonkeyPatch, + mode: AppMode, + fail_while_waiting: bool, +) -> None: + application = App( + id="app", + tenant_id="tenant", + name="App", + mode=mode, + description="", + enable_site=False, + enable_api=True, + max_active_requests=None, + ) + user = EndUser( + id="user", + tenant_id="tenant", + app_id="app", + type=EndUserType.SERVICE_API, + session_id="external-user", + ) + workflow = Workflow.new( + tenant_id="tenant", + app_id="app", + type=WorkflowType.CHAT.value if mode == AppMode.ADVANCED_CHAT else WorkflowType.WORKFLOW.value, + version="1", + created_by="owner", + features="{}", + graph=json.dumps( + {"nodes": [{"id": "start", "data": {"type": "start", "title": "Start", "variables": []}}], "edges": []} + ), + environment_variables=[], + conversation_variables=[], + rag_pipeline_variables=[], + ) + application.workflow_id = workflow.id + sqlite_session.add_all([application, user, workflow]) + sqlite_session.commit() + sqlite_session.refresh(application) + assert sqlite_session.in_transaction() + + module = chatflow_module if mode == AppMode.ADVANCED_CHAT else workflow_module + generator = module.AdvancedChatAppGenerator if mode == AppMode.ADVANCED_CHAT else module.WorkflowAppGenerator + converter = ( + module.AdvancedChatAppGenerateResponseConverter + if mode == AppMode.ADVANCED_CHAT + else module.WorkflowAppGenerateResponseConverter + ) + monkeypatch.setattr("services.app_generate_service.RateLimit", _DummyRateLimit) + monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", lambda *_args, **_kwargs: None) + monkeypatch.setattr(module, "TraceQueueManager", Mock(return_value=Mock(spec=TraceQueueManager))) + monkeypatch.setattr(generator, "_generate_worker", staticmethod(lambda **_kwargs: None)) + monkeypatch.setattr(converter, "convert", lambda *, response, **_kwargs: response) + waits = [] + + def wait_for_response(_self, **_kwargs): + assert not sqlite_session.in_transaction() + for row in (application, user, workflow): + assert inspect(row).detached + assert application.name == "App" + assert workflow.graph_dict["nodes"][0]["id"] == "start" + assert not sqlite_session.in_transaction() + waits.append("response") + if fail_while_waiting: + raise ValueError("model response failed") + return {"answer": "done"} + + monkeypatch.setattr( + generator, + "_handle_advanced_chat_response" if mode == AppMode.ADVANCED_CHAT else "_handle_response", + wait_for_response, + ) + with app.app_context(): + if fail_while_waiting: + with pytest.raises(ValueError, match="model response failed"): + AppGenerateService.generate( + app_model=application, + user=user, + args={"query": "Hello", "inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=False, + session=sqlite_session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + else: + result = AppGenerateService.generate( + app_model=application, + user=user, + args={"query": "Hello", "inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=False, + session=sqlite_session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + assert result == {"answer": "done"} + assert waits == ["response"] + assert not sqlite_session.in_transaction() diff --git a/api/tests/unit_tests/services/workflow/test_human_input_persistence.py b/api/tests/unit_tests/services/workflow/test_human_input_persistence.py new file mode 100644 index 00000000000000..2f2995cf6f16d9 --- /dev/null +++ b/api/tests/unit_tests/services/workflow/test_human_input_persistence.py @@ -0,0 +1,192 @@ +"""Human Input nodes persist through the execution's injected database.""" + +import json +from datetime import timedelta + +import pytest +from sqlalchemy import create_engine, event, select +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import QueuePool + +from core.app.entities.app_invoke_entities import InvokeFrom +from core.db import session_factory +from core.workflow.system_variables import build_system_variables +from enums.human_input import HumanInputFormStatus, RecipientType +from extensions.application_services.workflow import build_workflow_execution_dependencies +from extensions.application_services.workflow_variables import build_workflow_variable_service +from graphon.node_events import PauseRequestedEvent +from graphon.runtime import GraphRuntimeState, VariablePool +from libs.datetime_utils import naive_utc_now +from models.base import TypeBase +from models.human_input import HumanInputDelivery, HumanInputForm, HumanInputFormRecipient +from models.human_input_entities import HumanInputNodeData, UserActionConfig +from services.workflow.execution.adapters.node_factory import DifyNodeFactory +from tasks.app_generate.workflow_execute_task import AppExecutionParams, _Account, _AppRunner +from tests.workflow_test_utils import build_test_graph_init_params + + +@pytest.fixture(params=[False, True], ids=["retain-after-commit", "expire-after-commit"]) +def form_databases(monkeypatch, request): + engines = [create_engine("sqlite://", poolclass=QueuePool) for _ in range(2)] + for engine in engines: + TypeBase.metadata.create_all( + engine, tables=[model.__table__ for model in (HumanInputForm, HumanInputDelivery, HumanInputFormRecipient)] + ) + injected, global_sessions = [sessionmaker(engine, expire_on_commit=request.param) for engine in engines] + monkeypatch.setattr(session_factory, "create_session", global_sessions) + try: + yield injected, global_sessions + finally: + for engine in engines: + engine.dispose() + + +def factory(runtime, invoke_from=InvokeFrom.DEBUGGER): + state = GraphRuntimeState( + variable_pool=VariablePool.from_bootstrap( + system_variables=build_system_variables(app_id="app", workflow_execution_id="run") + ), + start_at=0, + ) + return DifyNodeFactory( + graph_init_params=build_test_graph_init_params(invoke_from=invoke_from), + graph_runtime_state=state, + workflow_runtime=runtime, + ) + + +def node_config(): + return { + "id": "human-node", + "data": HumanInputNodeData( + title="Approval", + form_content="Approve this request?", + user_actions=[UserActionConfig(id="approve", title="Approve")], + ).model_dump(), + } + + +@pytest.mark.parametrize("invoke_from", [InvokeFrom.DEBUGGER, InvokeFrom.SERVICE_API]) +@pytest.mark.parametrize("rebind", [False, True]) +@pytest.mark.parametrize("entry", ["factory", "engine-worker"]) +def test_node_creates_and_resumes_form_in_injected_database(form_databases, invoke_from, rebind, entry): + sessions, global_sessions = form_databases + if entry == "engine-worker": + runner = _AppRunner( + sessions.kw["bind"], + AppExecutionParams( + app_id="app", + workflow_id="workflow", + tenant_id="tenant", + user=_Account(user_id="user"), + args={}, + invoke_from=invoke_from, + ), + variables=build_workflow_variable_service(database_client=sessions), + ) + assert runner._session_factory.kw["expire_on_commit"] is True + runtime = runner._runtime + else: + runtime = build_workflow_execution_dependencies(sessions) + node_factory = factory(runtime, invoke_from) + if rebind: + node_factory = node_factory.with_runtime_state(node_factory.graph_runtime_state) + node = node_factory.create_node(node_config()) + node.bind_execution_id("human-execution") + events = list(node._run()) + assert any(isinstance(event, PauseRequestedEvent) for event in events) + assert sessions.kw["bind"].pool.checkedout() == 0 + assert global_sessions.kw["bind"].pool.checkedout() == 0 + + with sessions.begin() as session: + form = session.scalars(select(HumanInputForm)).one() + assert (form.tenant_id, form.app_id, form.workflow_run_id, form.node_id) == ( + "tenant", + "app", + "run", + "human-node", + ) + form_id = form.id + recipients = session.scalars(select(HumanInputFormRecipient)).all() + if invoke_from == InvokeFrom.DEBUGGER: + console = next(recipient for recipient in recipients if recipient.recipient_type == RecipientType.CONSOLE) + assert json.loads(console.recipient_payload)["account_id"] == "user" + form.selected_action_id = "approve" + form.submitted_data = "{}" + form.submitted_at = naive_utc_now() + form.expiration_time = naive_utc_now() + timedelta(hours=1) + form.status = HumanInputFormStatus.SUBMITTED + + completed = list(node._run()) + assert not any(isinstance(event, PauseRequestedEvent) for event in completed) + assert completed[-1].node_run_result.outputs["__action_id"].value == "approve" + with sessions() as session: + assert session.scalars(select(HumanInputForm)).one().id == form_id + with global_sessions() as session: + assert session.scalar(select(HumanInputForm)) is None + assert sessions.kw["bind"].pool.checkedout() == 0 + + +def test_human_input_node_requires_persistence_dependency(): + with pytest.raises(ValueError, match="execution dependencies are required for Human Input"): + factory(None).create_node(node_config()) + + +def test_create_form_commit_failure_does_not_return_a_pause_or_persist_form(form_databases): + sessions, _ = form_databases + node = factory(build_workflow_execution_dependencies(sessions)).create_node(node_config()) + node.bind_execution_id("human-execution") + + def fail_commit(_session): + raise RuntimeError("commit failed") + + event.listen(sessions, "before_commit", fail_commit) + try: + with pytest.raises(RuntimeError, match="commit failed"): + list(node._run()) + finally: + event.remove(sessions, "before_commit", fail_commit) + with sessions() as session: + assert session.scalar(select(HumanInputForm)) is None + assert session.scalar(select(HumanInputFormRecipient)) is None + assert session.scalar(select(HumanInputDelivery)) is None + + +def test_created_form_timeout_reaches_event_stream_in_same_database(form_databases): + from dataclasses import dataclass + + from graphon.graph import Graph + from graphon.graph_events import NodeRunHumanInputFormTimeoutEvent, NodeRunSucceededEvent + from services.workflow.execution.adapters.workflow_entry import iter_dify_graph_engine_events + + sessions, global_sessions = form_databases + runtime = build_workflow_execution_dependencies(sessions) + node = factory(runtime).create_node(node_config()) + node.bind_execution_id("timed-out-execution") + assert any(isinstance(event, PauseRequestedEvent) for event in node._run()) + deadline = naive_utc_now() - timedelta(minutes=1) + with sessions.begin() as session: + form = session.scalars(select(HumanInputForm)).one() + form.expiration_time = deadline + form.status = HumanInputFormStatus.TIMEOUT + + @dataclass + class NodeEvents: + graph: Graph + graph_runtime_state: GraphRuntimeState + + def run(self): + yield from node.run() + + events = list( + iter_dify_graph_engine_events( + NodeEvents(Graph(root_node=node), node.graph_runtime_state), human_form_reader=runtime.human_form_reader + ) + ) + timeout = next(event for event in events if isinstance(event, NodeRunHumanInputFormTimeoutEvent)) + assert timeout.id == "timed-out-execution" + assert timeout.expiration_time == deadline + assert isinstance(events[-1], NodeRunSucceededEvent) + with global_sessions() as session: + assert session.scalar(select(HumanInputForm)) is None + assert sessions.kw["bind"].pool.checkedout() == 0 diff --git a/api/tests/unit_tests/services/workflow/test_node_output_inspector_service.py b/api/tests/unit_tests/services/workflow/test_node_output_inspector_service.py index 285acab49f5684..e6ba63245f4f48 100644 --- a/api/tests/unit_tests/services/workflow/test_node_output_inspector_service.py +++ b/api/tests/unit_tests/services/workflow/test_node_output_inspector_service.py @@ -16,17 +16,9 @@ from sqlalchemy.orm import Session from core.workflow.file_reference import build_file_reference -from core.workflow.nodes.agent_v2.binding_resolver import WorkflowAgentBindingBundle +from enums.agent import WorkflowAgentBindingType from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus -from models.agent import ( - Agent, - AgentConfigSnapshot, - AgentScope, - AgentSource, - AgentStatus, - WorkflowAgentBindingType, - WorkflowAgentNodeBinding, -) +from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource, AgentStatus, WorkflowAgentNodeBinding from models.agent_config_entities import ( AgentSoulConfig, DeclaredArrayItem, @@ -34,6 +26,7 @@ DeclaredOutputType, WorkflowNodeJobConfig, ) +from models.agent_runtime_contracts import WorkflowAgentBindingBundle from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom from models.model import App from models.workflow import ( diff --git a/api/tests/unit_tests/services/workflow/test_runtime_gateway.py b/api/tests/unit_tests/services/workflow/test_runtime_gateway.py new file mode 100644 index 00000000000000..e54e1ef2982105 --- /dev/null +++ b/api/tests/unit_tests/services/workflow/test_runtime_gateway.py @@ -0,0 +1,2129 @@ +"""Runtime adaptation releases read transactions before external execution preparation.""" + +import json +from collections.abc import Callable, Generator, Iterable +from typing import Self, cast, override +from unittest.mock import Mock + +import pytest +from flask import Flask +from sqlalchemy import event, select +from sqlalchemy.orm import Session, sessionmaker + +from configs import dify_config +from core.app.entities.app_invoke_entities import WorkflowAppGenerateEntity +from core.app.file_access import DatabaseFileAccessController +from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository +from core.trigger.debug.event_selectors import TriggerDebugEvent +from extensions.application_services.workflow import build_workflow_execution_dependencies +from extensions.ext_storage import storage +from fields.workflow_run_fields import WorkflowRunNodeExecutionResponse +from graphon.entities import WorkflowNodeExecution +from graphon.file import File +from graphon.graph_engine.manager import GraphEngineManager +from machinery.context import RequestContext +from models.account import TenantAccountJoin, TenantAccountRole +from models.enums import CreatorUserRole, ExecutionOffLoadType +from models.model import AppMode +from models.workflow import ( + Workflow, + WorkflowDraftVariable, + WorkflowNodeExecutionModel, + WorkflowNodeExecutionOffload, + WorkflowNodeExecutionStatus, +) +from repositories.agent.retirement_repository import WorkflowAgentRetirementRepository +from repositories.factory import DifyAPIRepositoryFactory +from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository +from repositories.workflow.definition_repository import WorkflowDefinitionRepository +from repositories.workflow.draft_variable_repository import WorkflowDraftVariableRepository +from repositories.workflow.node_execution_repository import WorkflowNodeExecutionRepository +from services import workflow_service as workflow_module +from services.agent.workflow_publish_service import WorkflowAgentPublishService +from services.app_generate_service import AppGenerateService +from services.file_service import FileService +from services.workflow import runtime_gateway as module +from services.workflow.console_service import ( + ConsoleWorkflowService, + WorkflowAccess, + WorkflowAppLookup, + WorkflowConversion, + WorkflowDefinitionLifecycle, + WorkflowDefinitions, + WorkflowPresence, +) +from services.workflow.draft_service import WorkflowDraftService +from services.workflow.runtime_gateway import WorkflowRuntimeGateway +from services.workflow.variable_contracts import WorkflowExecutionVariables +from services.workflow.variable_service import WorkflowVariableService +from services.workflow_service import WorkflowService +from tests.unit_tests.model_factories import make_account, make_app, make_tenant, make_workflow +from tests.unit_tests.workflow_execution import debug_lease, set_debug_deadline + +CONTEXT = RequestContext("request", None, "account-1", "tenant-1") + + +class RecordingSession(Session): + closed = False + + @override + def close(self) -> None: + self.closed = True + super().close() + + +@pytest.fixture(name="runtime") +def runtime_gateway( + sqlite_session_factory: sessionmaker[Session], *, workflow_variables: WorkflowExecutionVariables +) -> tuple[WorkflowRuntimeGateway, list[RecordingSession]]: + with sqlite_session_factory.begin() as session: + session.add_all( + [ + make_app(mode=AppMode.WORKFLOW), + make_account(), + make_tenant(), + make_workflow(), + TenantAccountJoin(tenant_id="tenant-1", account_id="account-1", role=TenantAccountRole.OWNER), + ] + ) + sessions: list[RecordingSession] = [] + factory = cast( + sessionmaker[Session], + sessionmaker(bind=sqlite_session_factory.kw["bind"], class_=RecordingSession, expire_on_commit=False), + ) + + def opened(session: Session, _transaction: object, _connection: object) -> None: + assert isinstance(session, RecordingSession) + sessions.append(session) + + event.listen(factory, "after_begin", opened) + workflows = WorkflowService(session_maker=factory, runtime=build_workflow_execution_dependencies(factory)) + + def generate(**_kwargs: object) -> Generator[str, None, None]: + assert sessions + assert all(session.closed for session in sessions) + + def events() -> Generator[str, None, None]: + assert all(session.closed for session in sessions) + yield "data: ready\n\n" + + return events() + + generator = Mock(spec=AppGenerateService) + generator.generate_workflow_stream.side_effect = generate + gateway = WorkflowRuntimeGateway( + runtime=build_workflow_execution_dependencies(factory), + session_factory=factory, + workflows=workflows, + definitions=WorkflowDefinitionRepository(session_factory=factory), + reservations=WorkflowDebugReservationRepository(factory), + executions=WorkflowNodeExecutionRepository(factory), + generator=generator, + graph_engine=Mock(spec=GraphEngineManager), + file_access=DatabaseFileAccessController(), + variables=workflow_variables, + ) + return gateway, sessions + + +def test_generate_closes_session_before_stream_is_consumed( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], +) -> None: + gateway, sessions = runtime + stream = gateway.generate(CONTEXT, "app-1", {}, root_node_id=None, workflow=None) + assert all(session.closed for session in sessions) + assert list(stream) == ["data: ready\n\n"] + + +@pytest.mark.parametrize("rejection", ["expired", "claimed", "paused", "tenant", "app", "workflow", "mode"]) +def test_rejected_debug_handoff_only_notifies_owned_expired_stream( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + rejection: str, +) -> None: + from datetime import timedelta + + from core.app.entities.app_invoke_entities import InvokeFrom + from graphon.enums import WorkflowExecutionStatus + from libs.datetime_utils import naive_utc_now + from models.workflow import WorkflowRun + from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository + from services.agent.retirement_service import WorkflowAgentRetirementService + from tasks.app_generate import workflow_execute_task as task_module + + gateway, sessions = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + app, actor, workflow = gateway._definitions.debug_context(CONTEXT, "app-1", workflow_id=None, snapshot=snapshot) + params = task_module.AppExecutionParams.new( + app_model=app, + workflow=workflow, + user=actor, + args={}, + invoke_from=InvokeFrom.DEBUGGER, + workflow_run_id=snapshot.execution_id, + workflow_snapshot=snapshot, + ) + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + reservation = debug_lease(session, snapshot.execution_id) + assert run is not None + assert reservation is not None + if rejection in ("claimed", "paused"): + WorkflowDebugReservationRepository.claim(session, run, naive_utc_now()) + if rejection == "paused": + run.status = WorkflowExecutionStatus.PAUSED + set_debug_deadline(session, snapshot.execution_id, naive_utc_now() - timedelta(seconds=30)) + if rejection in ("tenant", "app", "workflow"): + params = params.model_copy(update={f"{rejection}_id": "wrong-owner"}) + elif rejection == "mode": + params = params.model_copy(update={"invoke_from": InvokeFrom.SERVICE_API}) + topic = Mock() + + def publish(_payload: bytes) -> None: + assert all(session.closed and not session.in_transaction() for session in sessions) + + topic.publish.side_effect = publish + get_topic = Mock(return_value=topic) + monkeypatch.setattr(task_module.WorkflowEventStream, "get_response_topic", get_topic) + finish = Mock(wraps=WorkflowAgentRetirementService.finish_execution) + monkeypatch.setattr(WorkflowAgentRetirementService, "finish_execution", finish) + runner = task_module._AppRunner(gateway._sessions, params, variables=gateway._variables) + engine = Mock() + monkeypatch.setattr(runner, "_run_app", engine) + with pytest.raises(ValueError): + runner.run() + engine.assert_not_called() + finish.assert_not_called() + if rejection == "expired": + payloads = [json.loads(call.args[0]) for call in topic.publish.call_args_list] + assert [payload["event"] for payload in payloads] == ["workflow_started", "workflow_finished"] + assert payloads[-1]["data"]["status"] == "failed" + assert payloads[-1]["workflow_run_id"] == snapshot.execution_id + assert "expired before worker startup" in payloads[-1]["data"]["error"] + else: + get_topic.assert_not_called() + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + assert run.status == ( + WorkflowExecutionStatus.PAUSED if rejection == "paused" else WorkflowExecutionStatus.RUNNING + ) + + +def test_claimed_worker_is_not_cancelled_by_stream_cleanup( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + from datetime import timedelta + + from graphon.enums import WorkflowExecutionStatus + from libs.datetime_utils import naive_utc_now + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + from repositories.agent.runtime_repository import WorkflowAgentExecutionRepository + from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository + from services.agent.retirement_service import WorkflowAgentRetirementService + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + WorkflowDebugReservationRepository.claim(session, run, naive_utc_now()) + gateway._cancel_trigger_debug(snapshot, CONTEXT.account_id) + # Even an expiry candidate selected before the claim must be rechecked + # under the run lock; a healthy running lease must remain owned. + assert ( + WorkflowDebugReservationRepository(sqlite_session_factory).expire( + snapshot.execution_id, naive_utc_now() + timedelta(seconds=1) + ) + is None + ) + with sqlite_session_factory() as session: + assert session.get(WorkflowRun, snapshot.execution_id) is not None + assert WorkflowAgentExecutionRepository.retained_execution_agent_ids(session, "tenant-1", ["agent-1"]) == { + "agent-1" + } + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + monkeypatch.setattr("core.db.session_factory._session_maker", sessionmaker()) + collector = Mock() + monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", collector) + WorkflowAgentRetirementService.finish_execution( + sessions=sqlite_session_factory, + tenant_id="tenant-1", + app_id="app-1", + workflow_id=snapshot.id, + execution_id=snapshot.execution_id, + account_id=CONTEXT.account_id, + status=WorkflowExecutionStatus.STOPPED, + ) + with sqlite_session_factory() as session: + agent = session.get(Agent, "agent-1") + assert agent is not None + assert agent.status == AgentStatus.ARCHIVED + collector.assert_called_once() + + +@pytest.mark.parametrize("storage_backend", ["sqlalchemy", "celery"]) +@pytest.mark.parametrize("stream_raises", [False, True]) +@pytest.mark.parametrize("publish_fails", [False, True]) +def test_paused_trigger_finalizes_once_despite_delayed_start_write( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + storage_backend: str, + stream_raises: bool, + publish_fails: bool, +) -> None: + from contextlib import contextmanager + + from core.app.entities.app_invoke_entities import InvokeFrom + from core.repositories.celery_workflow_execution_repository import CeleryWorkflowExecutionRepository + from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository + from graphon.entities import WorkflowExecution + from graphon.enums import WorkflowExecutionStatus, WorkflowType + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.enums import WorkflowRunTriggeredFrom + from models.workflow import WorkflowRun + from repositories.agent.runtime_repository import WorkflowAgentExecutionRepository + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks import workflow_execution_tasks as storage_module + from tasks.app_generate import workflow_execute_task as task_module + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + app, actor, workflow = gateway._definitions.debug_context(CONTEXT, "app-1", workflow_id=None, snapshot=snapshot) + params = task_module.AppExecutionParams.new( + app_model=app, + workflow=workflow, + user=actor, + args={}, + invoke_from=InvokeFrom.DEBUGGER, + workflow_run_id=snapshot.execution_id, + workflow_snapshot=snapshot, + ) + started = WorkflowExecution( + id_=snapshot.execution_id, + workflow_id=snapshot.id, + workflow_type=WorkflowType.WORKFLOW, + workflow_version="draft", + graph=snapshot.graph_dict, + inputs={}, + outputs={}, + status=WorkflowExecutionStatus.RUNNING, + error_message="", + total_tokens=0, + total_steps=0, + exceptions_count=0, + started_at=snapshot.created_at, + finished_at=None, + ) + paused = started.model_copy(update={"status": WorkflowExecutionStatus.PAUSED}) + repository_type = ( + CeleryWorkflowExecutionRepository if storage_backend == "celery" else SQLAlchemyWorkflowExecutionRepository + ) + writer = repository_type( + session_factory=sqlite_session_factory, + tenant_id="tenant-1", + app_id="app-1", + user=actor, + triggered_from=WorkflowRunTriggeredFrom.DEBUGGING, + ) + monkeypatch.setattr("core.db.session_factory.session_factory.create_session", gateway._sessions) + queued = Mock() + monkeypatch.setattr(storage_module.save_workflow_execution_task, "delay", queued) + if storage_backend == "celery": + writer.save(started) + writer.save(paused) + assert queued.call_count == 2 + + def deliver(index: int) -> None: + if storage_backend == "celery": + storage_module.save_workflow_execution_task(**queued.call_args_list[index].kwargs) + else: + writer.save((started, paused)[index]) + + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + collector = Mock() + monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", collector) + from repositories.workflow.execution_write_repository import WorkflowExecutionWriteRepository + + statuses: list[WorkflowExecutionStatus | None] = [] + original_finish = WorkflowExecutionWriteRepository.finish + + def record_finish( + self: WorkflowExecutionWriteRepository, + *, + tenant_id: str, + app_id: str, + workflow_id: str, + execution_id: str, + status: WorkflowExecutionStatus | None = None, + cancel: bool = False, + ) -> WorkflowExecutionStatus | None: + statuses.append(status) + return original_finish( + self, + tenant_id=tenant_id, + app_id=app_id, + workflow_id=workflow_id, + execution_id=execution_id, + status=status, + cancel=cancel, + ) + + monkeypatch.setattr(WorkflowExecutionWriteRepository, "finish", record_finish) + topic = Mock() + if publish_fails: + topic.publish.side_effect = RuntimeError("pause publish failed") + monkeypatch.setattr(WorkflowEventStream, "get_response_topic", lambda *_args: topic) + + def events(**_kwargs: object) -> Generator[dict[str, object], None, None]: + yield {"event": "workflow_paused", "data": {"status": "paused"}} + if stream_raises: + raise RuntimeError("stream failed after pause") + + @contextmanager + def execution_context(_user: object) -> Generator[None, None, None]: + try: + yield + finally: + # The publisher has finalized PAUSED, but the outer run() has not + # exited yet. Deliver the older asynchronous start write here. + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + assert run.status == WorkflowExecutionStatus.PAUSED + deliver(0) + + runner = task_module._AppRunner(gateway._sessions, params, variables=gateway._variables) + monkeypatch.setattr(runner, "_run_app", events) + monkeypatch.setattr(runner, "_setup_flask_context", execution_context) + if publish_fails: + with pytest.raises(RuntimeError, match="pause publish failed"): + runner.run() + assert [json.loads(call.args[0])["event"] for call in topic.publish.call_args_list] == [ + "workflow_paused", + "workflow_paused", + ] + elif stream_raises: + with pytest.raises(RuntimeError, match="stream failed after pause"): + runner.run() + else: + runner.run() + + assert statuses == [WorkflowExecutionStatus.PAUSED] + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + assert run.status == WorkflowExecutionStatus.PAUSED + assert WorkflowAgentExecutionRepository.retained_execution_agent_ids(session, "tenant-1", ["agent-1"]) == { + "agent-1" + } + + # Finally deliver the asynchronous pause write. The outer runner must not + # have poisoned the execution with FAILED or retired its removed Agent. + deliver(1) + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + assert run is not None + assert run.status == WorkflowExecutionStatus.PAUSED + assert agent is not None + assert agent.status == AgentStatus.ACTIVE + assert WorkflowAgentExecutionRepository.retained_execution_agent_ids(session, "tenant-1", ["agent-1"]) == { + "agent-1" + } + collector.assert_not_called() + + +@pytest.mark.parametrize("stream_raises", [False, True]) +def test_pause_publication_can_resume_before_old_worker_exits( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + stream_raises: bool, +) -> None: + from contextlib import nullcontext + + from core.app.entities.app_invoke_entities import InvokeFrom + from graphon.enums import WorkflowExecutionStatus + from libs.datetime_utils import naive_utc_now + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowPause, WorkflowRun + from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository + from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository + from repositories.workflow.execution_write_repository import save_workflow_run + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate import workflow_execute_task as task_module + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + app, actor, workflow = gateway._definitions.debug_context(CONTEXT, "app-1", workflow_id=None, snapshot=snapshot) + params = task_module.AppExecutionParams.new( + app_model=app, + workflow=workflow, + user=actor, + args={}, + invoke_from=InvokeFrom.DEBUGGER, + workflow_run_id=snapshot.execution_id, + workflow_snapshot=snapshot, + ) + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + collector = Mock() + monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", collector) + runs = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory) + + def events(**_kwargs: object) -> Generator[dict[str, object], None, None]: + # Pause-state persistence precedes the outgoing engine event. + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + run.status = WorkflowExecutionStatus.PAUSED + session.add(WorkflowPause(workflow_id=workflow.id, workflow_run_id=run.id, state_object_key="pause")) + yield {"event": "workflow_paused", "data": {"status": "paused"}} + if stream_raises: + raise RuntimeError("old stream failed after resume") + + def resume_on_publication(_payload: bytes) -> None: + pause = runs.get_workflow_pause(snapshot.execution_id) + assert pause is not None + runs.resume_workflow_pause(snapshot.execution_id, pause) + + topic = Mock() + topic.publish.side_effect = resume_on_publication + monkeypatch.setattr(WorkflowEventStream, "get_response_topic", lambda *_args: topic) + runner = task_module._AppRunner(gateway._sessions, params, variables=gateway._variables) + monkeypatch.setattr(runner, "_run_app", events) + monkeypatch.setattr(runner, "_setup_flask_context", lambda _user: nullcontext()) + if stream_raises: + with pytest.raises(RuntimeError, match="old stream failed after resume"): + runner.run() + else: + runner.run() + topic.publish.assert_called_once() + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + assert run is not None + assert run.status == WorkflowExecutionStatus.RUNNING + assert agent is not None + assert agent.status == AgentStatus.ACTIVE + session.expunge(run) + with sqlite_session_factory.begin() as session: + assert save_workflow_run(session, run) + collector.assert_not_called() + # A subsequently lost resumed worker must still be discoverable by expiry. + from datetime import timedelta + + assert snapshot.execution_id in [ + item.execution_id + for item in WorkflowDebugReservationRepository(sqlite_session_factory).pending_batch( + naive_utc_now() + timedelta(days=1), limit=100 + ) + ] + + +@pytest.mark.parametrize("failure_stage", ["prepare", "publish-setup"]) +def test_trigger_runner_finalizes_failure_before_stream_cleanup( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + failure_stage: str, +) -> None: + from core.app.entities.app_invoke_entities import InvokeFrom + from graphon.enums import WorkflowExecutionStatus + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + from services.agent.retirement_service import WorkflowAgentRetirementService + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate import workflow_execute_task as task_module + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + app, actor, workflow = gateway._definitions.debug_context(CONTEXT, "app-1", workflow_id=None, snapshot=snapshot) + params = task_module.AppExecutionParams.new( + app_model=app, + workflow=workflow, + user=actor, + args={}, + invoke_from=InvokeFrom.DEBUGGER, + workflow_run_id=snapshot.execution_id, + workflow_snapshot=snapshot, + ) + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + monkeypatch.setattr("core.db.session_factory.session_factory.create_session", gateway._sessions) + collector = Mock() + monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", collector) + finish = Mock(wraps=WorkflowAgentRetirementService.finish_execution) + monkeypatch.setattr(WorkflowAgentRetirementService, "finish_execution", finish) + topic = Mock() + if failure_stage == "publish-setup": + topic.side_effect = RuntimeError("stream setup failed") + monkeypatch.setattr(WorkflowEventStream, "get_response_topic", topic) + generate = Mock(return_value=(event for event in [])) + if failure_stage == "prepare": + generate.side_effect = RuntimeError("stream setup failed") + runner = task_module._AppRunner(gateway._sessions, params, variables=gateway._variables) + monkeypatch.setattr(runner, "_run_app", generate) + + with Flask(__name__).app_context(), pytest.raises(RuntimeError, match="stream setup failed"): + runner.run() + + finish.assert_called_once() + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + assert run is not None + assert agent is not None + assert run.status == WorkflowExecutionStatus.FAILED + assert agent.status == AgentStatus.ARCHIVED + collector.assert_called_once() + + +@pytest.mark.parametrize("mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]) +@pytest.mark.parametrize("failure", [None, "redis"]) +def test_real_stream_preparation_releases_reads_before_external_io( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + mode: AppMode, + failure: str | None, +) -> None: + from core.app.features.rate_limiting import rate_limit as limiter_module + from enums import DeploymentEdition + from models.model import App + from services.quota_service import QuotaCharge, QuotaService, unlimited + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate import workflow_execute_task as task_module + + gateway, sessions = runtime + gateway._generator = AppGenerateService + with sqlite_session_factory.begin() as session: + app = session.get(App, "app-1") + assert app is not None + app.mode = mode + app.max_active_requests = 2 + monkeypatch.setattr(dify_config, "DEPLOYMENT_EDITION", DeploymentEdition.CLOUD) + monkeypatch.setattr(dify_config, "PUBSUB_REDIS_CHANNEL_TYPE", "streams") + monkeypatch.setattr(limiter_module.RateLimit, "_instance_dict", {}) + observed: list[str] = [] + + def external(stage: str) -> None: + assert sessions + assert all(session.closed and not session.in_transaction() for session in sessions) + observed.append(stage) + if failure == stage: + raise RuntimeError("external failure") + + def reserve(*_args: object) -> QuotaCharge: + external("quota") + return unlimited() + + def redis_write(*_args: object) -> None: + external("redis") + + monkeypatch.setattr(QuotaService, "reserve", reserve) + monkeypatch.setattr(limiter_module.redis_client, "setex", redis_write) + monkeypatch.setattr(limiter_module.redis_client, "exists", lambda _key: False) + monkeypatch.setattr(limiter_module.redis_client, "hlen", lambda _key: 0) + monkeypatch.setattr(limiter_module.redis_client, "hset", redis_write) + monkeypatch.setattr(limiter_module.redis_client, "hdel", redis_write) + queued: list[str] = [] + + def enqueue(payload: str) -> None: + external("enqueue") + queued.append(payload) + + def retrieve_events(*_args: object, on_subscribe: Callable[[], None]) -> Generator[str, None, None]: + # This assertion runs during preparation, before the returned stream is consumed. + external("subscription-prepare") + + def events() -> Generator[str, None, None]: + on_subscribe() + yield from () + + return events() + + monkeypatch.setattr(task_module.workflow_based_app_execution_task, "delay", enqueue) + monkeypatch.setattr(WorkflowEventStream, "retrieve_events", retrieve_events) + if failure: + with Flask(__name__).app_context(), pytest.raises(RuntimeError, match="external failure"): + gateway.generate(CONTEXT, "app-1", {}, root_node_id=None, workflow=None) + assert queued == [] + assert observed == ["quota", "redis"] + else: + with Flask(__name__).app_context(): + stream = gateway.generate(CONTEXT, "app-1", {}, root_node_id=None, workflow=None) + assert observed[0] == "quota" + assert "redis" in observed + assert observed[-1] == "subscription-prepare" + assert not queued + assert list(stream) == [] + assert len(queued) == 1 + payload = task_module.AppExecutionParams.model_validate_json(queued[0]) + assert (payload.tenant_id, payload.app_id, payload.app_mode) == ("tenant-1", "app-1", mode) + assert all(session.closed and not session.in_transaction() for session in sessions) + + +def test_trigger_poll_runs_outside_database_session( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], monkeypatch: pytest.MonkeyPatch +) -> None: + gateway, sessions = runtime + + class Poller: + def poll(self) -> TriggerDebugEvent: + assert sessions + assert all(session.closed for session in sessions) + return TriggerDebugEvent(node_id="node", workflow_args={"inputs": {}}) + + monkeypatch.setattr(module, "create_event_poller", lambda **_kwargs: Poller()) + result = gateway.poll_trigger(CONTEXT, "app-1", ["node"], single_node=False, select_all=False) + assert result is not None + assert result.node_id == "node" + + +def test_schedule_single_node_does_not_poll( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, _ = runtime + with sqlite_session_factory.begin() as session: + workflow = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert workflow is not None + workflow.graph = json.dumps({"nodes": [{"id": "timer", "data": {"type": "trigger-schedule"}}], "edges": []}) + + def unexpected(**_kwargs: object) -> None: + pytest.fail("schedule single-step must not wait for an event") + + monkeypatch.setattr(module, "create_event_poller", unexpected) + result = gateway.poll_trigger(CONTEXT, "app-1", ["timer"], single_node=True, select_all=False) + assert result is not None + assert result.workflow_args == {} + + +def test_node_response_materializes_actor_extras_and_offload_flags( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], monkeypatch: pytest.MonkeyPatch +) -> None: + gateway, sessions = runtime + execution = WorkflowNodeExecutionModel( + id="execution", + tenant_id="tenant-1", + app_id="app-1", + node_id="node", + node_type="start", + created_by="account-1", + created_by_role=CreatorUserRole.ACCOUNT, + status=WorkflowNodeExecutionStatus.SUCCEEDED, + inputs='{"query":"value"}', + outputs="{}", + execution_metadata=None, + offload_data=[], + ) + monkeypatch.setattr(gateway._workflows, "run_draft_workflow_node", lambda **_kwargs: execution) + + def extras(_execution: WorkflowNodeExecutionModel, *, tool_providers) -> dict[str, object]: + assert tool_providers is gateway._runtime.tool_providers + assert all(session.closed for session in sessions) + return {} + + monkeypatch.setattr("services.workflow.runtime_gateway.node_execution_extras", extras) + result = gateway.run_node(CONTEXT, "app-1", "node", {"inputs": {}}, include_details=True, workflow=None) + assert all(session.closed for session in sessions) + response = WorkflowRunNodeExecutionResponse.model_validate(result).model_dump(mode="json") + assert response["inputs"] == {"query": "value"} + assert response["created_by_account"]["id"] == "account-1" + assert response["extras"] == {} + assert response["inputs_truncated"] is False + raw = gateway.run_node(CONTEXT, "app-1", "node", {"inputs": {}}, include_details=False, workflow=None) + assert raw["inputs"] == '{"query":"value"}' + assert raw["offload_data"] == [] + assert "created_by_account" not in raw + + +def test_trigger_response_preserves_all_offload_types( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], +) -> None: + gateway, sessions = runtime + records = [ + WorkflowNodeExecutionOffload( + tenant_id="tenant-1", + app_id="app-1", + node_execution_id="execution", + type_=kind, + file_id=f"file-{kind.value}", + ) + for kind in ExecutionOffLoadType + ] + with sqlite_session_factory.begin() as session: + session.add_all(records) + # Query and detach actual ORM rows so column names and attribute names differ + # exactly as in the runtime, without resolving file relationships. + with sqlite_session_factory() as session: + offloads = list(session.scalars(select(WorkflowNodeExecutionOffload))) + execution = WorkflowNodeExecutionModel(id="execution", offload_data=offloads) + raw = gateway._execution(execution, include_details=False) + assert {item["type_"] for item in raw["offload_data"]} == {"inputs", "outputs", "process_data"} + assert raw["offload_data"] == [ + { + "id": item.id, + "tenant_id": "tenant-1", + "app_id": "app-1", + "node_execution_id": "execution", + "type_": item.type_.value, + "file_id": item.file_id, + "created_at": item.created_at, + } + for item in offloads + ] + assert all(session.closed for session in sessions) + + +def test_file_config_and_access_controller_are_reused( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], monkeypatch: pytest.MonkeyPatch +) -> None: + gateway, _ = runtime + workflow = make_workflow() + monkeypatch.setattr(module.FileUploadConfigManager, "convert", lambda *_args, **_kwargs: None) + assert gateway._files(workflow, [{"id": "file"}]) == [] + config = object() + monkeypatch.setattr(module.FileUploadConfigManager, "convert", lambda *_args, **_kwargs: config) + received = [] + + def build(**kwargs: object) -> list[File]: + received.append(kwargs) + return [] + + monkeypatch.setattr(module.file_factory, "build_from_mappings", build) + gateway._files(workflow, [{"id": "file"}]) + assert received == [ + { + "mappings": [{"id": "file"}], + "tenant_id": "tenant-1", + "config": config, + "access_controller": gateway._file_access, + "sessions": gateway._sessions, + } + ] + + +@pytest.mark.parametrize("concurrent_change", ["configuration", "delete-node"]) +def test_trigger_consumption_and_real_execution_share_detached_revision( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + concurrent_change: str, +) -> None: + gateway, sessions = runtime + + graph: dict[str, object] = { + "nodes": [ + { + "id": "trigger", + "data": { + "type": "trigger-plugin", + "title": "Subscription A", + "plugin_id": "test/trigger", + "provider_id": "test/trigger/provider", + "subscription_id": "subscription-A", + "event_name": "event-A", + "plugin_unique_identifier": "test/trigger:1", + }, + } + ], + "edges": [], + } + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + draft.graph = json.dumps(graph) + + class ForbiddenDatabase: + @property + def engine(self) -> None: + pytest.fail("single-step execution must use the injected session factory") + + monkeypatch.setattr(workflow_module, "db", ForbiddenDatabase()) + + class Poller: + def poll(self) -> TriggerDebugEvent: + assert sessions + assert all(session.closed for session in sessions) + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + changed = json.loads(draft.graph) + if concurrent_change == "delete-node": + changed["nodes"] = [] + else: + changed["nodes"][0]["data"].update( + subscription_id="subscription-B", event_name="event-B", title="Subscription B" + ) + draft.graph = json.dumps(changed) + return TriggerDebugEvent(node_id="trigger", workflow_args={"inputs": {"payload": "from-A"}}) + + def poller(*, draft_workflow: Workflow, **_kwargs: object) -> Poller: + assert draft_workflow.graph_dict["nodes"][0]["data"]["subscription_id"] == "subscription-A" + return Poller() + + monkeypatch.setattr(module, "create_event_poller", poller) + service = ConsoleWorkflowService( + conversion=Mock(spec=WorkflowConversion), + agent_services=WorkflowAgentPublishService, + drafts=Mock(spec=WorkflowDraftService), + definitions=Mock(spec=WorkflowDefinitions), + lifecycle=Mock(spec=WorkflowDefinitionLifecycle), + runtime=gateway, + apps=Mock(spec=WorkflowAppLookup), + presence=Mock(spec=WorkflowPresence), + access=Mock(spec=WorkflowAccess), + ) + result = service.trigger(CONTEXT, "app-1", ["trigger"], single_node=True, select_all=False) + assert isinstance(result, dict) + assert result["status"] == WorkflowNodeExecutionStatus.SUCCEEDED + assert result["title"] == "Subscription A" + assert json.loads(result["outputs"])["payload"] == "from-A" + assert json.loads(result["execution_metadata"])["trigger_info"]["event_name"] == "event-A" + assert all(session.closed for session in sessions) + with sqlite_session_factory() as session: + execution = session.get(WorkflowNodeExecutionModel, result["id"]) + assert execution is not None + assert execution.title == "Subscription A" + variables = session.scalars( + select(WorkflowDraftVariable).where(WorkflowDraftVariable.node_id == "trigger") + ).all() + assert any(variable.name == "payload" and variable.get_value().value == "from-A" for variable in variables) + + +def test_single_node_reads_offloaded_outputs_after_repository_session_closes( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, sessions = runtime + content = "complete output " * 1000 + objects: dict[str, bytes] = {} + reads: list[str] = [] + monkeypatch.setattr(dify_config, "WORKFLOW_VARIABLE_TRUNCATION_MAX_SIZE", 256) + monkeypatch.setattr(dify_config, "WORKFLOW_VARIABLE_TRUNCATION_STRING_LENGTH", 128) + monkeypatch.setattr(storage, "save", lambda key, data: objects.__setitem__(key, data)) + + def load(key: str) -> bytes: + assert sessions + assert all(session.closed and not session.in_transaction() for session in sessions) + reads.append(key) + return objects[key] + + class IsolatedStorage: + def load(self, filename: str) -> bytes: + return load(filename) + + def delete(self, filename: str) -> None: + objects.pop(filename, None) + + def unexpected_global_read(_key: str) -> bytes: + pytest.fail("single-node execution must use the injected output reader") + + monkeypatch.setattr(storage, "load", unexpected_global_read) + gateway._variables = WorkflowVariableService( + repository=WorkflowDraftVariableRepository(sessions=sqlite_session_factory), + files=FileService(sqlite_session_factory), + executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(sqlite_session_factory), + storage=IsolatedStorage(), + defer_file_cleanup=lambda _ids: None, + ) + + class OffloadedExecutionRepository(SQLAlchemyWorkflowNodeExecutionRepository): + @override + def save(self, execution: WorkflowNodeExecution) -> None: + super().save(execution) + # Keep the real persistence and offload implementation. The reader must + # support a writer that stores the execution payload out of line. + self.save_execution_data(execution) + + monkeypatch.setattr( + workflow_module.DifyCoreRepositoryFactory, + "create_workflow_node_execution_repository", + lambda **kwargs: OffloadedExecutionRepository(**kwargs), + ) + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + { + "id": "start", + "data": { + "type": "start", + "title": "Start", + "variables": [ + { + "variable": "payload", + "label": "Payload", + "type": "text-input", + "max_length": 40000, + } + ], + }, + } + ], + "edges": [], + } + ) + response = gateway.run_node( + CONTEXT, "app-1", "start", {"inputs": {"payload": content}}, include_details=True, workflow=None + ) + assert response["status"] == "succeeded" + assert response["outputs_truncated"] is True + assert reads + with sqlite_session_factory() as session: + variable = session.scalar( + select(WorkflowDraftVariable).where( + WorkflowDraftVariable.node_id == "start", + WorkflowDraftVariable.name == "payload", + ) + ) + assert variable is not None + assert variable.get_value().value == content + + +def test_real_start_and_branch_execution_share_injected_variable_and_execution_storage( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, sessions = runtime + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + { + "id": "start", + "data": { + "type": "start", + "title": "Start", + "variables": [ + { + "variable": "payload", + "label": "Payload", + "type": "text-input", + "required": True, + "max_length": 100, + } + ], + }, + }, + { + "id": "branch", + "data": { + "type": "if-else", + "title": "Branch", + "cases": [ + { + "case_id": "matches", + "logical_operator": "and", + "conditions": [ + { + "id": "condition", + "varType": "string", + "variable_selector": ["start", "payload"], + "comparison_operator": "is", + "value": "stored value", + } + ], + } + ], + }, + }, + ], + "edges": [], + } + ) + + class ForbiddenDatabase: + @property + def engine(self) -> None: + pytest.fail("execution must not fall back to global db.engine") + + monkeypatch.setattr(workflow_module, "db", ForbiddenDatabase()) + start = gateway.run_node( + CONTEXT, "app-1", "start", {"inputs": {"payload": "stored value"}}, include_details=False, workflow=None + ) + assert start["status"] == "succeeded" + branch = gateway.run_node(CONTEXT, "app-1", "branch", {}, include_details=False, workflow=None) + assert branch["status"] == "succeeded" + assert json.loads(branch["outputs"]) == {"result": True, "selected_case_id": "matches"} + last_run = gateway.last_run(CONTEXT, "app-1", "branch") + assert last_run is not None + assert last_run["id"] == branch["id"] + assert all(session.closed for session in sessions) + + +@pytest.mark.parametrize("select_all", [False, True], ids=["full-execution", "run-all"]) +@pytest.mark.parametrize("concurrent_change", ["configuration", "delete-node"]) +def test_full_trigger_execution_preserves_revision_through_task_and_worker( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + select_all: bool, + concurrent_change: str, + *, + workflow_variables: WorkflowExecutionVariables, +) -> None: + from configs import dify_config + from core.app.apps import base_app_queue_manager + from core.app.features.rate_limiting.rate_limit import RateLimit + from core.ops.ops_trace_manager import TraceQueueManager + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from services.workflow.execution.adapters.workflow import app_generator as generator_module + from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator + from tasks.app_generate import workflow_execute_task as task_module + + gateway, sessions = runtime + gateway._generator = AppGenerateService + original_graph: dict[str, object] = { + "nodes": [ + { + "id": "trigger", + "data": { + "type": "trigger-plugin", + "title": "Subscription A", + "subscription_id": "subscription-A", + }, + } + ], + "edges": [], + } + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + draft.graph = json.dumps(original_graph) + + class Poller: + def poll(self) -> TriggerDebugEvent: + assert all(session.closed for session in sessions) + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + changed = json.loads(draft.graph) + if concurrent_change == "delete-node": + changed["nodes"] = [] + else: + changed["nodes"][0]["data"]["subscription_id"] = "subscription-B" + draft.graph = json.dumps(changed) + return TriggerDebugEvent(node_id="trigger", workflow_args={"inputs": {"payload": "from-A"}}) + + def poller(*, draft_workflow: Workflow, **_kwargs: object) -> Poller: + assert draft_workflow.graph_dict == original_graph + return Poller() + + def select_events(*, draft_workflow: Workflow, **kwargs: object) -> TriggerDebugEvent: + return poller(draft_workflow=draft_workflow, **kwargs).poll() + + monkeypatch.setattr(module, "create_event_poller", poller) + monkeypatch.setattr(module, "select_trigger_debug_events", select_events) + monkeypatch.setattr(dify_config, "PUBSUB_REDIS_CHANNEL_TYPE", "streams") + limiter = Mock(spec=RateLimit) + limiter.generate.side_effect = lambda response, *_args: response + + def guardrails(*, action: Callable[[RateLimit, str], object], **_kwargs: object) -> object: + return action(limiter, "request") + + monkeypatch.setattr(AppGenerateService, "_run_with_guardrails", guardrails) + queued: list[str] = [] + monkeypatch.setattr(task_module.workflow_based_app_execution_task, "delay", queued.append) + + def retrieve_events(*_args: object, on_subscribe: Callable[[], None]) -> Generator[str, None, None]: + assert all(session.closed for session in sessions) + on_subscribe() + yield from () + + monkeypatch.setattr(WorkflowEventStream, "retrieve_events", retrieve_events) + service = ConsoleWorkflowService( + conversion=Mock(spec=WorkflowConversion), + agent_services=WorkflowAgentPublishService, + drafts=Mock(spec=WorkflowDraftService), + definitions=Mock(spec=WorkflowDefinitions), + lifecycle=Mock(spec=WorkflowDefinitionLifecycle), + runtime=gateway, + apps=Mock(spec=WorkflowAppLookup), + presence=Mock(spec=WorkflowPresence), + access=Mock(spec=WorkflowAccess), + ) + with Flask(__name__).app_context(): + response = service.trigger(CONTEXT, "app-1", ["trigger"], single_node=False, select_all=select_all) + assert isinstance(response, Generator) + assert list(response) == [] + assert len(queued) == 1 + params = task_module.AppExecutionParams.model_validate_json(queued[0]) + assert params.workflow_snapshot is not None + assert json.loads(params.workflow_snapshot.graph) == original_graph + + # Leave the real task runner and generator/thread boundary intact. Only external + # transport, tracing and graph execution are replaced with local observers. + observed: list[tuple[dict[str, object], dict[str, object], str | None]] = [] + + class GraphRunner: + def __init__( + self, + *, + workflow: Workflow, + application_generate_entity: WorkflowAppGenerateEntity, + root_node_id: str | None, + **_kwargs: object, + ) -> None: + observed.append((dict(workflow.graph_dict), dict(application_generate_entity.inputs), root_node_id)) + + def run(self) -> None: + pass + + def empty_response(*_args: object, **_kwargs: object) -> Generator[str, None, None]: + yield from () + + monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", lambda *_args, **_kwargs: None) + monkeypatch.setattr(generator_module, "TraceQueueManager", Mock(return_value=Mock(spec=TraceQueueManager))) + monkeypatch.setattr(generator_module, "WorkflowAppRunner", GraphRunner) + monkeypatch.setattr(WorkflowAppGenerator, "_handle_response", empty_response) + monkeypatch.setattr(task_module, "_publish_streaming_response", lambda response, *_args, **_kwargs: list(response)) + with Flask(__name__).app_context(): + task_module._AppRunner(gateway._sessions, params, variables=workflow_variables).run() + assert observed == [(original_graph, {"payload": "from-A"}, "trigger")] + assert all(session.closed for session in sessions) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("workflow_snapshot.tenant_id", "other-tenant"), + ("workflow_snapshot.app_id", "other-app"), + ("workflow_snapshot.id", "other-workflow"), + ("tenant_id", "other-tenant"), + ("app_id", "other-app"), + ("app_mode", "advanced-chat"), + ("invoke_from", "service-api"), + ], +) +def test_task_rejects_snapshot_outside_its_debugger_owner( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + field: str, + value: str, + *, + workflow_variables: WorkflowExecutionVariables, +) -> None: + from core.app.entities.app_invoke_entities import InvokeFrom + from tasks.app_generate.workflow_execute_task import AppExecutionParams, _AppRunner + + gateway, sessions = runtime + app, actor, draft = gateway._draft(CONTEXT, "app-1", "Workflow not found") + params = AppExecutionParams.new( + app_model=app, + workflow=draft, + user=actor, + args={"inputs": {}}, + invoke_from=InvokeFrom.DEBUGGER, + workflow_snapshot=module.workflow_snapshot(draft), + ) + payload = json.loads(params.model_dump_json()) + if field.startswith("workflow_snapshot."): + payload["workflow_snapshot"][field.split(".")[1]] = value + else: + payload[field] = value + decoded = AppExecutionParams.model_validate_json(json.dumps(payload)) + with pytest.raises(ValueError, match="snapshot does not belong"): + _AppRunner(gateway._sessions, decoded, variables=workflow_variables).run() + assert all(session.closed for session in sessions) + + +@pytest.mark.parametrize("mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]) +@pytest.mark.parametrize("entry", ["iteration", "loop"]) +def test_container_debug_prepares_database_records_before_redis( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + mode: AppMode, + entry: str, +) -> None: + import threading + + from core.app.apps import base_app_queue_manager + from models.model import App, Conversation, Message + from services.workflow.execution.adapters.chatflow.app_generator import AdvancedChatAppGenerator + from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator + + gateway, sessions = runtime + gateway._generator = AppGenerateService + with sqlite_session_factory.begin() as session: + app = session.get(App, "app-1") + assert app is not None + app.mode = mode + writes: list[str] = [] + + def setex(key: str, *_args: object) -> None: + assert sessions + assert all(session.closed and not session.in_transaction() for session in sessions) + if mode == AppMode.ADVANCED_CHAT: + with sqlite_session_factory() as session: + assert session.scalar(select(Conversation.id)) is not None + assert session.scalar(select(Message.id)) is not None + writes.append(key) + + def response(*_args: object, **_kwargs: object) -> Generator[str, None, None]: + yield from () + + def unexpected_gateway_session(): + raise AssertionError("Single-node generation must use the repositories' sessions") + + monkeypatch.setattr(gateway, "_sessions", unexpected_gateway_session) + + # Leave actual generator initialization, record creation and queue initialization intact. + monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", setex) + monkeypatch.setattr(threading.Thread, "start", lambda _thread: None) + monkeypatch.setattr(threading.Thread, "join", lambda _thread, **_kwargs: None) + monkeypatch.setattr(WorkflowAppGenerator, "_handle_response", response) + monkeypatch.setattr(AdvancedChatAppGenerator, "_handle_advanced_chat_response", response) + with Flask(__name__).app_context(): + result = getattr(gateway, entry)(CONTEXT, "app-1", "container", {"item": "value"}) + list(result) + assert writes + assert all(session.closed and not session.in_transaction() for session in sessions) + + +@pytest.fixture +def agent_runtime( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], +) -> tuple[WorkflowRuntimeGateway, list[RecordingSession]]: + from enums.agent import WorkflowAgentBindingType + from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource, AgentStatus, WorkflowAgentNodeBinding + from models.agent_config_entities import AgentSoulConfig, WorkflowNodeJobConfig + + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + {"id": "trigger", "data": {"type": "trigger-plugin"}}, + {"id": "agent-node", "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}, + ], + "edges": [], + } + ) + session.add( + Agent( + id="agent-1", + tenant_id="tenant-1", + name="Inline", + scope=AgentScope.WORKFLOW_ONLY, + source=AgentSource.WORKFLOW, + status=AgentStatus.ACTIVE, + ) + ) + for number in (1, 2): + session.add( + AgentConfigSnapshot( + id=f"soul-{number}", + tenant_id="tenant-1", + agent_id="agent-1", + version=number, + config_snapshot=AgentSoulConfig(prompt={"system_prompt": f"Soul {number}"}), + ) + ) + session.add( + WorkflowAgentNodeBinding( + id="binding", + tenant_id="tenant-1", + app_id="app-1", + workflow_id=draft.id, + workflow_version="draft", + node_id="agent-node", + agent_id="agent-1", + binding_type=WorkflowAgentBindingType.INLINE_AGENT, + current_snapshot_id="soul-1", + node_job_config=WorkflowNodeJobConfig(workflow_prompt="Job A"), + ) + ) + + return runtime + + +@pytest.mark.parametrize("change", ["delete", "replace"]) +@pytest.mark.parametrize("select_all", [False, True]) +def test_trigger_pins_agent_job_and_soul_until_worker_finishes( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + change: str, + select_all: bool, +) -> None: + from core.app.features.rate_limiting.rate_limit import RateLimit + from graphon.enums import WorkflowExecutionStatus + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.agent_config_entities import WorkflowNodeJobConfig + from models.workflow import WorkflowRun + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate import workflow_execute_task as task_module + + gateway, sessions = agent_runtime + gateway._generator = AppGenerateService + + class Poller: + def poll(self) -> TriggerDebugEvent: + assert sessions + assert all(session.closed for session in sessions) + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + if change == "delete": + session.delete(binding) + draft = session.get(Workflow, binding.workflow_id) + assert draft is not None + draft.graph = '{"nodes": [], "edges": []}' + else: + binding.current_snapshot_id = "soul-2" + binding.node_job_config = WorkflowNodeJobConfig(workflow_prompt="Job B") + session.flush() + assert ( + WorkflowAgentRetirementRepository.archive_unowned( + session=session, tenant_id="tenant-1", agent_ids=["agent-1"], account_id="account-1" + ) + == [] + ) + return TriggerDebugEvent(node_id="trigger", workflow_args={"inputs": {"payload": "A"}}) + + monkeypatch.setattr(module, "create_event_poller", lambda **_kwargs: Poller()) + monkeypatch.setattr(module, "select_trigger_debug_events", lambda **_kwargs: Poller().poll()) + monkeypatch.setattr(dify_config, "PUBSUB_REDIS_CHANNEL_TYPE", "streams") + limiter = Mock(spec=RateLimit) + limiter.generate.side_effect = lambda response, *_args: response + monkeypatch.setattr( + AppGenerateService, "_run_with_guardrails", lambda *, action, **_kwargs: action(limiter, "request") + ) + queued: list[str] = [] + monkeypatch.setattr(task_module.workflow_based_app_execution_task, "delay", queued.append) + + def events(*_args: object, on_subscribe: Callable[[], None]) -> Generator[str, None, None]: + on_subscribe() + yield from () + + monkeypatch.setattr(WorkflowEventStream, "retrieve_events", events) + with Flask(__name__).app_context(): + event_data = gateway.poll_trigger(CONTEXT, "app-1", ["trigger"], single_node=False, select_all=select_all) + assert event_data is not None + assert event_data.workflow.execution_id is not None + list( + gateway.generate( + CONTEXT, "app-1", event_data.workflow_args, root_node_id="trigger", workflow=event_data.workflow + ) + ) + params = task_module.AppExecutionParams.model_validate_json(queued[0]) + assert params.workflow_run_id == event_data.workflow.execution_id + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", lambda **_kwargs: None) + + def run_app(_runner, *, workflow: Workflow, **_kwargs: object) -> Generator[str, None, None]: + assert workflow.graph_dict["nodes"][1]["id"] == "agent-node" + resolved = _runner._runtime.agent_bindings.resolve( + tenant_id="tenant-1", + app_id="app-1", + workflow_id=workflow.id, + node_id="agent-node", + workflow_run_id=params.workflow_run_id, + ) + assert resolved.binding.node_job_config_dict["workflow_prompt"] == "Job A" + assert resolved.snapshot.id == "soul-1" + assert resolved.snapshot.config_snapshot_dict["prompt"]["system_prompt"] == "Soul 1" + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, params.workflow_run_id) + assert run is not None + run.status = WorkflowExecutionStatus.SUCCEEDED + yield from () + + monkeypatch.setattr(task_module._AppRunner, "_run_app", run_app) + monkeypatch.setattr(task_module, "_publish_streaming_response", lambda response, *_args, **_kwargs: list(response)) + with Flask(__name__).app_context(): + task_module._AppRunner(gateway._sessions, params, variables=gateway._variables).run() + with sqlite_session_factory() as session: + agent = session.get(Agent, "agent-1") + assert agent is not None + assert agent.status == (AgentStatus.ARCHIVED if change == "delete" else AgentStatus.ACTIVE) + assert all(session.closed and not session.in_transaction() for session in sessions) + + +@pytest.mark.parametrize("outcome", ["empty", "poll-error", "prepare-error"]) +def test_trigger_cancels_reserved_agent_references_without_starting_execution( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + outcome: str, +) -> None: + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + + gateway, sessions = agent_runtime + gateway._generator = AppGenerateService + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", lambda **_kwargs: None) + + class Poller: + def poll(self) -> TriggerDebugEvent | None: + assert sessions + assert all(session.closed for session in sessions) + with sqlite_session_factory.begin() as session: + assert session.scalar(select(WorkflowRun.id)) is not None + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + if outcome == "poll-error": + raise ValueError("poll failed") + if outcome == "empty": + return None + return TriggerDebugEvent(node_id="trigger", workflow_args={"inputs": {}}) + + def denied(**_kwargs: object) -> None: + raise ValueError("prepare failed") + + monkeypatch.setattr(module, "create_event_poller", lambda **_kwargs: Poller()) + monkeypatch.setattr(AppGenerateService, "_run_with_guardrails", denied) + if outcome == "empty": + assert gateway.poll_trigger(CONTEXT, "app-1", ["trigger"], single_node=False, select_all=False) is None + elif outcome == "poll-error": + with pytest.raises(ValueError, match="poll failed"): + gateway.poll_trigger(CONTEXT, "app-1", ["trigger"], single_node=False, select_all=False) + else: + event_data = gateway.poll_trigger(CONTEXT, "app-1", ["trigger"], single_node=False, select_all=False) + assert event_data is not None + with Flask(__name__).app_context(), pytest.raises(ValueError, match="prepare failed"): + gateway.generate(CONTEXT, "app-1", {}, root_node_id="trigger", workflow=event_data.workflow) + with sqlite_session_factory() as session: + assert session.scalar(select(WorkflowRun.id)) is None + agent = session.get(Agent, "agent-1") + assert agent is not None + assert agent.status == AgentStatus.ARCHIVED + + +def test_paused_trigger_keeps_agent_references_until_resumed_stream_finishes( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + from graphon.entities import WorkflowStartReason + from graphon.enums import WorkflowExecutionStatus + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate.workflow_execute_task import _publish_streaming_response + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", lambda **_kwargs: None) + + class Topic: + def publish(self, _payload: bytes) -> None: + pass + + monkeypatch.setattr(WorkflowEventStream, "get_response_topic", lambda *_args: Topic()) + for event_name, status in [ + ("workflow_paused", WorkflowExecutionStatus.PAUSED), + ("workflow_finished", WorkflowExecutionStatus.SUCCEEDED), + ]: + _publish_streaming_response( + (item for item in [{"event": event_name, "data": {"status": status.value}}]), + snapshot.execution_id, + AppMode.WORKFLOW, + snapshot.id, + {}, + WorkflowStartReason.RESUMPTION, + sessions=gateway._sessions, + tenant_id="tenant-1", + app_id="app-1", + agent_snapshot=True, + ) + with sqlite_session_factory() as session: + agent = session.get(Agent, "agent-1") + run = session.get(WorkflowRun, snapshot.execution_id) + assert agent is not None + assert run is not None + assert run.status == status + assert agent.status == ( + AgentStatus.ACTIVE if status == WorkflowExecutionStatus.PAUSED else AgentStatus.ARCHIVED + ) + + +@pytest.mark.parametrize("channel_type", ["streams", "pubsub", "sharded"]) +@pytest.mark.parametrize( + "outcome", + [ + "broker-error", + "subscribe-error", + "unopened", + "response-unopened", + "close-before-subscribe", + "disconnect-after-enqueue", + "error-after-enqueue", + "complete", + ], +) +def test_reserved_trigger_stream_owns_cleanup_until_enqueue( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + channel_type: str, + outcome: str, +) -> None: + from core.app.features.rate_limiting.rate_limit import RateLimit + from graphon.enums import WorkflowExecutionStatus + from libs.helper import compact_generate_response + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate import workflow_execute_task as task_module + + gateway, sessions = agent_runtime + gateway._generator = AppGenerateService + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + session.flush() + assert ( + WorkflowAgentRetirementRepository.archive_unowned( + session=session, tenant_id="tenant-1", agent_ids=["agent-1"], account_id="account-1" + ) + == [] + ) + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", lambda **_kwargs: None) + monkeypatch.setattr(dify_config, "PUBSUB_REDIS_CHANNEL_TYPE", channel_type) + + def unexpected_timer(*_args: object, **_kwargs: object) -> None: + pytest.fail("a reservation must not enqueue after the stream releases its references") + + monkeypatch.setattr("services.app_generate_service.threading.Timer", unexpected_timer) + limiter = RateLimit("app-1", -1) + monkeypatch.setattr( + AppGenerateService, + "_run_with_guardrails", + lambda *, action, **_kwargs: action(limiter, RateLimit._UNLIMITED_REQUEST_ID), + ) + queued: list[str] = [] + + def enqueue(payload: str) -> None: + assert all(session.closed for session in sessions) + if outcome == "broker-error": + raise RuntimeError("broker unavailable") + queued.append(payload) + + monkeypatch.setattr(task_module.workflow_based_app_execution_task, "delay", enqueue) + + class Subscription: + def __enter__(self) -> Self: + if outcome == "subscribe-error": + raise RuntimeError("subscription unavailable") + return self + + def __exit__(self, *_args: object) -> None: + pass + + def receive(self, **_kwargs: object) -> bytes: + if outcome == "error-after-enqueue": + raise RuntimeError("connection lost") + if outcome == "complete": + return b'{"event": "workflow_finished"}' + return b'{"event": "workflow_started"}' + + class Topic: + def as_subscriber(self) -> Self: + return self + + def subscribe(self) -> Subscription: + return Subscription() + + monkeypatch.setattr(WorkflowEventStream, "get_response_topic", lambda *_args: Topic()) + with Flask(__name__).test_request_context(): + stream = gateway.generate(CONTEXT, "app-1", {}, root_node_id="trigger", workflow=snapshot) + assert isinstance(stream, Generator) + assert not queued + if outcome == "response-unopened": + response = compact_generate_response(stream) + response.close() + elif outcome == "unopened": + stream.close() + elif outcome == "close-before-subscribe": + assert next(stream) == "event: ping\n\n" + stream.close() + elif outcome == "disconnect-after-enqueue": + next(stream) # Initial ping precedes subscribing and dispatch. + next(stream) + stream.close() + elif outcome == "complete": + list(stream) + else: + with pytest.raises(RuntimeError, match="unavailable|connection lost"): + list(stream) + stream.close() # Repeated closure must be harmless. + + enqueued = outcome in {"disconnect-after-enqueue", "error-after-enqueue", "complete"} + assert len(queued) == int(enqueued) + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + assert agent is not None + if enqueued: + assert run is not None + assert run.status == WorkflowExecutionStatus.RUNNING + assert agent.status == AgentStatus.ACTIVE + else: + assert run is None + assert agent.status == AgentStatus.ARCHIVED + assert all(session.closed for session in sessions) + + +@pytest.mark.parametrize("other_execution", [None, "running", "paused"]) +def test_global_timeout_retires_paused_trigger_agent_after_last_execution( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + other_execution: str | None, +) -> None: + from datetime import timedelta + from types import SimpleNamespace + + from enums.human_input import HumanInputFormKind, HumanInputFormStatus + from graphon.enums import WorkflowExecutionStatus + from libs.datetime_utils import naive_utc_now + from models.agent import ( + Agent, + AgentHomeSnapshot, + AgentStatus, + AgentWorkingResourceStatus, + WorkflowAgentNodeBinding, + ) + from models.human_input import HumanInputForm + from models.workflow import WorkflowRun + from tasks import human_input_timeout_tasks as timeout_module + + gateway, sessions = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + if other_execution is not None: + _, other = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + with sqlite_session_factory.begin() as session: + other_run = session.get(WorkflowRun, other.execution_id) + assert other_run is not None + other_run.status = WorkflowExecutionStatus(other_execution) + now = naive_utc_now() + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert run is not None + assert binding is not None + run.status = WorkflowExecutionStatus.PAUSED + session.delete(binding) + session.add( + AgentHomeSnapshot( + id="home-1", + tenant_id="tenant-1", + agent_id="agent-1", + snapshot_ref="home-ref", + status=AgentWorkingResourceStatus.ACTIVE, + ) + ) + session.add( + HumanInputForm( + id="timeout-form", + tenant_id="tenant-1", + app_id="app-1", + workflow_run_id=run.id, + node_id="agent-node", + form_kind=HumanInputFormKind.RUNTIME, + form_definition='{"form_content": "", "rendered_content": ""}', + rendered_content="", + status=HumanInputFormStatus.WAITING, + created_at=now - timedelta(hours=2), + expiration_time=now + timedelta(hours=1), + ) + ) + assert session.scalar(select(WorkflowAgentNodeBinding.agent_id)) == "agent-1" + monkeypatch.setattr(timeout_module, "db", SimpleNamespace(engine=sqlite_session_factory.kw["bind"])) + monkeypatch.setattr(dify_config, "HUMAN_INPUT_GLOBAL_TIMEOUT_SECONDS", 3600) + collector = Mock() + monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", collector) + + timeout_module.check_and_handle_human_input_timeouts() + + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + home = session.get(AgentHomeSnapshot, "home-1") + form = session.get(HumanInputForm, "timeout-form") + assert run is not None + assert agent is not None + assert home is not None + assert form is not None + assert run.status == WorkflowExecutionStatus.STOPPED + assert form.status == HumanInputFormStatus.EXPIRED + assert run.finished_at is not None + if other_execution is None: + assert agent.status == AgentStatus.ARCHIVED + assert home.status == AgentWorkingResourceStatus.RETIRED + collector.assert_called_once() + assert collector.call_args.kwargs["purge_agent_ids"] == ["agent-1"] + assert collector.call_args.kwargs["home_snapshot_ids"] == ["home-1"] + else: + assert agent.status == AgentStatus.ACTIVE + assert home.status == AgentWorkingResourceStatus.ACTIVE + collector.assert_not_called() + assert all(session.closed for session in sessions) + + +def test_running_lease_renews_and_fences_dead_workers( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], +) -> None: + from datetime import timedelta + + from libs.datetime_utils import naive_utc_now + from models.workflow import WorkflowRun + from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository + from repositories.workflow.execution_write_repository import save_workflow_run + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + now = naive_utc_now() + timeout = timedelta(seconds=dify_config.WORKFLOW_DEBUG_RESERVATION_TIMEOUT) + repository = WorkflowDebugReservationRepository(sqlite_session_factory) + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + WorkflowDebugReservationRepository.claim(session, run, now) + # A run can outlive its original startup deadline while its worker renews. + for _ in range(4): + now += timeout / 2 + assert repository.renew( + tenant_id=snapshot.tenant_id, + app_id=snapshot.app_id, + workflow_id=snapshot.id, + execution_id=snapshot.execution_id, + now=now, + ) + assert repository.pending_batch(now, limit=100) == [] + assert repository.expire(snapshot.execution_id, now) is None + with sqlite_session_factory() as session: + stale_start = session.get(WorkflowRun, snapshot.execution_id) + assert stale_start is not None + session.expunge(stale_start) + # Simulate process death: no more heartbeats, even if no Celery terminal task arrives. + now += timedelta(days=365) + assert [item.execution_id for item in repository.pending_batch(now, limit=100)] == [snapshot.execution_id] + assert ( + repository.renew( + tenant_id=snapshot.tenant_id, + app_id=snapshot.app_id, + workflow_id=snapshot.id, + execution_id=snapshot.execution_id, + now=now, + ) + is False + ) + assert repository.expire(snapshot.execution_id, now) is not None + with sqlite_session_factory.begin() as session: + assert save_workflow_run(session, stale_start) is False + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + reservation = debug_lease(session, snapshot.execution_id) + assert run is not None + assert run.status.value == "failed" + assert run.error == "Trigger debug worker lease expired" + # Cleanup retry evidence survives the terminal transition. + assert reservation is not None + + +def test_paused_debug_lease_is_protected_and_refreshed_atomically_on_resume( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], +) -> None: + from datetime import timedelta + + from graphon.enums import WorkflowExecutionStatus + from libs.datetime_utils import naive_utc_now + from models.workflow import WorkflowPause, WorkflowRun + from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository + from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + now = naive_utc_now() + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + lease = debug_lease(session, snapshot.execution_id) + assert run is not None + assert lease is not None + WorkflowDebugReservationRepository.claim(session, run, now) + run.status = WorkflowExecutionStatus.PAUSED + set_debug_deadline(session, snapshot.execution_id, now - timedelta(days=365)) + session.add(WorkflowPause(workflow_id=run.workflow_id, workflow_run_id=run.id, state_object_key="pause")) + reservations = WorkflowDebugReservationRepository(sqlite_session_factory) + assert reservations.pending_batch(now + timedelta(days=365), limit=100) == [] + assert reservations.expire(snapshot.execution_id, now) is None + runs = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory) + pause = runs.get_workflow_pause(snapshot.execution_id) + assert pause is not None + runs.resume_workflow_pause(snapshot.execution_id, pause) + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + lease = debug_lease(session, snapshot.execution_id) + assert run is not None + assert lease is not None + assert run.status == WorkflowExecutionStatus.RUNNING + assert lease.expires_at > now + assert reservations.expire(snapshot.execution_id, now) is None + + +@pytest.mark.parametrize("failed_row", ["reservation", "reference"]) +def test_reservation_and_execution_owners_roll_back_together( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + failed_row: str, +) -> None: + from models.workflow import WorkflowRun + + gateway, _ = agent_runtime + from models.agent import WorkflowAgentNodeBinding + + row_type = WorkflowRun if failed_row == "reservation" else WorkflowAgentNodeBinding + + def fail(session: Session, _context: object, _instances: object) -> None: + if any(isinstance(row, row_type) for row in session.new): + raise RuntimeError("reservation persistence failed") + + event.listen(gateway._reservations._sessions, "before_flush", fail) + try: + with pytest.raises(RuntimeError, match="reservation persistence failed"): + gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + finally: + event.remove(gateway._reservations._sessions, "before_flush", fail) + with sqlite_session_factory() as session: + assert session.scalar(select(WorkflowRun)) is None + assert ( + session.scalar( + select(WorkflowAgentNodeBinding).where(WorkflowAgentNodeBinding.workflow_version == "execution") + ) + is None + ) + + +@pytest.mark.parametrize("cleanup_fails", [True, False]) +def test_terminal_notification_survives_cleanup_dispatch_failure( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + cleanup_fails: bool, +) -> None: + from graphon.entities import WorkflowStartReason + from graphon.enums import WorkflowExecutionStatus + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + from services.agent.retirement_service import WorkflowAgentRetirementService + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate.workflow_execute_task import _publish_streaming_response + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + received: list[dict[str, object]] = [] + attempted: list[set[str]] = [] + + class Topic: + def publish(self, payload: bytes) -> None: + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + assert run.status == WorkflowExecutionStatus.SUCCEEDED + received.append(json.loads(payload)) + + def dispatch(*, purge_agent_ids: Iterable[str], **_kwargs: object) -> None: + assert received == [{"event": "workflow_finished", "data": {"status": "succeeded"}}] + attempted.append(set(purge_agent_ids)) + if cleanup_fails and len(attempted) == 1: + raise RuntimeError("resource broker unavailable") + + monkeypatch.setattr(WorkflowEventStream, "get_response_topic", lambda *_args: Topic()) + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", dispatch) + _publish_streaming_response( + (item for item in [{"event": "workflow_finished", "data": {"status": "succeeded"}}]), + snapshot.execution_id, + AppMode.WORKFLOW, + snapshot.id, + {}, + WorkflowStartReason.RESUMPTION, + sessions=gateway._sessions, + tenant_id="tenant-1", + app_id="app-1", + agent_snapshot=True, + ) + assert len(received) == 1 + assert attempted == [{"agent-1"}] + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + assert run is not None + assert run.status == WorkflowExecutionStatus.SUCCEEDED + assert agent is not None + assert agent.status == AgentStatus.ARCHIVED + assert (debug_lease(session, snapshot.execution_id) is not None) == cleanup_fails + if cleanup_fails: + # The real entry point must recover candidates without callers retaining Agent IDs. + WorkflowAgentRetirementService.finish_execution( + sessions=gateway._sessions, + tenant_id="tenant-1", + app_id="app-1", + workflow_id=snapshot.id, + execution_id=snapshot.execution_id, + account_id=None, + ) + assert attempted == [{"agent-1"}, {"agent-1"}] + with sqlite_session_factory() as session: + assert debug_lease(session, snapshot.execution_id) is None + assert len(received) == 1 + + +@pytest.mark.parametrize("outcome", ["succeeded", "failed", "stopped", "partial-succeeded"]) +@pytest.mark.parametrize("cleanup_fails", [False, True]) +def test_completed_debug_redelivery_replays_persisted_outcome_without_execution_or_cleanup( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + outcome: str, + cleanup_fails: bool, +) -> None: + from core.app.entities.app_invoke_entities import InvokeFrom + from core.app.entities.task_entities import WorkflowFinishStreamResponse + from graphon.enums import WorkflowExecutionStatus + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + from tasks.app_generate import workflow_execute_task as task_module + + status = WorkflowExecutionStatus(outcome) + gateway, sessions = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + app, actor, workflow = gateway._definitions.debug_context(CONTEXT, "app-1", workflow_id=None, snapshot=snapshot) + params = task_module.AppExecutionParams.new( + app_model=app, + workflow=workflow, + user=actor, + args={}, + invoke_from=InvokeFrom.DEBUGGER, + workflow_run_id=snapshot.execution_id, + workflow_snapshot=snapshot, + ) + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + + engine_calls: list[str] = [] + cleanup_calls: list[set[str]] = [] + received: list[bytes] = [] + execution_error = None if status == WorkflowExecutionStatus.SUCCEEDED else "Actual engine outcome" + + def execute(_runner: task_module._AppRunner, **_kwargs: object) -> Generator[dict[str, object], None, None]: + engine_calls.append(params.workflow_run_id) + yield {"event": "workflow_started"} + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, params.workflow_run_id) + assert run is not None + run.outputs = json.dumps({"answer": "stored output"}) + run.error = execution_error + run.elapsed_time = 1.5 + run.total_tokens = 42 + run.total_steps = 3 + yield {"event": "workflow_finished", "data": {"status": status.value}} + + def collect(*, purge_agent_ids: Iterable[str], **_kwargs: object) -> None: + cleanup_calls.append(set(purge_agent_ids)) + if cleanup_fails: + raise RuntimeError("Cleanup broker unavailable") + + class Topic: + def publish(self, payload: bytes) -> None: + assert all(session.closed and not session.in_transaction() for session in sessions) + received.append(payload) + + monkeypatch.setattr(task_module._AppRunner, "_run_app", execute) + monkeypatch.setattr(task_module.WorkflowEventStream, "get_response_topic", lambda *_args: Topic()) + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", collect) + with Flask(__name__).app_context(): + assert task_module._AppRunner(gateway._sessions, params, variables=gateway._variables).run() is None + assert [json.loads(payload)["event"] for payload in received] == ["workflow_started", "workflow_finished"] + assert engine_calls == [snapshot.execution_id] + assert cleanup_calls == [{"agent-1"}] + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + assert run is not None + assert run.status == status + assert agent is not None + assert agent.status == AgentStatus.ARCHIVED + assert (debug_lease(session, snapshot.execution_id) is not None) == cleanup_fails + persisted = run.to_dict() + + received.clear() + # The delivery boundary deserializes a fresh copy of the original task payload. + for _ in range(2): + redelivery = task_module.AppExecutionParams.model_validate_json(params.model_dump_json()) + assert task_module._AppRunner(gateway._sessions, redelivery, variables=gateway._variables).run() is None + assert [json.loads(payload)["event"] for payload in received] == ["workflow_finished", "workflow_finished"] + for payload in received: + replay = WorkflowFinishStreamResponse.model_validate_json(payload) + assert replay.workflow_run_id == snapshot.execution_id + assert replay.data.status == status + assert replay.data.error == execution_error + assert replay.data.outputs == {"answer": "stored output"} + assert replay.data.elapsed_time == 1.5 + assert replay.data.total_steps == 3 + assert replay.data.total_tokens == 42 + + # Blocking redeliveries have no stream to replay and also exit without work. + blocking = params.model_copy(update={"streaming": False}) + assert task_module._AppRunner(gateway._sessions, blocking, variables=gateway._variables).run() is None + assert len(received) == 2 + assert engine_calls == [snapshot.execution_id] + assert cleanup_calls == [{"agent-1"}] + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + assert run.to_dict() == persisted + assert (debug_lease(session, snapshot.execution_id) is not None) == cleanup_fails diff --git a/api/tests/unit_tests/services/workflow/test_workflow_converter_additional.py b/api/tests/unit_tests/services/workflow/test_workflow_converter_additional.py index 3cc8739623de94..dc24dc2c74b0c3 100644 --- a/api/tests/unit_tests/services/workflow/test_workflow_converter_additional.py +++ b/api/tests/unit_tests/services/workflow/test_workflow_converter_additional.py @@ -1,13 +1,9 @@ from __future__ import annotations import json -from types import SimpleNamespace from typing import Any -from unittest.mock import MagicMock, call import pytest -from sqlalchemy import event, select -from sqlalchemy.orm import Session from core.app.app_config.entities import ( AdvancedChatMessageEntity, @@ -19,11 +15,10 @@ ModelConfigEntity, PromptTemplateEntity, ) -from core.helper import encrypter from core.prompt.utils.prompt_template_parser import PromptTemplateParser -from models.api_based_extension import APIBasedExtension, APIBasedExtensionPoint -from models.model import Account, App, AppMode, AppModelConfig, IconType -from models.workflow import Workflow, WorkflowType +from models.api_based_extension import APIBasedExtensionPoint +from models.model import App, AppMode +from models.workflow_conversion import ConversionExtension from services.workflow import workflow_converter as converter_module from services.workflow.workflow_converter import WorkflowConverter @@ -59,24 +54,6 @@ def _app_model(**kwargs: Any) -> App: return App(**defaults) -def _account(**kwargs: Any) -> Account: - account_id = kwargs.pop("id", "account-1") - account = Account( - name=kwargs.pop("name", "Converter user"), - email=kwargs.pop("email", "user@example.com"), - **kwargs, - ) - account.id = account_id - return account - - -def _app_model_config(**kwargs: Any) -> AppModelConfig: - config_id = kwargs.pop("id", "config-1") - config = AppModelConfig(app_id=kwargs.pop("app_id", "app-1"), **kwargs) - config.id = config_id - return config - - def _build_start_graph() -> dict[str, Any]: return { "nodes": [ @@ -112,22 +89,17 @@ def test__convert_to_start_node(default_variables: list[VariableEntity]) -> None assert result["data"]["variables"][0]["variable"] == "text_input" -def test__convert_to_http_request_node_for_chatbot( - default_variables: list[VariableEntity], unbound_session: Session -) -> None: +def test__convert_to_http_request_node_for_chatbot(default_variables: list[VariableEntity]) -> None: app_model = _app_model(id="app_id", tenant_id="tenant_id", mode=AppMode.CHAT) - extension = APIBasedExtension( - tenant_id="tenant_id", + extension = ConversionExtension( + id="api_based_extension_id", name="api-1", - api_key="encrypted_api_key", + api_key="api_key", api_endpoint="https://dify.ai", ) - extension.id = "api_based_extension_id" workflow_converter = WorkflowConverter() - workflow_converter._get_api_based_extension = MagicMock(return_value=extension) - encrypter.decrypt_token = MagicMock(return_value="api_key") external_data_variables = [ ExternalDataVariableEntity( @@ -138,10 +110,11 @@ def test__convert_to_http_request_node_for_chatbot( ] nodes, mapping = workflow_converter._convert_to_http_request_node( - app_model=app_model, + app_id=app_model.id, + app_mode=AppMode(app_model.mode), variables=default_variables, external_data_variables=external_data_variables, - session=unbound_session, + extensions={extension.id: extension}, ) assert len(nodes) == 2 @@ -154,22 +127,17 @@ def test__convert_to_http_request_node_for_chatbot( assert mapping == {"external_variable": "code_1"} -def test__convert_to_http_request_node_for_workflow_app( - default_variables: list[VariableEntity], unbound_session: Session -) -> None: +def test__convert_to_http_request_node_for_workflow_app(default_variables: list[VariableEntity]) -> None: app_model = _app_model(id="app_id", tenant_id="tenant_id", mode=AppMode.WORKFLOW) - extension = APIBasedExtension( - tenant_id="tenant_id", + extension = ConversionExtension( + id="api_based_extension_id", name="api-1", - api_key="encrypted_api_key", + api_key="api_key", api_endpoint="https://dify.ai", ) - extension.id = "api_based_extension_id" workflow_converter = WorkflowConverter() - workflow_converter._get_api_based_extension = MagicMock(return_value=extension) - encrypter.decrypt_token = MagicMock(return_value="api_key") external_data_variables = [ ExternalDataVariableEntity( @@ -180,10 +148,11 @@ def test__convert_to_http_request_node_for_workflow_app( ] nodes, _ = workflow_converter._convert_to_http_request_node( - app_model=app_model, + app_id=app_model.id, + app_mode=AppMode(app_model.mode), variables=default_variables, external_data_variables=external_data_variables, - session=unbound_session, + extensions={extension.id: extension}, ) body = json.loads(nodes[0]["data"]["body"]["data"]) @@ -377,328 +346,8 @@ def test__convert_to_answer_node() -> None: assert node["data"]["type"] == BuiltinNodeTypes.ANSWER -def test_convert_to_workflow_should_raise_when_app_model_config_is_missing( - converter: WorkflowConverter, unbound_session: Session -) -> None: - app_model = _app_model(app_model_config_id=None) - - with pytest.raises(ValueError, match="App model config is required"): - converter.convert_to_workflow( - app_model=app_model, - account=_account(id="account-1"), - name="new-app", - icon_type="emoji", - icon="robot", - icon_background="#fff", - session=unbound_session, - ) - - assert not unbound_session.in_transaction() - - -@pytest.mark.parametrize( - ("source_mode", "expected_mode"), - [ - (AppMode.CHAT, AppMode.ADVANCED_CHAT), - (AppMode.COMPLETION, AppMode.WORKFLOW), - ], -) -def test_convert_to_workflow_should_create_new_app_with_fallback_fields( - converter: WorkflowConverter, - monkeypatch: pytest.MonkeyPatch, - source_mode: AppMode, - expected_mode: AppMode, - sqlite_session: Session, -) -> None: - app_model_config = AppModelConfig(app_id="source-app") - app_model_config.id = "config-1" - sqlite_session.add(app_model_config) - sqlite_session.flush() - workflow = Workflow( - tenant_id="tenant-1", - app_id="source-app", - type=WorkflowType.WORKFLOW, - version=Workflow.VERSION_DRAFT, - graph="{}", - features="{}", - created_by="account-1", - environment_variables=[], - conversation_variables=[], - ) - monkeypatch.setattr(converter, "convert_app_model_config_to_workflow", MagicMock(return_value=workflow)) - phase_events: list[str] = [] - event.listen(sqlite_session, "after_commit", lambda _session: phase_events.append("commit")) - - send_mock = MagicMock(side_effect=lambda *_args, **_kwargs: phase_events.append("signal")) - monkeypatch.setattr(converter_module.app_was_created, "send", send_mock) - - account = _account(id="account-1") - app_model = _app_model( - tenant_id="tenant-1", - id="source-app", - name="Source App", - mode=source_mode, - icon_type=IconType.EMOJI, - icon="sparkles", - icon_background="#123456", - enable_site=True, - enable_api=True, - api_rpm=10, - api_rph=100, - is_public=False, - app_model_config_id="config-1", - ) - - new_app = converter.convert_to_workflow( - app_model=app_model, - account=account, - name="", - icon_type="", - icon="", - icon_background="", - session=sqlite_session, - ) - - assert new_app.name == "Source App(workflow)" - assert new_app.mode == expected_mode - assert new_app.icon_type == IconType.EMOJI - assert new_app.icon == "sparkles" - assert new_app.icon_background == "#123456" - assert new_app.created_by == "account-1" - assert workflow.app_id == new_app.id - assert sqlite_session.get(App, new_app.id) is new_app - assert phase_events == ["commit", "signal", "commit"] - send_mock.assert_called_once_with(new_app, account=account, session=sqlite_session) - - -def test_convert_app_model_config_to_workflow_should_build_advanced_chat_graph_and_features( - converter: WorkflowConverter, - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -) -> None: - app_model = _app_model(id="app-1", tenant_id="tenant-1", mode=AppMode.CHAT) - app_config = SimpleNamespace( - variables=[SimpleNamespace(variable="name")], - external_data_variables=[SimpleNamespace(variable="ext")], - dataset=SimpleNamespace(id="dataset"), - model=SimpleNamespace(), - prompt_template=SimpleNamespace(), - additional_features=SimpleNamespace(file_upload=SimpleNamespace()), - app_model_config_dict={ - "opening_statement": "hello", - "suggested_questions": ["q1"], - "suggested_questions_after_answer": True, - "speech_to_text": True, - "text_to_speech": {"enabled": True}, - "file_upload": {"enabled": True}, - "sensitive_word_avoidance": {"enabled": True}, - "retriever_resource": {"enabled": True}, - }, - ) - - monkeypatch.setattr(converter, "_get_new_app_mode", MagicMock(return_value=AppMode.ADVANCED_CHAT)) - monkeypatch.setattr(converter, "_convert_to_app_config", MagicMock(return_value=app_config)) - monkeypatch.setattr( - converter, - "_convert_to_start_node", - MagicMock( - return_value={"id": "start", "position": None, "data": {"type": BuiltinNodeTypes.START, "variables": []}} - ), - ) - monkeypatch.setattr( - converter, - "_convert_to_http_request_node", - MagicMock( - return_value=( - [{"id": "http", "position": None, "data": {"type": BuiltinNodeTypes.HTTP_REQUEST}}], - {"ext": "code_1"}, - ) - ), - ) - monkeypatch.setattr( - converter, - "_convert_to_knowledge_retrieval_node", - MagicMock( - return_value={"id": "knowledge", "position": None, "data": {"type": BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL}} - ), - ) - monkeypatch.setattr( - converter, - "_convert_to_llm_node", - MagicMock(return_value={"id": "llm", "position": None, "data": {"type": BuiltinNodeTypes.LLM}}), - ) - monkeypatch.setattr( - converter, - "_convert_to_answer_node", - MagicMock(return_value={"id": "answer", "position": None, "data": {"type": BuiltinNodeTypes.ANSWER}}), - ) - workflow = converter.convert_app_model_config_to_workflow( - app_model=app_model, - app_model_config=_app_model_config(id="cfg"), - account_id="account-1", - session=sqlite_session, - ) - - graph = json.loads(workflow.graph) - node_ids = [node["id"] for node in graph["nodes"]] - assert node_ids == ["start", "http", "knowledge", "llm", "answer"] - - features = json.loads(workflow.features) - assert "opening_statement" in features - assert "retriever_resource" in features - assert sqlite_session.scalar(select(Workflow).where(Workflow.id == workflow.id)) is workflow - - -def test_convert_app_model_config_to_workflow_should_build_workflow_mode_with_end_node( - converter: WorkflowConverter, - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -) -> None: - app_model = _app_model(id="app-1", tenant_id="tenant-1", mode=AppMode.COMPLETION) - app_config = SimpleNamespace( - variables=[SimpleNamespace(variable="name")], - external_data_variables=[], - dataset=SimpleNamespace(id="dataset"), - model=SimpleNamespace(), - prompt_template=SimpleNamespace(), - additional_features=None, - app_model_config_dict={ - "text_to_speech": {"enabled": False}, - "file_upload": {"enabled": False}, - "sensitive_word_avoidance": {"enabled": False}, - }, - ) - - monkeypatch.setattr(converter, "_get_new_app_mode", MagicMock(return_value=AppMode.WORKFLOW)) - monkeypatch.setattr(converter, "_convert_to_app_config", MagicMock(return_value=app_config)) - monkeypatch.setattr( - converter, - "_convert_to_start_node", - MagicMock( - return_value={"id": "start", "position": None, "data": {"type": BuiltinNodeTypes.START, "variables": []}} - ), - ) - monkeypatch.setattr(converter, "_convert_to_knowledge_retrieval_node", MagicMock(return_value=None)) - monkeypatch.setattr( - converter, - "_convert_to_llm_node", - MagicMock(return_value={"id": "llm", "position": None, "data": {"type": BuiltinNodeTypes.LLM}}), - ) - monkeypatch.setattr( - converter, - "_convert_to_end_node", - MagicMock(return_value={"id": "end", "position": None, "data": {"type": BuiltinNodeTypes.END}}), - ) - workflow = converter.convert_app_model_config_to_workflow( - app_model=app_model, - app_model_config=_app_model_config(id="cfg"), - account_id="account-1", - session=sqlite_session, - ) - - graph = json.loads(workflow.graph) - node_ids = [node["id"] for node in graph["nodes"]] - assert node_ids == ["start", "llm", "end"] - - features = json.loads(workflow.features) - assert set(features.keys()) == {"text_to_speech", "file_upload", "sensitive_word_avoidance"} - assert sqlite_session.scalar(select(Workflow).where(Workflow.id == workflow.id)) is workflow - - -def test_convert_to_app_config_should_route_to_correct_manager( - converter: WorkflowConverter, - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -) -> None: - agent_result = SimpleNamespace(kind="agent") - chat_result = SimpleNamespace(kind="chat") - completion_result = SimpleNamespace(kind="completion") - agent_get_app_config = MagicMock(return_value=agent_result) - chat_get_app_config = MagicMock(return_value=chat_result) - completion_get_app_config = MagicMock(return_value=completion_result) - load_annotation_reply = MagicMock(return_value={"enabled": False}) - monkeypatch.setattr(converter_module.AgentChatAppConfigManager, "get_app_config", agent_get_app_config) - monkeypatch.setattr(converter_module.ChatAppConfigManager, "get_app_config", chat_get_app_config) - monkeypatch.setattr(converter_module.CompletionAppConfigManager, "get_app_config", completion_get_app_config) - monkeypatch.setattr(converter_module, "load_annotation_reply_config", load_annotation_reply) - agent_mode_app = _app_model(id="app-1", mode=AppMode.AGENT_CHAT, app_model_config_id="cfg-1") - agent_flag_app = _app_model(id="app-2", mode=AppMode.CHAT, app_model_config_id="cfg-2") - chat_app = _app_model(id="app-3", mode=AppMode.CHAT, app_model_config_id="cfg-3") - completion_app = _app_model(id="app-4", mode=AppMode.COMPLETION, app_model_config_id="cfg-4") - agent_mode_config = _app_model_config(id="cfg-1", app_id="app-1") - agent_flag_config = _app_model_config( - id="cfg-2", app_id="app-2", agent_mode=json.dumps({"enabled": True, "strategy": "react"}) - ) - chat_config = _app_model_config(id="cfg-3", app_id="app-3") - completion_config = _app_model_config(id="cfg-4", app_id="app-4") - sqlite_session.add_all( - [ - agent_mode_app, - agent_flag_app, - chat_app, - completion_app, - agent_mode_config, - agent_flag_config, - chat_config, - completion_config, - ] - ) - sqlite_session.commit() - - from_agent_mode = converter._convert_to_app_config( - app_model=agent_mode_app, - app_model_config=agent_mode_config, - session=sqlite_session, - ) - from_agent_flag = converter._convert_to_app_config( - app_model=agent_flag_app, - app_model_config=agent_flag_config, - session=sqlite_session, - ) - from_chat_mode = converter._convert_to_app_config( - app_model=chat_app, - app_model_config=chat_config, - session=sqlite_session, - ) - from_completion_mode = converter._convert_to_app_config( - app_model=completion_app, - app_model_config=completion_config, - session=sqlite_session, - ) - - assert from_agent_mode is agent_result - assert from_agent_flag is agent_result - assert from_chat_mode is chat_result - assert from_completion_mode is completion_result - load_annotation_reply.assert_has_calls( - [ - call(sqlite_session, "app-1"), - call(sqlite_session, "app-2"), - call(sqlite_session, "app-3"), - call(sqlite_session, "app-4"), - ] - ) - assert all( - manager_call.kwargs["annotation_reply"] == {"enabled": False} - for manager in (agent_get_app_config, chat_get_app_config, completion_get_app_config) - for manager_call in manager.call_args_list - ) - - -def test_convert_to_app_config_should_raise_for_invalid_app_mode( - converter: WorkflowConverter, unbound_session: Session -) -> None: - app_model = _app_model(mode=AppMode.WORKFLOW) - - with pytest.raises(ValueError, match="Invalid app mode"): - converter._convert_to_app_config( - app_model=app_model, app_model_config=_app_model_config(id="cfg"), session=unbound_session - ) - - def test_convert_to_http_request_node_should_skip_non_api_and_missing_extension_id( converter: WorkflowConverter, - unbound_session: Session, ) -> None: app_model = _app_model(id="app-1", tenant_id="tenant-1", mode=AppMode.CHAT) external_data_variables = [ @@ -707,10 +356,11 @@ def test_convert_to_http_request_node_should_skip_non_api_and_missing_extension_ ] nodes, mapping = converter._convert_to_http_request_node( - app_model=app_model, + app_id=app_model.id, + app_mode=AppMode(app_model.mode), variables=[], external_data_variables=external_data_variables, - session=unbound_session, + extensions={}, ) assert nodes == [] @@ -856,53 +506,13 @@ def test_replace_template_variables_should_replace_start_and_external_references assert result == "Hello {{#start.name#}} from {{#start.city#}} with {{#code_1.result#}}" -def test_graph_helpers_should_create_edges_append_nodes_and_choose_mode(converter: WorkflowConverter) -> None: +def test_graph_helpers_should_create_edges_and_append_nodes(converter: WorkflowConverter) -> None: graph = {"nodes": [{"id": "start", "position": None, "data": {"type": BuiltinNodeTypes.START}}], "edges": []} node = {"id": "llm", "position": None, "data": {"type": BuiltinNodeTypes.LLM}} edge = converter._create_edge("start", "llm") updated_graph = converter._append_node(graph, node) - workflow_mode = converter._get_new_app_mode(_app_model(mode=AppMode.COMPLETION)) - advanced_chat_mode = converter._get_new_app_mode(_app_model(mode=AppMode.CHAT)) assert edge == {"id": "start-llm", "source": "start", "target": "llm"} assert updated_graph["nodes"][-1]["id"] == "llm" assert updated_graph["edges"][-1]["source"] == "start" - assert workflow_mode == AppMode.WORKFLOW - assert advanced_chat_mode == AppMode.ADVANCED_CHAT - - -def test_get_api_based_extension_should_raise_when_extension_not_found( - converter: WorkflowConverter, - sqlite_session: Session, -) -> None: - with pytest.raises(ValueError, match="API Based Extension not found"): - converter._get_api_based_extension(tenant_id="tenant-1", api_based_extension_id="ext-1", session=sqlite_session) - - -def test_get_api_based_extension_should_return_entity_when_found( - converter: WorkflowConverter, - sqlite_session: Session, -) -> None: - extension = APIBasedExtension( - tenant_id="tenant-1", - name="API extension", - api_key="encrypted", - api_endpoint="https://example.com", - ) - extension.id = "ext-1" - decoy = APIBasedExtension( - tenant_id="other-tenant", - name="Other tenant API extension", - api_key="encrypted", - api_endpoint="https://example.com", - ) - decoy.id = "ext-other" - sqlite_session.add_all([extension, decoy]) - sqlite_session.commit() - - result = converter._get_api_based_extension( - tenant_id="tenant-1", api_based_extension_id="ext-1", session=sqlite_session - ) - - assert result is extension diff --git a/api/tests/unit_tests/services/workflow/test_workflow_human_input_delivery.py b/api/tests/unit_tests/services/workflow/test_workflow_human_input_delivery.py index 7a7348ac8c1990..abde190669d327 100644 --- a/api/tests/unit_tests/services/workflow/test_workflow_human_input_delivery.py +++ b/api/tests/unit_tests/services/workflow/test_workflow_human_input_delivery.py @@ -1,255 +1,266 @@ -"""Tests for human-input delivery with a persisted draft workflow.""" +"""Exercise human input through the real console runtime and injected SQLite store.""" import json -import uuid -from unittest.mock import MagicMock +from uuid import uuid4 import pytest +from sqlalchemy import delete, select from sqlalchemy.orm import Session, sessionmaker -from graphon.enums import BuiltinNodeTypes -from models import Account -from models.human_input_delivery import ( - EmailDeliveryConfig, - EmailDeliveryMethod, - EmailRecipients, - ExternalRecipient, - MemberRecipient, +from extensions.application_services.human_input import build_human_input_debug_service +from factories.file_factory import builders +from graphon.variables import StringVariable +from models.human_input import HumanInputForm, HumanInputFormRecipient +from models.human_input_delivery import EmailDeliveryConfig, EmailDeliveryMethod, EmailRecipients, ExternalRecipient +from models.human_input_entities import ( + FileInputConfig, + HumanInputNodeData, + ParagraphInputConfig, + UserActionConfig, ) -from models.human_input_entities import HumanInputNodeData -from models.model import App, AppMode -from models.workflow import Workflow, WorkflowType -from services import workflow_service as workflow_service_module -from services.workflow_service import WorkflowService -from tests.unit_tests.model_factories import make_account, make_app - -TENANT_ID = "11111111-1111-1111-1111-111111111111" -APP_ID = "22222222-2222-2222-2222-222222222222" -ACCOUNT_ID = "33333333-3333-3333-3333-333333333333" - - -def _make_service(sqlite_session: Session) -> WorkflowService: - return WorkflowService( - session_maker=sessionmaker(bind=sqlite_session.get_bind(), expire_on_commit=False), +from models.workflow import Workflow, WorkflowDraftVariable +from repositories.human_input import form_repository +from services import human_input_delivery_test_service as delivery_module +from services import workflow_service as workflow_module +from services.entities.feature_entities import FeatureModel +from services.human_input.debug_service import HumanInputDebugService +from services.workflow.runtime_gateway import WorkflowRuntimeGateway +from tests.unit_tests.model_factories import make_upload_file +from tests.unit_tests.services.workflow.test_runtime_gateway import CONTEXT, RecordingSession, runtime_gateway # noqa: F401 + + +@pytest.fixture +def human_debug( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], +) -> tuple[HumanInputDebugService, list[RecordingSession]]: + gateway, sessions = runtime + return build_human_input_debug_service(database_client=gateway._sessions), sessions + + +def save_human_input_draft(factory: sessionmaker[Session], *, enabled: bool, debug: bool) -> str: + method = EmailDeliveryMethod( + id=uuid4(), + enabled=enabled, + config=EmailDeliveryConfig( + recipients=EmailRecipients( + include_bound_group=False, items=[ExternalRecipient(email="recipient@example.com")] + ), + subject="Test {{recipient_email}}", + body="{{form_content}} {{#url#}}", + debug_mode=debug, + ), ) - - -def _app() -> App: - return make_app( - app_id=APP_ID, - tenant_id=TENANT_ID, - mode=AppMode.WORKFLOW, - icon_type=None, - enable_site=False, - enable_api=False, + node = HumanInputNodeData( + title="Human Input", + delivery_methods=[method], + form_content="{{#conversation.greeting#}} {{#upstream.output#}}", + inputs=[ParagraphInputConfig(output_variable_name="answer")], + user_actions=[UserActionConfig(id="approve", title="Approve")], ) - - -def _account() -> Account: - return make_account(account_id=ACCOUNT_ID, name="Test User", email="test@example.com") - - -def _build_node_config(delivery_methods: list[EmailDeliveryMethod]) -> dict[str, object]: - return { - "id": "node-1", - "data": HumanInputNodeData( - title="Human Input", - type=BuiltinNodeTypes.HUMAN_INPUT, - delivery_methods=delivery_methods, - form_content="Test content", - inputs=[], - user_actions=[], - ), - } - - -def _persist_workflow(sqlite_session: Session, delivery_methods: list[EmailDeliveryMethod]) -> None: - node_config = _build_node_config(delivery_methods) - node_data = node_config["data"] - assert isinstance(node_data, HumanInputNodeData) - workflow = Workflow.new( - tenant_id=TENANT_ID, - app_id=APP_ID, - type=WorkflowType.WORKFLOW, - version=Workflow.VERSION_DRAFT, - graph=json.dumps( + with factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == Workflow.VERSION_DRAFT)) + assert draft is not None + draft.graph = json.dumps( { "nodes": [ { - "id": node_config["id"], - "data": node_data.model_dump(mode="json"), + "id": "human", + "data": { + **node.model_dump(mode="json"), + "type": "human-input", + }, } ], "edges": [], } - ), - features="{}", - created_by=ACCOUNT_ID, - environment_variables=[], - conversation_variables=[], - rag_pipeline_variables=[], - ) - sqlite_session.add(workflow) - sqlite_session.commit() - sqlite_session.expunge_all() - - -def _make_email_method(enabled: bool = True, debug_mode: bool = False) -> EmailDeliveryMethod: - return EmailDeliveryMethod( - id=uuid.uuid4(), - enabled=enabled, - config=EmailDeliveryConfig( - recipients=EmailRecipients( - include_bound_group=False, - items=[ExternalRecipient(email="tester@example.com")], - ), - subject="Test subject", - body="Test body", - debug_mode=debug_mode, - ), - ) - - -@pytest.mark.parametrize("sqlite_session", [(Workflow,)], indirect=True) -def test_human_input_delivery_requires_draft_workflow(sqlite_session: Session): - service = _make_service(sqlite_session) - app_model = _app() - account = _account() - - with pytest.raises(ValueError, match="Workflow not initialized"): - service.test_human_input_delivery( - app_model=app_model, - account=account, - node_id="node-1", - delivery_method_id="delivery-1", - session=sqlite_session, ) - assert sqlite_session.in_transaction() + draft.conversation_variables = [StringVariable(name="greeting", value="Hello")] + return str(method.id) -@pytest.mark.parametrize("sqlite_session", [(Workflow,)], indirect=True) -def test_human_input_delivery_allows_disabled_method( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): - service = _make_service(sqlite_session) - delivery_method = _make_email_method(enabled=False) - _persist_workflow(sqlite_session, [delivery_method]) - service._build_human_input_variable_pool = MagicMock(return_value=MagicMock()) # type: ignore[attr-defined] - node_stub = MagicMock() - node_stub.render_form_content_before_submission.return_value = "rendered" - node_stub.resolve_default_values.return_value = {} - service._build_human_input_node_for_debugging = MagicMock(return_value=node_stub) # type: ignore[attr-defined] - service._create_human_input_delivery_test_form = MagicMock( # type: ignore[attr-defined] - return_value=("form-1", []) - ) - - test_service_instance = MagicMock() - monkeypatch.setattr( - workflow_service_module, - "HumanInputDeliveryTestService", - MagicMock(return_value=test_service_instance), - ) +def forbid_global_database(monkeypatch: pytest.MonkeyPatch) -> None: + # Fail if any step falls back to a process-global database instead of the + # injected factory, including form persistence and email member lookup. + monkeypatch.setattr(workflow_module, "db", object()) + monkeypatch.setattr(delivery_module, "db", object()) + monkeypatch.setattr(form_repository, "session_factory", object()) + monkeypatch.setattr(builders, "session_factory", object()) - app_model = _app() - account = _account() - service.test_human_input_delivery( - app_model=app_model, - account=account, - node_id="node-1", - delivery_method_id=str(delivery_method.id), - session=sqlite_session, +def test_preview_and_submission_use_injected_database( + human_debug: tuple[HumanInputDebugService, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, sessions = human_debug + save_human_input_draft(sqlite_session_factory, enabled=True, debug=False) + forbid_global_database(monkeypatch) + preview = gateway.preview_form(CONTEXT, "app-1", "human", {"#upstream.output#": "world"}) + assert preview["form_content"] == "Hello world" + assert sessions + assert all(session.closed for session in sessions) + outputs = gateway.submit_form( + CONTEXT, + "app-1", + "human", + inputs={"#upstream.output#": "world"}, + form_inputs={"answer": "yes"}, + action="approve", ) - - test_service_instance.send_test.assert_called_once() - assert sqlite_session.in_transaction() + assert outputs["answer"] == "yes" + assert outputs["__action_value"] == "Approve" + assert all(session.closed for session in sessions) + with sqlite_session_factory() as session: + variables = session.scalars( + select(WorkflowDraftVariable).where( + WorkflowDraftVariable.app_id == "app-1", WorkflowDraftVariable.node_id == "human" + ) + ).all() + assert {variable.name: variable.get_value().value for variable in variables}["answer"] == "yes" + + +@pytest.mark.parametrize("submit", [False, True]) +def test_debug_secret_is_decrypted_after_repository_session_closes( + human_debug: tuple[HumanInputDebugService, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + submit: bool, +) -> None: + service, sessions = human_debug + save_human_input_draft(sqlite_session_factory, enabled=True, debug=False) + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == Workflow.VERSION_DRAFT)) + assert draft is not None + graph = dict(draft.graph_dict) + graph["nodes"][0]["data"]["form_content"] = "Token: {{#env.KEY#}}" + draft.graph = json.dumps(graph) + draft._environment_variables = json.dumps( + {"KEY": {"id": "secret-id", "name": "KEY", "value_type": "secret", "value": "wrapped"}} + ) + unwrapped: list[str] = [] + + def decrypt(*, tenant_id: str, token: str) -> str: + assert tenant_id == CONTEXT.active_workspace_id + assert sessions + assert all(session.closed and not session.in_transaction() for session in sessions) + unwrapped.append(token) + return "plaintext" + + monkeypatch.setattr("core.helper.encrypter.decrypt_token", decrypt) + if submit: + result = service.submit_form( + CONTEXT, "app-1", "human", inputs={}, form_inputs={"answer": "yes"}, action="approve" + ) + assert result["__rendered_content"] == "Token: plaintext" + else: + result = service.preview_form(CONTEXT, "app-1", "human", {}) + assert result["form_content"] == "Token: plaintext" + assert unwrapped == ["wrapped"] -@pytest.mark.parametrize("sqlite_session", [(Workflow,)], indirect=True) -def test_human_input_delivery_dispatches_to_test_service( +@pytest.mark.parametrize(("enabled", "debug"), [(True, False), (False, False), (True, True)]) +def test_email_delivery_commits_form_and_closes_sessions_before_external_send( + human_debug: tuple[HumanInputDebugService, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): - service = _make_service(sqlite_session) - delivery_method = _make_email_method(enabled=True) - _persist_workflow(sqlite_session, [delivery_method]) - service._build_human_input_variable_pool = MagicMock(return_value=MagicMock()) # type: ignore[attr-defined] - node_stub = MagicMock() - node_stub.render_form_content_before_submission.return_value = "rendered" - node_stub.resolve_default_values.return_value = {} - service._build_human_input_node_for_debugging = MagicMock(return_value=node_stub) # type: ignore[attr-defined] - service._create_human_input_delivery_test_form = MagicMock( # type: ignore[attr-defined] - return_value=("form-1", []) - ) - - test_service_instance = MagicMock() + enabled: bool, + debug: bool, +) -> None: + gateway, sessions = human_debug + method_id = save_human_input_draft(sqlite_session_factory, enabled=enabled, debug=debug) + forbid_global_database(monkeypatch) monkeypatch.setattr( - workflow_service_module, - "HumanInputDeliveryTestService", - MagicMock(return_value=test_service_instance), + delivery_module.FeatureService, + "get_features", + lambda *_args, **_kwargs: FeatureModel(human_input_email_delivery_enabled=True), ) - - app_model = _app() - account = _account() - - service.test_human_input_delivery( - app_model=app_model, - account=account, - node_id="node-1", - delivery_method_id=str(delivery_method.id), - inputs={"#node-1.output#": "value"}, - session=sqlite_session, + monkeypatch.setattr(delivery_module.mail, "is_inited", lambda: True) + sent: list[str] = [] + recipient = "test@example.com" if debug else "recipient@example.com" + + def send(*, to: str, subject: str, html: str) -> None: + assert sessions + assert all(session.closed and not session.in_transaction() for session in sessions) + # A separate reader sees the committed form and token before mail leaves. + with sqlite_session_factory() as session: + form = session.scalar(select(HumanInputForm).where(HumanInputForm.app_id == "app-1")) + assert form is not None + assert form.rendered_content == "Hello world" + saved_recipient = session.scalar( + select(HumanInputFormRecipient).where(HumanInputFormRecipient.form_id == form.id) + ) + assert saved_recipient is not None + assert saved_recipient.access_token + assert saved_recipient.access_token in html + assert to == recipient + assert subject == f"Test {recipient}" + assert "Hello world" in html + sent.append(to) + + monkeypatch.setattr(delivery_module.mail, "send", send) + gateway.test_delivery( + CONTEXT, "app-1", "human", inputs={"#upstream.output#": "world"}, delivery_method_id=method_id ) + assert sent == [recipient] - pool_args = service._build_human_input_variable_pool.call_args.kwargs - assert pool_args["manual_inputs"] == {"#node-1.output#": "value"} - test_service_instance.send_test.assert_called_once() - assert sqlite_session.in_transaction() +def test_human_input_requires_draft( + human_debug: tuple[HumanInputDebugService, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], +) -> None: + gateway, sessions = human_debug + with sqlite_session_factory.begin() as session: + session.execute(delete(Workflow)) + with pytest.raises(ValueError, match="Workflow not initialized"): + gateway.test_delivery(CONTEXT, "app-1", "human", inputs={}, delivery_method_id="missing") + assert all(session.closed for session in sessions) -@pytest.mark.parametrize("sqlite_session", [(Workflow,)], indirect=True) -def test_human_input_delivery_debug_mode_overrides_recipients( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): - service = _make_service(sqlite_session) - delivery_method = _make_email_method(enabled=True, debug_mode=True) - _persist_workflow(sqlite_session, [delivery_method]) - service._build_human_input_variable_pool = MagicMock(return_value=MagicMock()) # type: ignore[attr-defined] - node_stub = MagicMock() - node_stub.render_form_content_before_submission.return_value = "rendered" - node_stub.resolve_default_values.return_value = {} - service._build_human_input_node_for_debugging = MagicMock(return_value=node_stub) # type: ignore[attr-defined] - service._create_human_input_delivery_test_form = MagicMock( # type: ignore[attr-defined] - return_value=("form-1", []) - ) - test_service_instance = MagicMock() - monkeypatch.setattr( - workflow_service_module, - "HumanInputDeliveryTestService", - MagicMock(return_value=test_service_instance), +def test_file_preview_and_submission_load_upload_from_injected_database( + human_debug: tuple[HumanInputDebugService, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, sessions = human_debug + file_id = str(uuid4()) + node = HumanInputNodeData( + title="Attachment", + form_content="{{#upstream.file#}}", + inputs=[FileInputConfig(output_variable_name="attachment")], + user_actions=[UserActionConfig(id="approve", title="Approve")], ) - - app_model = _app() - account = _account() - - service.test_human_input_delivery( - app_model=app_model, - account=account, - node_id="node-1", - delivery_method_id=str(delivery_method.id), - session=sqlite_session, + with sqlite_session_factory.begin() as session: + session.add(make_upload_file(file_id=file_id)) + draft = session.scalar(select(Workflow).where(Workflow.version == Workflow.VERSION_DRAFT)) + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + { + "id": "human", + "data": { + **node.model_dump(mode="json"), + "type": "human-input", + }, + } + ], + "edges": [], + } + ) + forbid_global_database(monkeypatch) + file_mapping = {"type": "document", "transfer_method": "local_file", "upload_file_id": file_id} + preview = gateway.preview_form(CONTEXT, "app-1", "human", {"#upstream.file#": file_mapping}) + assert "test.txt" in str(preview["form_content"]) + outputs = gateway.submit_form( + CONTEXT, + "app-1", + "human", + inputs={"#upstream.file#": file_mapping}, + form_inputs={"attachment": file_mapping}, + action="approve", ) - - test_service_instance.send_test.assert_called_once() - sent_method = test_service_instance.send_test.call_args.kwargs["method"] - assert isinstance(sent_method, EmailDeliveryMethod) - assert sent_method.config.debug_mode is True - assert sent_method.config.recipients.include_bound_group is False - assert len(sent_method.config.recipients.items) == 1 - recipient = sent_method.config.recipients.items[0] - assert isinstance(recipient, MemberRecipient) - assert recipient.reference_id == account.id - assert sqlite_session.in_transaction() + attachment = outputs["attachment"] + assert isinstance(attachment, dict) + assert attachment["filename"] == "test.txt" + assert sessions + assert all(session.closed for session in sessions) diff --git a/api/tests/unit_tests/tasks/test_human_input_timeout_tasks.py b/api/tests/unit_tests/tasks/test_human_input_timeout_tasks.py index 90cbb0f43f41a3..01590c24d2bba4 100644 --- a/api/tests/unit_tests/tasks/test_human_input_timeout_tasks.py +++ b/api/tests/unit_tests/tasks/test_human_input_timeout_tasks.py @@ -9,10 +9,10 @@ from sqlalchemy.orm import Session, sessionmaker import core.db.session_factory as session_factory_module -from core.repositories.human_input_repository import HumanInputFormSubmissionRepository from enums.human_input import HumanInputFormKind, HumanInputFormStatus from models.human_input import HumanInputForm from models.human_input_entities import FormDefinition +from repositories.human_input.form_repository import HumanInputFormSubmissionRepository from tasks import human_input_timeout_tasks as task_module from tests.unit_tests.config_override import apply_config_overrides diff --git a/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py b/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py index e1911e3aa10086..05dcd2ab94bd24 100644 --- a/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py +++ b/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py @@ -10,10 +10,11 @@ import tasks.remove_app_and_related_data_task as remove_app_task_module from enums import DeploymentEdition +from enums.agent import WorkflowAgentBindingType from graphon.enums import WorkflowExecutionStatus from libs.archive_storage import ArchiveStorageNotConfiguredError from models import AppStar -from models.agent import WorkflowAgentBindingType, WorkflowAgentNodeBinding +from models.agent import WorkflowAgentNodeBinding from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom from models.workflow import WorkflowArchiveLog from tasks.remove_app_and_related_data_task import ( diff --git a/api/tests/unit_tests/tasks/test_resume_agent_app_task.py b/api/tests/unit_tests/tasks/test_resume_agent_app_task.py index d85376cd179190..76fd4079259174 100644 --- a/api/tests/unit_tests/tasks/test_resume_agent_app_task.py +++ b/api/tests/unit_tests/tasks/test_resume_agent_app_task.py @@ -24,8 +24,11 @@ @pytest.fixture -def task_session(mocker: MockerFixture, sqlite_session_factory: sessionmaker[Session]) -> Iterator[Session]: +def task_session( + mocker: MockerFixture, sqlite_session_factory: sessionmaker[Session], workflow_application +) -> Iterator[Session]: """Bind the task's Flask-SQLAlchemy session proxy to the shared SQLite database.""" + assert mod.application_services() is workflow_application registry = scoped_session(sqlite_session_factory) mocker.patch.object(mod.db, "session", registry) session = registry() @@ -115,7 +118,8 @@ def test_resume_happy_path_account_user_sets_tenant_and_runs(mocker: MockerFixtu assert call.kwargs["user"] is account assert call.kwargs["app_model"] is app assert call.kwargs["invoke_from"] == InvokeFrom.WEB_APP - assert isinstance(call.kwargs["session"], Session) + assert "session" not in call.kwargs + assert not task_session.in_transaction() assert account.current_tenant_id == tenant_id diff --git a/api/tests/unit_tests/tasks/test_retry_document_indexing_task.py b/api/tests/unit_tests/tasks/test_retry_document_indexing_task.py index 2adf8cd37e5b6e..4e7b43df6a0faa 100644 --- a/api/tests/unit_tests/tasks/test_retry_document_indexing_task.py +++ b/api/tests/unit_tests/tasks/test_retry_document_indexing_task.py @@ -4,16 +4,18 @@ import pytest from sqlalchemy.orm import Session -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType from enums import DeploymentEdition from models import Account, Tenant, TenantAccountJoin from models.account import TenantAccountRole from models.dataset import Dataset, Document from models.enums import DatasetRuntimeMode, DataSourceType, DocumentCreatedFrom, IndexingStatus +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator from tasks.retry_document_indexing_task import retry_document_indexing_task from tests.unit_tests.config_override import config_overrides_context +pytestmark = pytest.mark.usefixtures("workflow_application") + @config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) @pytest.mark.parametrize("runtime_mode", [DatasetRuntimeMode.GENERAL, DatasetRuntimeMode.RAG_PIPELINE]) diff --git a/api/tests/unit_tests/tasks/test_workflow_execute_task.py b/api/tests/unit_tests/tasks/test_workflow_execute_task.py index 51ab113cbb0476..c1f16054ce169f 100644 --- a/api/tests/unit_tests/tasks/test_workflow_execute_task.py +++ b/api/tests/unit_tests/tasks/test_workflow_execute_task.py @@ -23,6 +23,7 @@ from models.model import App, AppMode, Conversation, Message from models.workflow import Workflow, WorkflowRun, WorkflowType from repositories.sqlalchemy_api_workflow_run_repository import _WorkflowRunError +from services.workflow.variable_contracts import WorkflowExecutionVariables from tasks.app_generate import workflow_execute_task as workflow_execute_task_module from tasks.app_generate.workflow_execute_task import ( AppExecutionParams, @@ -69,6 +70,29 @@ def _single_event_generator(payload): yield payload +@pytest.mark.parametrize("app_mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]) +def test_worker_passes_its_binding_resolver_to_generation( + monkeypatch: pytest.MonkeyPatch, + sqlite_session_factory: sessionmaker[Session], + workflow_variables: WorkflowExecutionVariables, + app_mode: AppMode, +) -> None: + app = _make_app() + app.mode = app_mode + workflow, user = _make_workflow(), _make_account() + params = AppExecutionParams.new(app, workflow, user, {}, InvokeFrom.DEBUGGER) + runner = _AppRunner(sqlite_session_factory, params, variables=workflow_variables) + generator = MagicMock() + monkeypatch.setattr( + workflow_execute_task_module, + "WorkflowAppGenerator" if app_mode == AppMode.WORKFLOW else "AdvancedChatAppGenerator", + generator, + ) + runner._run_app(app=app, workflow=workflow, user=user, pause_state_config=MagicMock()) + assert generator.call_args.kwargs["runtime"] is runner._runtime + assert runner._runtime.agent_bindings._sessions is sqlite_session_factory + + def _decode_published_payload(payload: bytes) -> dict[str, object] | str: return json.loads(payload.decode()) @@ -344,13 +368,13 @@ def test_get_error_message(event: str | Mapping[str, object] | BaseModel, expect def mock_topic(monkeypatch: pytest.MonkeyPatch) -> MagicMock: topic = MagicMock() monkeypatch.setattr( - "tasks.app_generate.workflow_execute_task.MessageBasedAppGenerator.get_response_topic", + "tasks.app_generate.workflow_execute_task.WorkflowEventStream.get_response_topic", lambda *_args, **_kwargs: topic, ) return topic -def test_publish_streaming_response_with_uuid(mock_topic: MagicMock): +def test_publish_streaming_response_with_uuid(mock_topic: MagicMock, sqlite_session_factory: sessionmaker[Session]): workflow_run_id = uuid.uuid4() response_stream = iter( [ @@ -366,13 +390,19 @@ def test_publish_streaming_response_with_uuid(mock_topic: MagicMock): workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) payloads = _published_payloads(mock_topic) assert [payload["event"] for payload in payloads] == ["workflow_started", "workflow_finished"] -def test_publish_streaming_response_coerces_string_uuid(mock_topic: MagicMock): +def test_publish_streaming_response_coerces_string_uuid( + mock_topic: MagicMock, sqlite_session_factory: sessionmaker[Session] +): workflow_run_id = uuid.uuid4() response_stream = iter([{"event": "workflow_paused", "task_id": "task-id"}]) @@ -383,6 +413,10 @@ def test_publish_streaming_response_coerces_string_uuid(mock_topic: MagicMock): workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) payloads = _published_payloads(mock_topic) @@ -390,7 +424,7 @@ def test_publish_streaming_response_coerces_string_uuid(mock_topic: MagicMock): def test_publish_streaming_response_publishes_started_then_failed_terminal_when_iteration_raises( - mock_topic: MagicMock, + mock_topic: MagicMock, sqlite_session_factory: sessionmaker[Session] ): def _response_stream(): if False: @@ -405,6 +439,10 @@ def _response_stream(): workflow_id="workflow-id", inputs={"foo": "bar"}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) payloads = _published_payloads(mock_topic) @@ -416,8 +454,7 @@ def _response_stream(): def test_publish_streaming_response_recovers_when_workflow_started_publish_fails_first( - mock_topic: MagicMock, - caplog: pytest.LogCaptureFixture, + mock_topic: MagicMock, caplog: pytest.LogCaptureFixture, sqlite_session_factory: sessionmaker[Session] ): caplog.set_level(logging.ERROR, logger="tasks.app_generate.workflow_execute_task") response_stream = iter([{"event": "workflow_started", "task_id": "task-id"}]) @@ -444,6 +481,10 @@ def _publish(payload: bytes) -> None: workflow_id="workflow-id", inputs={"file": object()}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) assert [payload["event"] for payload in successful_payloads] == ["workflow_started", "workflow_finished"] @@ -457,8 +498,7 @@ def _publish(payload: bytes) -> None: def test_publish_streaming_response_publishes_failed_terminal_without_duplicate_started_on_publish_error( - mock_topic: MagicMock, - caplog: pytest.LogCaptureFixture, + mock_topic: MagicMock, caplog: pytest.LogCaptureFixture, sqlite_session_factory: sessionmaker[Session] ): caplog.set_level(logging.ERROR, logger="tasks.app_generate.workflow_execute_task") response_stream = iter( @@ -490,6 +530,10 @@ def _publish(payload: bytes) -> None: workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) assert [payload["event"] for payload in successful_payloads] == ["workflow_started", "workflow_finished"] @@ -501,8 +545,7 @@ def _publish(payload: bytes) -> None: def test_publish_streaming_response_recovers_when_workflow_finished_publish_fails_first( - mock_topic: MagicMock, - caplog: pytest.LogCaptureFixture, + mock_topic: MagicMock, caplog: pytest.LogCaptureFixture, sqlite_session_factory: sessionmaker[Session] ): caplog.set_level(logging.ERROR, logger="tasks.app_generate.workflow_execute_task") response_stream = iter( @@ -534,19 +577,21 @@ def _publish(payload: bytes) -> None: workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) assert [payload["event"] for payload in successful_payloads] == ["workflow_started", "workflow_finished"] assert successful_payloads[1]["task_id"] == "task-id" - assert successful_payloads[1]["data"]["status"] == WorkflowExecutionStatus.FAILED - assert successful_payloads[1]["data"]["error"] == "finished publish failed" + assert successful_payloads[1]["data"]["status"] == WorkflowExecutionStatus.SUCCEEDED assert "workflow-run-id" in caplog.text - assert "publishing fallback terminal event" in caplog.text + assert "Retrying engine terminal event publication" in caplog.text def test_publish_streaming_response_publishes_failed_terminal_on_exhaustion_without_terminal_event( - mock_topic: MagicMock, - caplog: pytest.LogCaptureFixture, + mock_topic: MagicMock, caplog: pytest.LogCaptureFixture, sqlite_session_factory: sessionmaker[Session] ): caplog.set_level(logging.WARNING, logger="tasks.app_generate.workflow_execute_task") response_stream = iter( @@ -567,6 +612,10 @@ def test_publish_streaming_response_publishes_failed_terminal_on_exhaustion_with workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) payloads = _published_payloads(mock_topic) @@ -578,7 +627,9 @@ def test_publish_streaming_response_publishes_failed_terminal_on_exhaustion_with assert "ended without a terminal event" in caplog.text -def test_publish_streaming_response_uses_error_message_for_failed_terminal(mock_topic: MagicMock): +def test_publish_streaming_response_uses_error_message_for_failed_terminal( + mock_topic: MagicMock, sqlite_session_factory: sessionmaker[Session] +): def response_stream() -> Generator[str | Mapping[str, object] | BaseModel, None, None]: yield { "event": "error", @@ -595,6 +646,10 @@ def response_stream() -> Generator[str | Mapping[str, object] | BaseModel, None, workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) payloads = _published_payloads(mock_topic) @@ -610,7 +665,9 @@ def response_stream() -> Generator[str | Mapping[str, object] | BaseModel, None, assert finished_data["error"] == "LLM provider and model are required." -def test_publish_streaming_response_does_not_publish_synthetic_failure_after_terminal_event(mock_topic: MagicMock): +def test_publish_streaming_response_does_not_publish_synthetic_failure_after_terminal_event( + mock_topic: MagicMock, sqlite_session_factory: sessionmaker[Session] +): response_stream = iter( [ { @@ -649,6 +706,10 @@ def test_publish_streaming_response_does_not_publish_synthetic_failure_after_ter workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) payloads = _published_payloads(mock_topic) @@ -659,6 +720,8 @@ def test_app_runner_streaming_failure_publishes_started_then_failed_workflow_fin mock_topic: MagicMock, monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, ): exec_params = AppExecutionParams( app_id="app-id", @@ -672,7 +735,7 @@ def test_app_runner_streaming_failure_publishes_started_then_failed_workflow_fin workflow_run_id="workflow-run-id", ) _persist_app_and_workflow(sqlite_session_factory) - runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params) + runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params, variables=workflow_variables) monkeypatch.setattr(runner, "_resolve_user", _make_account) monkeypatch.setattr(runner, "_setup_flask_context", lambda _user: nullcontext()) @@ -708,7 +771,7 @@ def test_app_runner_streaming_failure_publishes_started_then_failed_workflow_fin def test_app_runner_resolves_account_without_switching_tenant( - sqlite_session_factory: sessionmaker[Session], + sqlite_session_factory: sessionmaker[Session], *, workflow_variables: WorkflowExecutionVariables ): account_id = str(uuid.uuid4()) exec_params = AppExecutionParams( @@ -727,7 +790,7 @@ def test_app_runner_resolves_account_without_switching_tenant( account.id = account_id session.add(account) session.commit() - runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params) + runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params, variables=workflow_variables) resolved_user = runner._resolve_user() @@ -753,6 +816,8 @@ def test_app_runner_streaming_failure_keeps_existing_pre_runtime_helper_behavior mock_topic: MagicMock, monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, ): exec_params = AppExecutionParams( app_id="app-id", @@ -766,13 +831,13 @@ def test_app_runner_streaming_failure_keeps_existing_pre_runtime_helper_behavior workflow_run_id="workflow-run-id", ) _persist_app_and_workflow(sqlite_session_factory) - runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params) + runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params, variables=workflow_variables) monkeypatch.setattr(runner, "_resolve_user", _make_account) monkeypatch.setattr(runner, "_setup_flask_context", lambda _user: nullcontext()) monkeypatch.setattr(runner, "_run_app", lambda **_kwargs: (_ for _ in ()).throw(ValueError("Invalid upload file"))) monkeypatch.setattr( - "core.workflow.workflow_entry.WorkflowEntry.handle_special_values", + "services.workflow.execution.adapters.workflow_entry.WorkflowEntry.handle_special_values", lambda value: (_ for _ in ()).throw(AssertionError("pre-runtime helper should not normalize inputs")), ) @@ -787,6 +852,8 @@ def test_app_runner_streaming_failure_keeps_existing_pre_runtime_helper_behavior def test_app_runner_streaming_success_calls_publish_streaming_response_with_full_signature( monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, ): exec_params = AppExecutionParams( app_id="app-id", @@ -800,7 +867,7 @@ def test_app_runner_streaming_success_calls_publish_streaming_response_with_full workflow_run_id="workflow-run-id", ) _persist_app_and_workflow(sqlite_session_factory) - runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params) + runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params, variables=workflow_variables) response_stream = _single_event_generator({"event": "message"}) publish_streaming_response = MagicMock() @@ -821,6 +888,12 @@ def test_app_runner_streaming_success_calls_publish_streaming_response_with_full exec_params.workflow_id, exec_params.args.get("inputs", {}), WorkflowStartReason.INITIAL, + sessions=sqlite_session_factory, + tenant_id=exec_params.tenant_id, + app_id="app-id", + agent_snapshot=False, + on_finalized=runner._mark_execution_finalized, + on_terminal=None, ) @@ -828,6 +901,7 @@ def test_resume_app_execution_queries_message_by_conversation_and_workflow_run( monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], + workflow_variables: WorkflowExecutionVariables, ): workflow_run_id = "run-id" conversation_id = "conversation-id" @@ -869,7 +943,7 @@ def test_resume_app_execution_queries_message_by_conversation_and_workflow_run( monkeypatch.setattr("tasks.app_generate.workflow_execute_task._resume_advanced_chat", resume_advanced_chat) monkeypatch.setattr("tasks.app_generate.workflow_execute_task._resume_workflow", MagicMock()) - _resume_app_execution({"workflow_run_id": workflow_run_id}) + _resume_app_execution({"workflow_run_id": workflow_run_id}, variables=workflow_variables) workflow_run_repo.resume_workflow_pause.assert_called_once_with(workflow_run_id, pause_entity) resume_advanced_chat.assert_called_once() @@ -881,6 +955,7 @@ def test_resume_app_execution_returns_early_when_advanced_chat_missing_conversat monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], + workflow_variables: WorkflowExecutionVariables, ): workflow_run_id = "run-id" _persist_resumption_models(sqlite_session_factory, workflow_run_id=workflow_run_id) @@ -916,7 +991,7 @@ def test_resume_app_execution_returns_early_when_advanced_chat_missing_conversat resume_advanced_chat = MagicMock() monkeypatch.setattr("tasks.app_generate.workflow_execute_task._resume_advanced_chat", resume_advanced_chat) - _resume_app_execution({"workflow_run_id": workflow_run_id}) + _resume_app_execution({"workflow_run_id": workflow_run_id}, variables=workflow_variables) workflow_run_repo.resume_workflow_pause.assert_not_called() resume_advanced_chat.assert_not_called() @@ -926,6 +1001,7 @@ def test_resume_app_execution_clears_stale_cancellation_signals_before_resuming( monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], + workflow_variables: WorkflowExecutionVariables, ): """A resumed run reuses the paused task ID, so it must not inherit its cancellation signals. @@ -970,7 +1046,7 @@ def test_resume_app_execution_clears_stale_cancellation_signals_before_resuming( monkeypatch.setattr("tasks.app_generate.workflow_execute_task.clear_app_task_cancellation_signals", clear_signals) monkeypatch.setattr("tasks.app_generate.workflow_execute_task._resume_workflow", resume_workflow) - _resume_app_execution({"workflow_run_id": workflow_run_id}) + _resume_app_execution({"workflow_run_id": workflow_run_id}, variables=workflow_variables) clear_signals.assert_called_once_with(generate_entity.task_id) # Clearing after the engine started would let it observe the stale abort first. @@ -981,6 +1057,7 @@ def test_resume_app_execution_keeps_cancellation_signals_when_resume_is_abandone monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], + workflow_variables: WorkflowExecutionVariables, ): """No attempt is starting, so nothing may clear the signals guarding this task.""" workflow_run_id = "run-id" @@ -1019,15 +1096,16 @@ def test_resume_app_execution_keeps_cancellation_signals_when_resume_is_abandone monkeypatch.setattr("tasks.app_generate.workflow_execute_task.clear_app_task_cancellation_signals", clear_signals) monkeypatch.setattr("tasks.app_generate.workflow_execute_task._resume_advanced_chat", MagicMock()) - _resume_app_execution({"workflow_run_id": workflow_run_id}) + _resume_app_execution({"workflow_run_id": workflow_run_id}, variables=workflow_variables) clear_signals.assert_not_called() def test_resume_advanced_chat_publishes_events_for_originally_blocking_runs( monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, ): generate_entity = _build_advanced_chat_generate_entity(conversation_id="conversation-id") generate_entity.stream = False @@ -1038,7 +1116,7 @@ def test_resume_advanced_chat_publishes_events_for_originally_blocking_runs( generator_instance.resume.return_value = response_stream monkeypatch.setattr( "tasks.app_generate.workflow_execute_task.AdvancedChatAppGenerator", - lambda: generator_instance, + lambda **_dependencies: generator_instance, ) publish_streaming_response = MagicMock() @@ -1066,12 +1144,12 @@ def test_resume_advanced_chat_publishes_events_for_originally_blocking_runs( pause_state_config=MagicMock(), workflow_run_id="workflow-run-id", workflow_run=_make_workflow_run(), - session=sqlite_session, + variables=workflow_variables, ) resumed_entity = generator_instance.resume.call_args.kwargs["application_generate_entity"] assert resumed_entity.stream is True - assert generator_instance.resume.call_args.kwargs["session"] is sqlite_session + assert "session" not in generator_instance.resume.call_args.kwargs publish_streaming_response.assert_called_once_with( response_stream, "workflow-run-id", @@ -1079,12 +1157,18 @@ def test_resume_advanced_chat_publishes_events_for_originally_blocking_runs( workflow.id, generate_entity.inputs, WorkflowStartReason.RESUMPTION, + sessions=sqlite_session_factory, + tenant_id=workflow.tenant_id, + app_id=workflow.app_id, + agent_snapshot=False, ) def test_resume_workflow_publishes_events_for_originally_blocking_runs( monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, ): generate_entity = _build_workflow_generate_entity(stream=False) workflow = _make_workflow() @@ -1094,7 +1178,7 @@ def test_resume_workflow_publishes_events_for_originally_blocking_runs( generator_instance.resume.return_value = response_stream monkeypatch.setattr( "tasks.app_generate.workflow_execute_task.WorkflowAppGenerator", - lambda: generator_instance, + lambda **_dependencies: generator_instance, ) publish_streaming_response = MagicMock() @@ -1125,6 +1209,7 @@ def test_resume_workflow_publishes_events_for_originally_blocking_runs( workflow_run=_make_workflow_run(), workflow_run_repo=workflow_run_repo, pause_entity=pause_entity, + variables=workflow_variables, ) resumed_entity = generator_instance.resume.call_args.kwargs["application_generate_entity"] @@ -1136,6 +1221,11 @@ def test_resume_workflow_publishes_events_for_originally_blocking_runs( workflow.id, generate_entity.inputs, WorkflowStartReason.RESUMPTION, + sessions=sqlite_session_factory, + tenant_id=workflow.tenant_id, + app_id=workflow.app_id, + agent_snapshot=False, + on_terminal=None, ) workflow_run_repo.delete_workflow_pause.assert_called_once_with(pause_entity) @@ -1143,6 +1233,8 @@ def test_resume_workflow_publishes_events_for_originally_blocking_runs( def test_resume_workflow_ignores_missing_old_pause_after_repause( monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, ): generate_entity = _build_workflow_generate_entity(stream=False) workflow = _make_workflow() @@ -1152,7 +1244,7 @@ def test_resume_workflow_ignores_missing_old_pause_after_repause( generator_instance.resume.return_value = response_stream monkeypatch.setattr( "tasks.app_generate.workflow_execute_task.WorkflowAppGenerator", - lambda: generator_instance, + lambda **_dependencies: generator_instance, ) publish_streaming_response = MagicMock() @@ -1184,6 +1276,7 @@ def test_resume_workflow_ignores_missing_old_pause_after_repause( workflow_run=_make_workflow_run(), workflow_run_repo=workflow_run_repo, pause_entity=pause_entity, + variables=workflow_variables, ) publish_streaming_response.assert_called_once_with( @@ -1193,5 +1286,10 @@ def test_resume_workflow_ignores_missing_old_pause_after_repause( workflow.id, generate_entity.inputs, WorkflowStartReason.RESUMPTION, + sessions=sqlite_session_factory, + tenant_id=workflow.tenant_id, + app_id=workflow.app_id, + agent_snapshot=False, + on_terminal=None, ) workflow_run_repo.delete_workflow_pause.assert_called_once_with(pause_entity) diff --git a/api/tests/unit_tests/tools/test_api_tool.py b/api/tests/unit_tests/tools/test_api_tool.py index cc639db27ea30e..10789277de3bcf 100644 --- a/api/tests/unit_tests/tools/test_api_tool.py +++ b/api/tests/unit_tests/tools/test_api_tool.py @@ -65,9 +65,7 @@ def setup_method(self): ) @patch("core.tools.custom_tool.tool.ssrf_proxy.get") - def test_invoke_with_json_response_creates_text_message_with_serialized_json( - self, mock_get: Mock, database_session: Session - ) -> None: + def test_invoke_with_json_response_creates_text_message_with_serialized_json(self, mock_get: Mock) -> None: """Test that when upstream returns JSON, the output Text message contains JSON-serialized string.""" # Use the response's real JSON parsing and content decoding. json_response_data = { @@ -78,7 +76,7 @@ def test_invoke_with_json_response_creates_text_message_with_serialized_json( mock_get.return_value = httpx.Response(200, json=json_response_data) # Invoke the tool - result_generator = self.api_tool._invoke(session=database_session, user_id="test_user", tool_parameters={}) + result_generator = self.api_tool._invoke(user_id="test_user", tool_parameters={}) # Get the result from the generator result = list(result_generator) @@ -139,7 +137,7 @@ def test_invoke_with_json_response_creates_text_message_with_serialized_json( ids=operator.itemgetter(0), ) def test_invoke_with_non_dict_json_response_creates_text_message_with_serialized_json( - self, mock_get: Mock, test_case, database_session: Session + self, mock_get: Mock, test_case ) -> None: """Test that when upstream returns a non-dict JSON, the output Text message contains JSON-serialized string.""" # Explicit content also covers JSON null, which is distinct from no body. @@ -149,7 +147,7 @@ def test_invoke_with_non_dict_json_response_creates_text_message_with_serialized ) # Invoke the tool - result_generator = self.api_tool._invoke(session=database_session, user_id="test_user", tool_parameters={}) + result_generator = self.api_tool._invoke(user_id="test_user", tool_parameters={}) # Get the result from the generator result = list(result_generator) @@ -173,16 +171,14 @@ def test_invoke_with_non_dict_json_response_creates_text_message_with_serialized assert json_message is None, "_invoke should not yield a JSON message for JSON array response" @patch("core.tools.custom_tool.tool.ssrf_proxy.get") - def test_invoke_with_text_response_creates_text_message_with_original_text( - self, mock_get: Mock, database_session: Session - ) -> None: + def test_invoke_with_text_response_creates_text_message_with_original_text(self, mock_get: Mock) -> None: """Test that when upstream returns plain text, the output Text message contains the original text.""" # Plain text exercises the response's real JSON decoding failure. text_response_data = "This is a plain text response" mock_get.return_value = httpx.Response(200, text=text_response_data) # Invoke the tool - result_generator = self.api_tool._invoke(session=database_session, user_id="test_user", tool_parameters={}) + result_generator = self.api_tool._invoke(user_id="test_user", tool_parameters={}) # Get the result from the generator result = list(result_generator) @@ -198,12 +194,12 @@ def test_invoke_with_text_response_creates_text_message_with_original_text( assert message.message.text == text_response_data @patch("core.tools.custom_tool.tool.ssrf_proxy.get") - def test_invoke_with_empty_response(self, mock_get: Mock, database_session: Session) -> None: + def test_invoke_with_empty_response(self, mock_get: Mock) -> None: """Test that empty responses are handled correctly.""" mock_get.return_value = httpx.Response(200, content=b"", headers={"content-type": "application/json"}) # Invoke the tool - result_generator = self.api_tool._invoke(session=database_session, user_id="test_user", tool_parameters={}) + result_generator = self.api_tool._invoke(user_id="test_user", tool_parameters={}) # Get the result from the generator result = list(result_generator) @@ -219,11 +215,11 @@ def test_invoke_with_empty_response(self, mock_get: Mock, database_session: Sess assert "Empty response from the tool" in message.message.text @patch("core.tools.custom_tool.tool.ssrf_proxy.get") - def test_invoke_with_error_response(self, mock_get: Mock, database_session: Session) -> None: + def test_invoke_with_error_response(self, mock_get: Mock) -> None: """Test that error responses are handled correctly.""" mock_get.return_value = httpx.Response(404, text="Not Found") - result_generator = self.api_tool._invoke(session=database_session, user_id="test_user", tool_parameters={}) + result_generator = self.api_tool._invoke(user_id="test_user", tool_parameters={}) # Invoke the tool and expect an error with pytest.raises(Exception) as exc_info: diff --git a/api/tests/unit_tests/tools/test_mcp_tool.py b/api/tests/unit_tests/tools/test_mcp_tool.py index e841bd03c501af..16f3e8716958cc 100644 --- a/api/tests/unit_tests/tools/test_mcp_tool.py +++ b/api/tests/unit_tests/tools/test_mcp_tool.py @@ -65,7 +65,7 @@ class TestMCPToolInvoke: ), ], ) - def test_invoke_image_or_audio_yields_blob(self, content_factory, mime_type, orm_session: Session) -> None: + def test_invoke_image_or_audio_yields_blob(self, content_factory, mime_type) -> None: tool = _make_mcp_tool() raw = b"\x00\x01test-bytes\x02" b64 = base64.b64encode(raw).decode() @@ -73,7 +73,7 @@ def test_invoke_image_or_audio_yields_blob(self, content_factory, mime_type, orm result = CallToolResult(content=[content]) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - messages = list(tool._invoke(session=orm_session, user_id="test_user", tool_parameters={})) + messages = list(tool._invoke(user_id="test_user", tool_parameters={})) assert len(messages) == 1 msg = messages[0] @@ -82,14 +82,14 @@ def test_invoke_image_or_audio_yields_blob(self, content_factory, mime_type, orm assert msg.message.blob == raw assert msg.meta == {"mime_type": mime_type} - def test_invoke_embedded_text_resource_yields_text(self, orm_session: Session) -> None: + def test_invoke_embedded_text_resource_yields_text(self) -> None: tool = _make_mcp_tool() text_resource = TextResourceContents(uri="file://test.txt", mimeType="text/plain", text="hello world") content = EmbeddedResource(type="resource", resource=text_resource) result = CallToolResult(content=[content]) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - messages = list(tool._invoke(session=orm_session, user_id="test_user", tool_parameters={})) + messages = list(tool._invoke(user_id="test_user", tool_parameters={})) assert len(messages) == 1 msg = messages[0] @@ -101,7 +101,7 @@ def test_invoke_embedded_text_resource_yields_text(self, orm_session: Session) - ("mime_type", "expected_mime"), [("application/pdf", "application/pdf"), (None, "application/octet-stream")], ) - def test_invoke_embedded_blob_resource_yields_blob(self, mime_type, expected_mime, orm_session: Session) -> None: + def test_invoke_embedded_blob_resource_yields_blob(self, mime_type, expected_mime) -> None: tool = _make_mcp_tool() raw = b"binary-data" b64 = base64.b64encode(raw).decode() @@ -110,7 +110,7 @@ def test_invoke_embedded_blob_resource_yields_blob(self, mime_type, expected_mim result = CallToolResult(content=[content]) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - messages = list(tool._invoke(session=orm_session, user_id="test_user", tool_parameters={})) + messages = list(tool._invoke(user_id="test_user", tool_parameters={})) assert len(messages) == 1 msg = messages[0] @@ -119,12 +119,12 @@ def test_invoke_embedded_blob_resource_yields_blob(self, mime_type, expected_mim assert msg.message.blob == raw assert msg.meta == {"mime_type": expected_mime} - def test_invoke_yields_variables_when_structured_content_and_schema(self, orm_session: Session) -> None: + def test_invoke_yields_variables_when_structured_content_and_schema(self) -> None: tool = _make_mcp_tool(output_schema={"type": "object"}) result = CallToolResult(content=[], structuredContent={"a": 1, "b": "x"}) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - messages = list(tool._invoke(session=orm_session, user_id="test_user", tool_parameters={})) + messages = list(tool._invoke(user_id="test_user", tool_parameters={})) # Expect two variable messages corresponding to keys a and b assert len(messages) == 2 @@ -134,12 +134,12 @@ def test_invoke_yields_variables_when_structured_content_and_schema(self, orm_se values = {m.message.variable_name: m.message.variable_value for m in var_msgs} assert values == {"a": 1, "b": "x"} - def test_invoke_yields_json_when_structured_content_has_no_output_schema(self, orm_session: Session) -> None: + def test_invoke_yields_json_when_structured_content_has_no_output_schema(self) -> None: tool = _make_mcp_tool() result = CallToolResult(content=[], structuredContent={"a": 1, "b": "x"}) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - messages = list(tool._invoke(session=orm_session, user_id="test_user", tool_parameters={})) + messages = list(tool._invoke(user_id="test_user", tool_parameters={})) assert len(messages) == 1 msg = messages[0] @@ -288,7 +288,7 @@ def test_derive_usage_from_result_calculates_total_tokens(self) -> None: assert usage.prompt_tokens == 100 assert usage.completion_tokens == 50 - def test_invoke_sets_latest_usage_from_meta(self, orm_session: Session) -> None: + def test_invoke_sets_latest_usage_from_meta(self) -> None: """Test that _invoke sets _latest_usage from result meta.""" tool = _make_mcp_tool() meta = { @@ -303,7 +303,7 @@ def test_invoke_sets_latest_usage_from_meta(self, orm_session: Session) -> None: result = CallToolResult(content=[TextContent(type="text", text="test")], _meta=meta) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - list(tool._invoke(session=orm_session, user_id="test_user", tool_parameters={})) + list(tool._invoke(user_id="test_user", tool_parameters={})) # Verify latest_usage was set correctly assert tool.latest_usage.prompt_tokens == 200 @@ -311,13 +311,13 @@ def test_invoke_sets_latest_usage_from_meta(self, orm_session: Session) -> None: assert tool.latest_usage.total_tokens == 300 assert tool.latest_usage.total_price == Decimal("0.003") - def test_invoke_with_no_meta_returns_empty_usage(self, orm_session: Session) -> None: + def test_invoke_with_no_meta_returns_empty_usage(self) -> None: """Test that _invoke returns empty usage when no meta is present.""" tool = _make_mcp_tool() result = CallToolResult(content=[TextContent(type="text", text="test")], _meta=None) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - list(tool._invoke(session=orm_session, user_id="test_user", tool_parameters={})) + list(tool._invoke(user_id="test_user", tool_parameters={})) # Verify latest_usage is empty assert tool.latest_usage.total_tokens == 0 From 14c772fdf19eb45b32faa9c9847bc13fadde1963 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:51:44 +0000 Subject: [PATCH 02/20] [autofix.ci] apply automated fixes --- api/openapi/markdown/console-openapi.md | 94 +++++++------------ api/openapi/markdown/service-openapi.md | 2 +- api/openapi/markdown/web-openapi.md | 2 +- .../generated/api/console/apps/orpc.gen.ts | 94 ++----------------- .../generated/api/console/apps/types.gen.ts | 12 +-- .../generated/api/console/apps/zod.gen.ts | 8 +- .../generated/api/console/rag/orpc.gen.ts | 30 +++--- .../generated/api/console/rag/types.gen.ts | 44 ++++++--- .../generated/api/console/rag/zod.gen.ts | 45 +++++++-- .../api/console/snippets/orpc.gen.ts | 4 +- 10 files changed, 136 insertions(+), 199 deletions(-) diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md index 2f4d95bc57d21f..759c421256b5af 100644 --- a/api/openapi/markdown/console-openapi.md +++ b/api/openapi/markdown/console-openapi.md @@ -1730,8 +1730,6 @@ Get advanced chat workflow run list | 200 | Workflow runs count retrieved successfully | **application/json**: [WorkflowRunCountResponse](#workflowruncountresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/preview -**Preview human input form content and placeholders** - Get human input form preview for advanced chat workflow #### Parameters @@ -1754,8 +1752,6 @@ Get human input form preview for advanced chat workflow | 200 | Human input form preview | **application/json**: [HumanInputFormPreviewResponse](#humaninputformpreviewresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/run -**Submit human input form preview** - Submit human input form preview for advanced chat workflow #### Parameters @@ -1778,8 +1774,6 @@ Submit human input form preview for advanced chat workflow | 200 | Human input form submission result | **application/json**: [HumanInputFormSubmitResponse](#humaninputformsubmitresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/iteration/nodes/{node_id}/run -**Run draft workflow iteration node** - Run draft workflow iteration node for advanced chat #### Parameters @@ -1804,8 +1798,6 @@ Run draft workflow iteration node for advanced chat | 404 | Node not found | | ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/loop/nodes/{node_id}/run -**Run draft workflow loop node** - Run draft workflow loop node for advanced chat #### Parameters @@ -1830,8 +1822,6 @@ Run draft workflow loop node for advanced chat | 404 | Node not found | | ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/run -**Run draft workflow** - Run draft workflow for advanced chat application #### Parameters @@ -2650,11 +2640,7 @@ Get conversation variables for an application | 200 | Conversation variables retrieved successfully | **application/json**: [PaginatedConversationVariableResponse](#paginatedconversationvariableresponse)
| ### [POST] /apps/{app_id}/convert-to-workflow -**Convert basic mode of chatbot app to workflow mode** - Convert application to workflow mode -Convert expert mode of chatbot app to workflow mode -Convert Completion App to Workflow App #### Parameters @@ -3443,8 +3429,6 @@ Get workflow application execution logs | 200 | Workflow runs count retrieved successfully | **application/json**: [WorkflowRunCountResponse](#workflowruncountresponse)
| ### [POST] /apps/{app_id}/workflow-runs/tasks/{task_id}/stop -**Stop workflow task** - Stop running workflow task #### Parameters @@ -3809,8 +3793,6 @@ Get workflow daily token cost statistics | 200 | Daily token cost statistics retrieved successfully | **application/json**: [WorkflowDailyTokenCostStatisticResponse](#workflowdailytokencoststatisticresponse)
| ### [GET] /apps/{app_id}/workflows -**Get published workflows** - Get all published workflows for an application #### Parameters @@ -3830,8 +3812,6 @@ Get all published workflows for an application | 200 | Published workflows retrieved successfully | **application/json**: [WorkflowPaginationResponse](#workflowpaginationresponse)
| ### [GET] /apps/{app_id}/workflows/default-workflow-block-configs -**Get default block config** - Get default block configurations for workflow #### Parameters @@ -3847,8 +3827,6 @@ Get default block configurations for workflow | 200 | Default block configurations retrieved successfully | **application/json**: [DefaultBlockConfigsResponse](#defaultblockconfigsresponse)
| ### [GET] /apps/{app_id}/workflows/default-workflow-block-configs/{block_type} -**Get default block config** - Get default block configuration by type #### Parameters @@ -3867,8 +3845,6 @@ Get default block configuration by type | 404 | Block type not found | | ### [GET] /apps/{app_id}/workflows/draft -**Get draft workflow** - Get draft workflow for an application #### Parameters @@ -3885,8 +3861,6 @@ Get draft workflow for an application | 404 | Draft workflow not found | | ### [POST] /apps/{app_id}/workflows/draft -**Sync draft workflow** - Sync draft workflow configuration #### Parameters @@ -3947,7 +3921,7 @@ Update conversation variables for workflow draft | 200 | Conversation variables updated successfully | **application/json**: [SimpleResultResponse](#simpleresultresponse)
| ### [GET] /apps/{app_id}/workflows/draft/environment-variables -**Get draft workflow** +**Get environment variables** Get environment variables for workflow @@ -4007,8 +3981,6 @@ Update draft workflow features | 200 | Workflow features updated successfully | **application/json**: [SimpleResultResponse](#simpleresultresponse)
| ### [POST] /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/delivery-test -**Test human input delivery** - Test human input delivery for workflow #### Parameters @@ -4031,8 +4003,6 @@ Test human input delivery for workflow | 200 | Human input delivery test result | **application/json**: [EmptyObjectResponse](#emptyobjectresponse)
| ### [POST] /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/preview -**Preview human input form content and placeholders** - Get human input form preview for workflow #### Parameters @@ -4055,8 +4025,6 @@ Get human input form preview for workflow | 200 | Human input form preview | **application/json**: [HumanInputFormPreviewResponse](#humaninputformpreviewresponse)
| ### [POST] /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/run -**Submit human input form preview** - Submit human input form preview for workflow #### Parameters @@ -4079,7 +4047,7 @@ Submit human input form preview for workflow | 200 | Human input form submission result | **application/json**: [HumanInputFormSubmitResponse](#humaninputformsubmitresponse)
| ### [POST] /apps/{app_id}/workflows/draft/iteration/nodes/{node_id}/run -**Run draft workflow iteration node** +Run draft workflow iteration node #### Parameters @@ -4103,7 +4071,7 @@ Submit human input form preview for workflow | 404 | Node not found | | ### [POST] /apps/{app_id}/workflows/draft/loop/nodes/{node_id}/run -**Run draft workflow loop node** +Run draft workflow loop node #### Parameters @@ -4274,7 +4242,7 @@ Get last run result for draft workflow node | 404 | Node last run not found | | ### [POST] /apps/{app_id}/workflows/draft/nodes/{node_id}/run -**Run draft workflow node** +Run draft workflow node #### Parameters @@ -4298,7 +4266,7 @@ Get last run result for draft workflow node | 404 | Node not found | | ### [POST] /apps/{app_id}/workflows/draft/nodes/{node_id}/trigger/run -**Poll for trigger events and execute single node when event arrives** +Poll for trigger events and execute single node when event arrives #### Parameters @@ -4348,7 +4316,7 @@ Get variables for a specific node | 200 | Node variables retrieved successfully | **application/json**: [WorkflowDraftVariableListResponse](#workflowdraftvariablelistresponse)
| ### [POST] /apps/{app_id}/workflows/draft/run -**Run draft workflow** +Run draft workflow #### Parameters @@ -4456,7 +4424,7 @@ Get system variables for workflow | 200 | System variables retrieved successfully | **application/json**: [WorkflowDraftVariableListResponse](#workflowdraftvariablelistresponse)
| ### [POST] /apps/{app_id}/workflows/draft/trigger/run -**Poll for trigger events and execute full workflow when event arrives** +Poll for trigger events and execute full workflow when event arrives #### Parameters @@ -4479,7 +4447,7 @@ Get system variables for workflow | 500 | Internal server error | | ### [POST] /apps/{app_id}/workflows/draft/trigger/run-all -**Full workflow debug when the start node is a trigger** +Full workflow debug when the start node is a trigger #### Parameters @@ -4517,7 +4485,7 @@ Delete all draft workflow variables | 204 | Workflow variables deleted successfully | ### [GET] /apps/{app_id}/workflows/draft/variables -**Get draft workflow** +**List draft workflow variables without loading their values** Get draft workflow variables @@ -4611,8 +4579,6 @@ Reset a workflow variable to its default value | 404 | Variable not found | | ### [GET] /apps/{app_id}/workflows/publish -**Get published workflow** - Get published workflow for an application #### Parameters @@ -4628,8 +4594,6 @@ Get published workflow for an application | 200 | Published workflow retrieved successfully, or null if not found | **application/json**: [WorkflowResponse](#workflowresponse)
| ### [POST] /apps/{app_id}/workflows/publish -**Publish workflow** - #### Parameters | Name | Located in | Description | Required | Schema | @@ -4736,8 +4700,6 @@ Full value for one declared output of a published run. | 200 | Success | **application/json**: [WebhookTriggerResponse](#webhooktriggerresponse)
| ### [DELETE] /apps/{app_id}/workflows/{workflow_id} -**Delete workflow** - #### Parameters | Name | Located in | Description | Required | Schema | @@ -4749,11 +4711,9 @@ Full value for one declared output of a published run. | Code | Description | | ---- | ----------- | -| 200 | Success | +| 204 | Workflow deleted successfully | ### [PATCH] /apps/{app_id}/workflows/{workflow_id} -**Update workflow attributes** - Update workflow by ID #### Parameters @@ -7827,11 +7787,17 @@ Update account-level Step-by-step Tour state | ---- | ---------- | ----------- | -------- | ------ | | pipeline_id | path | | Yes | string (uuid) | +#### Request Body + +| Required | Schema | +| -------- | ------ | +| Yes | **application/json**: [DraftWorkflowSyncPayload](#draftworkflowsyncpayload)
| + #### Responses -| Code | Description | -| ---- | ----------- | -| 200 | Success | +| Code | Description | Schema | +| ---- | ----------- | ------ | +| 200 | Success | **application/json**: [RagPipelineWorkflowSyncResponse](#ragpipelineworkflowsyncresponse)
| ### [POST] /rag/pipelines/{pipeline_id}/workflows/draft/datasource/nodes/{node_id}/run **Run rag pipeline datasource** @@ -7877,7 +7843,7 @@ Update account-level Step-by-step Tour state | 200 | Datasource variables set successfully | **application/json**: [WorkflowRunNodeExecutionResponse](#workflowrunnodeexecutionresponse)
| ### [GET] /rag/pipelines/{pipeline_id}/workflows/draft/environment-variables -**Get draft workflow** +**Get environment variables** #### Parameters @@ -8079,7 +8045,7 @@ Update account-level Step-by-step Tour state | 204 | Workflow variables deleted successfully | ### [GET] /rag/pipelines/{pipeline_id}/workflows/draft/variables -**Get draft workflow** +**List draft pipeline variables without loading their values** #### Parameters @@ -8131,6 +8097,12 @@ Update account-level Step-by-step Tour state | pipeline_id | path | | Yes | string (uuid) | | variable_id | path | | Yes | string (uuid) | +#### Request Body + +| Required | Schema | +| -------- | ------ | +| Yes | **application/json**: [WorkflowDraftVariablePatchPayload](#workflowdraftvariablepatchpayload)
| + #### Responses | Code | Description | Schema | @@ -8168,7 +8140,7 @@ Update account-level Step-by-step Tour state | 200 | Published workflow retrieved successfully, or null if not exist | **application/json**: [WorkflowResponse](#workflowresponse)
| ### [POST] /rag/pipelines/{pipeline_id}/workflows/publish -**Publish workflow** +**Publish a prepared Pipeline and its dataset settings atomically** #### Parameters @@ -8280,8 +8252,6 @@ Update account-level Step-by-step Tour state | 200 | Success | **application/json**: [RagPipelineOpaqueResponse](#ragpipelineopaqueresponse)
| ### [DELETE] /rag/pipelines/{pipeline_id}/workflows/{workflow_id} -**Delete a published workflow version that is not currently active on the pipeline** - #### Parameters | Name | Located in | Description | Required | Schema | @@ -9074,7 +9044,9 @@ Reset a draft workflow variable to its default value (snippet scope) | 400 | No draft workflow found | | ### [DELETE] /snippets/{snippet_id}/workflows/{workflow_id} -**Delete a published snippet workflow version** +**Delete a version and retire its unowned Agents through the shared use case** + +Delete a published snippet workflow version #### Parameters @@ -15533,10 +15505,10 @@ Legacy Chat App model config used only for follow-up question generation. | status | string | | Yes | | time_cost | number
integer | | Yes | | tool_icon | | | No | -| tool_input | object | | Yes | +| tool_input | | | Yes | | tool_label | string | | Yes | | tool_name | string | | Yes | -| tool_output | object | | Yes | +| tool_output | | | Yes | | tool_parameters | object | | Yes | #### AgentUserSatisfactionRateStatisticResponse diff --git a/api/openapi/markdown/service-openapi.md b/api/openapi/markdown/service-openapi.md index a30ee05de6fa0a..86f95c03781c20 100644 --- a/api/openapi/markdown/service-openapi.md +++ b/api/openapi/markdown/service-openapi.md @@ -1018,7 +1018,7 @@ Execute the full knowledge pipeline for a knowledge base. Published runs are que | 200 | Pipeline execution result. Published runs return a JSON object containing `batch`, `dataset`, and `documents`. Draft runs return `text/event-stream` for streaming mode or a workflow result JSON object for blocking mode. | **application/json**: [PipelineRunJsonResponse](#pipelinerunjsonresponse)
**text/event-stream**: string
| | 400 | Bad request - invalid payload or pipeline is not configured | | | 401 | Unauthorized - invalid API token | | -| 403 | `forbidden` : Forbidden. | | +| 403 | `forbidden` : AccessDeniedError. | | | 404 | `not_found` : Dataset not found. | | | 500 | `pipeline_run_error` : Pipeline execution failed. | | diff --git a/api/openapi/markdown/web-openapi.md b/api/openapi/markdown/web-openapi.md index 1f6d7a8e723090..cdd295149db3c4 100644 --- a/api/openapi/markdown/web-openapi.md +++ b/api/openapi/markdown/web-openapi.md @@ -361,7 +361,7 @@ GET /api/form/human_input/ | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Form retrieved successfully | **application/json**: [HumanInputFormDefinitionResponse](#humaninputformdefinitionresponse)
| -| 403 | Forbidden | | +| 403 | AccessDeniedError | | | 404 | Form not found | | | 412 | Form already submitted or expired | | | 429 | Too many requests | | diff --git a/packages/contracts/generated/api/console/apps/orpc.gen.ts b/packages/contracts/generated/api/console/apps/orpc.gen.ts index 8df754ae8f6ee2..8197443bd438b7 100644 --- a/packages/contracts/generated/api/console/apps/orpc.gen.ts +++ b/packages/contracts/generated/api/console/apps/orpc.gen.ts @@ -632,8 +632,6 @@ export const workflowRuns = { } /** - * Preview human input form content and placeholders - * * Get human input form preview for advanced chat workflow */ export const post4 = oc @@ -643,7 +641,6 @@ export const post4 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormPreview', path: '/apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/preview', - summary: 'Preview human input form content and placeholders', tags: ['console'], }) .input( @@ -659,8 +656,6 @@ export const preview = { } /** - * Submit human input form preview - * * Submit human input form preview for advanced chat workflow */ export const post5 = oc @@ -670,7 +665,6 @@ export const post5 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/run', - summary: 'Submit human input form preview', tags: ['console'], }) .input( @@ -703,8 +697,6 @@ export const humanInput = { } /** - * Run draft workflow iteration node - * * Run draft workflow iteration node for advanced chat */ export const post6 = oc @@ -714,7 +706,6 @@ export const post6 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftIterationNodesByNodeIdRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/iteration/nodes/{node_id}/run', - summary: 'Run draft workflow iteration node', tags: ['console'], }) .input( @@ -742,8 +733,6 @@ export const iteration = { } /** - * Run draft workflow loop node - * * Run draft workflow loop node for advanced chat */ export const post7 = oc @@ -753,7 +742,6 @@ export const post7 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftLoopNodesByNodeIdRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/loop/nodes/{node_id}/run', - summary: 'Run draft workflow loop node', tags: ['console'], }) .input( @@ -781,8 +769,6 @@ export const loop = { } /** - * Run draft workflow - * * Run draft workflow for advanced chat application */ export const post8 = oc @@ -792,7 +778,6 @@ export const post8 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/run', - summary: 'Run draft workflow', tags: ['console'], }) .input( @@ -1750,21 +1735,15 @@ export const conversationVariables = { } /** - * Convert basic mode of chatbot app to workflow mode - * * Convert application to workflow mode - * Convert expert mode of chatbot app to workflow mode - * Convert Completion App to Workflow App */ export const post21 = oc .route({ - description: - 'Convert application to workflow mode\nConvert expert mode of chatbot app to workflow mode\nConvert Completion App to Workflow App', + description: 'Convert application to workflow mode', inputStructure: 'detailed', method: 'POST', operationId: 'postAppsByAppIdConvertToWorkflow', path: '/apps/{app_id}/convert-to-workflow', - summary: 'Convert basic mode of chatbot app to workflow mode', tags: ['console'], }) .input( @@ -2600,8 +2579,6 @@ export const count3 = { } /** - * Stop workflow task - * * Stop running workflow task */ export const post38 = oc @@ -2611,7 +2588,6 @@ export const post38 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowRunsTasksByTaskIdStop', path: '/apps/{app_id}/workflow-runs/tasks/{task_id}/stop', - summary: 'Stop workflow task', tags: ['console'], }) .input(z.object({ params: zPostAppsByAppIdWorkflowRunsTasksByTaskIdStopPath })) @@ -3142,8 +3118,6 @@ export const workflow = { } /** - * Get default block config - * * Get default block configuration by type */ export const get61 = oc @@ -3153,7 +3127,6 @@ export const get61 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsByBlockType', path: '/apps/{app_id}/workflows/default-workflow-block-configs/{block_type}', - summary: 'Get default block config', tags: ['console'], }) .input( @@ -3169,8 +3142,6 @@ export const byBlockType = { } /** - * Get default block config - * * Get default block configurations for workflow */ export const get62 = oc @@ -3180,7 +3151,6 @@ export const get62 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDefaultWorkflowBlockConfigs', path: '/apps/{app_id}/workflows/default-workflow-block-configs', - summary: 'Get default block config', tags: ['console'], }) .input(z.object({ params: zGetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsPath })) @@ -3232,7 +3202,7 @@ export const conversationVariables2 = { } /** - * Get draft workflow + * Get environment variables * * Get environment variables for workflow */ @@ -3243,7 +3213,7 @@ export const get64 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDraftEnvironmentVariables', path: '/apps/{app_id}/workflows/draft/environment-variables', - summary: 'Get draft workflow', + summary: 'Get environment variables', tags: ['console'], }) .input(z.object({ params: zGetAppsByAppIdWorkflowsDraftEnvironmentVariablesPath })) @@ -3299,8 +3269,6 @@ export const features = { } /** - * Test human input delivery - * * Test human input delivery for workflow */ export const post46 = oc @@ -3310,7 +3278,6 @@ export const post46 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdDeliveryTest', path: '/apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/delivery-test', - summary: 'Test human input delivery', tags: ['console'], }) .input( @@ -3326,8 +3293,6 @@ export const deliveryTest = { } /** - * Preview human input form content and placeholders - * * Get human input form preview for workflow */ export const post47 = oc @@ -3337,7 +3302,6 @@ export const post47 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormPreview', path: '/apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/preview', - summary: 'Preview human input form content and placeholders', tags: ['console'], }) .input( @@ -3353,8 +3317,6 @@ export const preview4 = { } /** - * Submit human input form preview - * * Submit human input form preview for workflow */ export const post48 = oc @@ -3364,7 +3326,6 @@ export const post48 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormRun', path: '/apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/run', - summary: 'Submit human input form preview', tags: ['console'], }) .input( @@ -3398,8 +3359,6 @@ export const humanInput2 = { } /** - * Run draft workflow iteration node - * * Run draft workflow iteration node */ export const post49 = oc @@ -3409,7 +3368,6 @@ export const post49 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftIterationNodesByNodeIdRun', path: '/apps/{app_id}/workflows/draft/iteration/nodes/{node_id}/run', - summary: 'Run draft workflow iteration node', tags: ['console'], }) .input( @@ -3437,8 +3395,6 @@ export const iteration2 = { } /** - * Run draft workflow loop node - * * Run draft workflow loop node */ export const post50 = oc @@ -3448,7 +3404,6 @@ export const post50 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftLoopNodesByNodeIdRun', path: '/apps/{app_id}/workflows/draft/loop/nodes/{node_id}/run', - summary: 'Run draft workflow loop node', tags: ['console'], }) .input( @@ -3634,8 +3589,6 @@ export const lastRun = { } /** - * Run draft workflow node - * * Run draft workflow node */ export const post55 = oc @@ -3645,7 +3598,6 @@ export const post55 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftNodesByNodeIdRun', path: '/apps/{app_id}/workflows/draft/nodes/{node_id}/run', - summary: 'Run draft workflow node', tags: ['console'], }) .input( @@ -3661,8 +3613,6 @@ export const run8 = { } /** - * Poll for trigger events and execute single node when event arrives - * * Poll for trigger events and execute single node when event arrives */ export const post56 = oc @@ -3672,7 +3622,6 @@ export const post56 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRun', path: '/apps/{app_id}/workflows/draft/nodes/{node_id}/trigger/run', - summary: 'Poll for trigger events and execute single node when event arrives', tags: ['console'], }) .input(z.object({ params: zPostAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRunPath })) @@ -3735,8 +3684,6 @@ export const nodes7 = { } /** - * Run draft workflow - * * Run draft workflow */ export const post57 = oc @@ -3746,7 +3693,6 @@ export const post57 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftRun', path: '/apps/{app_id}/workflows/draft/run', - summary: 'Run draft workflow', tags: ['console'], }) .input( @@ -3876,8 +3822,6 @@ export const systemVariables = { } /** - * Poll for trigger events and execute full workflow when event arrives - * * Poll for trigger events and execute full workflow when event arrives */ export const post58 = oc @@ -3887,7 +3831,6 @@ export const post58 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftTriggerRun', path: '/apps/{app_id}/workflows/draft/trigger/run', - summary: 'Poll for trigger events and execute full workflow when event arrives', tags: ['console'], }) .input( @@ -3903,8 +3846,6 @@ export const run11 = { } /** - * Full workflow debug when the start node is a trigger - * * Full workflow debug when the start node is a trigger */ export const post59 = oc @@ -3914,7 +3855,6 @@ export const post59 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftTriggerRunAll', path: '/apps/{app_id}/workflows/draft/trigger/run-all', - summary: 'Full workflow debug when the start node is a trigger', tags: ['console'], }) .input( @@ -4028,7 +3968,7 @@ export const delete13 = oc .output(zDeleteAppsByAppIdWorkflowsDraftVariablesResponse) /** - * Get draft workflow + * List draft workflow variables without loading their values * * Get draft workflow variables */ @@ -4039,7 +3979,7 @@ export const get75 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDraftVariables', path: '/apps/{app_id}/workflows/draft/variables', - summary: 'Get draft workflow', + summary: 'List draft workflow variables without loading their values', tags: ['console'], }) .input( @@ -4057,8 +3997,6 @@ export const variables2 = { } /** - * Get draft workflow - * * Get draft workflow for an application */ export const get76 = oc @@ -4068,15 +4006,12 @@ export const get76 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDraft', path: '/apps/{app_id}/workflows/draft', - summary: 'Get draft workflow', tags: ['console'], }) .input(z.object({ params: zGetAppsByAppIdWorkflowsDraftPath })) .output(zGetAppsByAppIdWorkflowsDraftResponse) /** - * Sync draft workflow - * * Sync draft workflow configuration */ export const post60 = oc @@ -4086,7 +4021,6 @@ export const post60 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraft', path: '/apps/{app_id}/workflows/draft', - summary: 'Sync draft workflow', tags: ['console'], }) .input( @@ -4115,8 +4049,6 @@ export const draft2 = { } /** - * Get published workflow - * * Get published workflow for an application */ export const get77 = oc @@ -4126,22 +4058,17 @@ export const get77 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsPublish', path: '/apps/{app_id}/workflows/publish', - summary: 'Get published workflow', tags: ['console'], }) .input(z.object({ params: zGetAppsByAppIdWorkflowsPublishPath })) .output(zGetAppsByAppIdWorkflowsPublishResponse) -/** - * Publish workflow - */ export const post61 = oc .route({ inputStructure: 'detailed', method: 'POST', operationId: 'postAppsByAppIdWorkflowsPublish', path: '/apps/{app_id}/workflows/publish', - summary: 'Publish workflow', tags: ['console'], }) .input( @@ -4307,24 +4234,19 @@ export const restore = { post: post62, } -/** - * Delete workflow - */ export const delete14 = oc .route({ inputStructure: 'detailed', method: 'DELETE', operationId: 'deleteAppsByAppIdWorkflowsByWorkflowId', path: '/apps/{app_id}/workflows/{workflow_id}', - summary: 'Delete workflow', + successStatus: 204, tags: ['console'], }) .input(z.object({ params: zDeleteAppsByAppIdWorkflowsByWorkflowIdPath })) .output(zDeleteAppsByAppIdWorkflowsByWorkflowIdResponse) /** - * Update workflow attributes - * * Update workflow by ID */ export const patch3 = oc @@ -4334,7 +4256,6 @@ export const patch3 = oc method: 'PATCH', operationId: 'patchAppsByAppIdWorkflowsByWorkflowId', path: '/apps/{app_id}/workflows/{workflow_id}', - summary: 'Update workflow attributes', tags: ['console'], }) .input( @@ -4352,8 +4273,6 @@ export const byWorkflowId = { } /** - * Get published workflows - * * Get all published workflows for an application */ export const get83 = oc @@ -4363,7 +4282,6 @@ export const get83 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflows', path: '/apps/{app_id}/workflows', - summary: 'Get published workflows', tags: ['console'], }) .input( diff --git a/packages/contracts/generated/api/console/apps/types.gen.ts b/packages/contracts/generated/api/console/apps/types.gen.ts index 2d4fbb1b6c22ad..bec2b43430dfe0 100644 --- a/packages/contracts/generated/api/console/apps/types.gen.ts +++ b/packages/contracts/generated/api/console/apps/types.gen.ts @@ -2496,14 +2496,10 @@ export type AgentToolCallResponse = { status: string time_cost: number | number tool_icon?: unknown - tool_input: { - [key: string]: unknown - } + tool_input: unknown tool_label: string tool_name: string - tool_output: { - [key: string]: unknown - } + tool_output: unknown tool_parameters: { [key: string]: unknown } @@ -7596,9 +7592,7 @@ export type DeleteAppsByAppIdWorkflowsByWorkflowIdData = { } export type DeleteAppsByAppIdWorkflowsByWorkflowIdResponses = { - 200: { - [key: string]: unknown - } + 204: void } export type DeleteAppsByAppIdWorkflowsByWorkflowIdResponse = diff --git a/packages/contracts/generated/api/console/apps/zod.gen.ts b/packages/contracts/generated/api/console/apps/zod.gen.ts index eb7445c1fc7415..2570468168d888 100644 --- a/packages/contracts/generated/api/console/apps/zod.gen.ts +++ b/packages/contracts/generated/api/console/apps/zod.gen.ts @@ -2195,10 +2195,10 @@ export const zAgentToolCallResponse = z.object({ status: z.string(), time_cost: z.union([z.number(), z.int()]), tool_icon: z.unknown().optional(), - tool_input: z.record(z.string(), z.unknown()), + tool_input: z.unknown(), tool_label: z.string(), tool_name: z.string(), - tool_output: z.record(z.string(), z.unknown()), + tool_output: z.unknown(), tool_parameters: z.record(z.string(), z.unknown()), }) @@ -7508,9 +7508,9 @@ export const zDeleteAppsByAppIdWorkflowsByWorkflowIdPath = z.object({ }) /** - * Success + * Workflow deleted successfully */ -export const zDeleteAppsByAppIdWorkflowsByWorkflowIdResponse = z.record(z.string(), z.unknown()) +export const zDeleteAppsByAppIdWorkflowsByWorkflowIdResponse = z.void() export const zPatchAppsByAppIdWorkflowsByWorkflowIdBody = zWorkflowUpdatePayload diff --git a/packages/contracts/generated/api/console/rag/orpc.gen.ts b/packages/contracts/generated/api/console/rag/orpc.gen.ts index d2a27ca561d997..223139073f73ca 100644 --- a/packages/contracts/generated/api/console/rag/orpc.gen.ts +++ b/packages/contracts/generated/api/console/rag/orpc.gen.ts @@ -75,6 +75,7 @@ import { zPatchRagPipelineCustomizedTemplatesByTemplateIdResponse, zPatchRagPipelinesByPipelineIdWorkflowsByWorkflowIdPath, zPatchRagPipelinesByPipelineIdWorkflowsByWorkflowIdResponse, + zPatchRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdBody, zPatchRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdPath, zPatchRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdResponse, zPostRagPipelineCustomizedTemplatesByTemplateIdPath, @@ -89,6 +90,7 @@ import { zPostRagPipelinesByPipelineIdWorkflowRunsTasksByTaskIdStopResponse, zPostRagPipelinesByPipelineIdWorkflowsByWorkflowIdRestorePath, zPostRagPipelinesByPipelineIdWorkflowsByWorkflowIdRestoreResponse, + zPostRagPipelinesByPipelineIdWorkflowsDraftBody, zPostRagPipelinesByPipelineIdWorkflowsDraftDatasourceNodesByNodeIdRunBody, zPostRagPipelinesByPipelineIdWorkflowsDraftDatasourceNodesByNodeIdRunPath, zPostRagPipelinesByPipelineIdWorkflowsDraftDatasourceNodesByNodeIdRunResponse, @@ -609,7 +611,7 @@ export const datasource = { } /** - * Get draft workflow + * Get environment variables */ export const get12 = oc .route({ @@ -617,7 +619,7 @@ export const get12 = oc method: 'GET', operationId: 'getRagPipelinesByPipelineIdWorkflowsDraftEnvironmentVariables', path: '/rag/pipelines/{pipeline_id}/workflows/draft/environment-variables', - summary: 'Get draft workflow', + summary: 'Get environment variables', tags: ['console'], }) .input(z.object({ params: zGetRagPipelinesByPipelineIdWorkflowsDraftEnvironmentVariablesPath })) @@ -924,7 +926,10 @@ export const patch2 = oc tags: ['console'], }) .input( - z.object({ params: zPatchRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdPath }), + z.object({ + body: zPatchRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdBody, + params: zPatchRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdPath, + }), ) .output(zPatchRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdResponse) @@ -948,7 +953,7 @@ export const delete4 = oc .output(zDeleteRagPipelinesByPipelineIdWorkflowsDraftVariablesResponse) /** - * Get draft workflow + * List draft pipeline variables without loading their values */ export const get19 = oc .route({ @@ -956,7 +961,7 @@ export const get19 = oc method: 'GET', operationId: 'getRagPipelinesByPipelineIdWorkflowsDraftVariables', path: '/rag/pipelines/{pipeline_id}/workflows/draft/variables', - summary: 'Get draft workflow', + summary: 'List draft pipeline variables without loading their values', tags: ['console'], }) .input( @@ -1000,7 +1005,12 @@ export const post15 = oc summary: 'Sync draft workflow', tags: ['console'], }) - .input(z.object({ params: zPostRagPipelinesByPipelineIdWorkflowsDraftPath })) + .input( + z.object({ + body: zPostRagPipelinesByPipelineIdWorkflowsDraftBody, + params: zPostRagPipelinesByPipelineIdWorkflowsDraftPath, + }), + ) .output(zPostRagPipelinesByPipelineIdWorkflowsDraftResponse) export const draft = { @@ -1034,7 +1044,7 @@ export const get21 = oc .output(zGetRagPipelinesByPipelineIdWorkflowsPublishResponse) /** - * Publish workflow + * Publish a prepared Pipeline and its dataset settings atomically */ export const post16 = oc .route({ @@ -1042,7 +1052,7 @@ export const post16 = oc method: 'POST', operationId: 'postRagPipelinesByPipelineIdWorkflowsPublish', path: '/rag/pipelines/{pipeline_id}/workflows/publish', - summary: 'Publish workflow', + summary: 'Publish a prepared Pipeline and its dataset settings atomically', tags: ['console'], }) .input(z.object({ params: zPostRagPipelinesByPipelineIdWorkflowsPublishPath })) @@ -1216,9 +1226,6 @@ export const restore = { post: post20, } -/** - * Delete a published workflow version that is not currently active on the pipeline - */ export const delete5 = oc .route({ inputStructure: 'detailed', @@ -1226,7 +1233,6 @@ export const delete5 = oc operationId: 'deleteRagPipelinesByPipelineIdWorkflowsByWorkflowId', path: '/rag/pipelines/{pipeline_id}/workflows/{workflow_id}', successStatus: 204, - summary: 'Delete a published workflow version that is not currently active on the pipeline', tags: ['console'], }) .input(z.object({ params: zDeleteRagPipelinesByPipelineIdWorkflowsByWorkflowIdPath })) diff --git a/packages/contracts/generated/api/console/rag/types.gen.ts b/packages/contracts/generated/api/console/rag/types.gen.ts index 70953beb7726cf..e6186ad9b34a82 100644 --- a/packages/contracts/generated/api/console/rag/types.gen.ts +++ b/packages/contracts/generated/api/console/rag/types.gen.ts @@ -180,6 +180,31 @@ export type WorkflowResponse = { version_number?: number | null } +export type DraftWorkflowSyncPayload = { + conversation_variables?: Array<{ + [key: string]: unknown + }> | null + environment_variables?: Array<{ + [key: string]: unknown + }> | null + features?: { + [key: string]: unknown + } | null + graph: { + [key: string]: unknown + } + hash?: string | null + rag_pipeline_variables?: Array<{ + [key: string]: unknown + }> | null +} + +export type RagPipelineWorkflowSyncResponse = { + hash: string + result: string + updated_at: number +} + export type DatasourceNodeRunPayload = { credential_id?: string | null datasource_type: string @@ -277,6 +302,11 @@ export type WorkflowDraftVariableResponse = { visible: boolean } +export type WorkflowDraftVariablePatchPayload = { + name?: string | null + value?: unknown | null +} + export type RagPipelineWorkflowPublishResponse = { created_at: number result: string @@ -304,12 +334,6 @@ export type PublishedWorkflowRunPayload = { start_node_id: string } -export type RagPipelineWorkflowSyncResponse = { - hash: string - result: string - updated_at: number -} - export type ImportStatus = 'completed' | 'completed-with-warnings' | 'failed' | 'pending' export type DatasetDocMetadataResponse = { @@ -1122,7 +1146,7 @@ export type GetRagPipelinesByPipelineIdWorkflowsDraftResponse = GetRagPipelinesByPipelineIdWorkflowsDraftResponses[keyof GetRagPipelinesByPipelineIdWorkflowsDraftResponses] export type PostRagPipelinesByPipelineIdWorkflowsDraftData = { - body?: never + body: DraftWorkflowSyncPayload path: { pipeline_id: string } @@ -1131,9 +1155,7 @@ export type PostRagPipelinesByPipelineIdWorkflowsDraftData = { } export type PostRagPipelinesByPipelineIdWorkflowsDraftResponses = { - 200: { - [key: string]: unknown - } + 200: RagPipelineWorkflowSyncResponse } export type PostRagPipelinesByPipelineIdWorkflowsDraftResponse = @@ -1428,7 +1450,7 @@ export type GetRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdRespon GetRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdResponses[keyof GetRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdResponses] export type PatchRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdData = { - body?: never + body: WorkflowDraftVariablePatchPayload path: { pipeline_id: string variable_id: string diff --git a/packages/contracts/generated/api/console/rag/zod.gen.ts b/packages/contracts/generated/api/console/rag/zod.gen.ts index a5afe6f494881d..3d92e1996df244 100644 --- a/packages/contracts/generated/api/console/rag/zod.gen.ts +++ b/packages/contracts/generated/api/console/rag/zod.gen.ts @@ -76,6 +76,27 @@ export const zDefaultBlockConfigsResponse = z.array(z.record(z.string(), z.unkno */ export const zDefaultBlockConfigResponse = z.record(z.string(), z.unknown()) +/** + * DraftWorkflowSyncPayload + */ +export const zDraftWorkflowSyncPayload = z.object({ + conversation_variables: z.array(z.record(z.string(), z.unknown())).nullish(), + environment_variables: z.array(z.record(z.string(), z.unknown())).nullish(), + features: z.record(z.string(), z.unknown()).nullish(), + graph: z.record(z.string(), z.unknown()), + hash: z.string().nullish(), + rag_pipeline_variables: z.array(z.record(z.string(), z.unknown())).nullish(), +}) + +/** + * RagPipelineWorkflowSyncResponse + */ +export const zRagPipelineWorkflowSyncResponse = z.object({ + hash: z.string(), + result: z.string(), + updated_at: z.int(), +}) + /** * DatasourceNodeRunPayload */ @@ -126,6 +147,14 @@ export const zDraftWorkflowRunPayload = z.object({ start_node_id: z.string(), }) +/** + * WorkflowDraftVariablePatchPayload + */ +export const zWorkflowDraftVariablePatchPayload = z.object({ + name: z.string().nullish(), + value: z.unknown().nullish(), +}) + /** * RagPipelineWorkflowPublishResponse */ @@ -156,15 +185,6 @@ export const zPublishedWorkflowRunPayload = z.object({ start_node_id: z.string(), }) -/** - * RagPipelineWorkflowSyncResponse - */ -export const zRagPipelineWorkflowSyncResponse = z.object({ - hash: z.string(), - result: z.string(), - updated_at: z.int(), -}) - /** * ImportStatus */ @@ -1169,6 +1189,8 @@ export const zGetRagPipelinesByPipelineIdWorkflowsDraftPath = z.object({ */ export const zGetRagPipelinesByPipelineIdWorkflowsDraftResponse = zWorkflowResponse +export const zPostRagPipelinesByPipelineIdWorkflowsDraftBody = zDraftWorkflowSyncPayload + export const zPostRagPipelinesByPipelineIdWorkflowsDraftPath = z.object({ pipeline_id: z.uuid(), }) @@ -1176,7 +1198,7 @@ export const zPostRagPipelinesByPipelineIdWorkflowsDraftPath = z.object({ /** * Success */ -export const zPostRagPipelinesByPipelineIdWorkflowsDraftResponse = z.record(z.string(), z.unknown()) +export const zPostRagPipelinesByPipelineIdWorkflowsDraftResponse = zRagPipelineWorkflowSyncResponse export const zPostRagPipelinesByPipelineIdWorkflowsDraftDatasourceNodesByNodeIdRunBody = zDatasourceNodeRunPayload @@ -1382,6 +1404,9 @@ export const zGetRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdPath export const zGetRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdResponse = zWorkflowDraftVariableResponse +export const zPatchRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdBody = + zWorkflowDraftVariablePatchPayload + export const zPatchRagPipelinesByPipelineIdWorkflowsDraftVariablesByVariableIdPath = z.object({ pipeline_id: z.uuid(), variable_id: z.uuid(), diff --git a/packages/contracts/generated/api/console/snippets/orpc.gen.ts b/packages/contracts/generated/api/console/snippets/orpc.gen.ts index 4e664987a77a71..219e988ec2b34a 100644 --- a/packages/contracts/generated/api/console/snippets/orpc.gen.ts +++ b/packages/contracts/generated/api/console/snippets/orpc.gen.ts @@ -877,7 +877,7 @@ export const restore = { } /** - * Delete a published snippet workflow version + * Delete a version and retire its unowned Agents through the shared use case * * Delete a published snippet workflow version */ @@ -889,7 +889,7 @@ export const delete4 = oc operationId: 'deleteSnippetsBySnippetIdWorkflowsByWorkflowId', path: '/snippets/{snippet_id}/workflows/{workflow_id}', successStatus: 204, - summary: 'Delete a published snippet workflow version', + summary: 'Delete a version and retire its unowned Agents through the shared use case', tags: ['console'], }) .input(z.object({ params: zDeleteSnippetsBySnippetIdWorkflowsByWorkflowIdPath })) From eaed5ed719667884e72db04103b5fa926f7f5023 Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Fri, 9 Oct 2026 13:43:06 +0800 Subject: [PATCH 03/20] fix: preserve explicit tool sessions and validate application service activation --- api/core/tools/__base/tool.py | 6 +- .../builtin_tool/providers/audio/tools/asr.py | 3 + .../builtin_tool/providers/audio/tools/tts.py | 3 + .../providers/code/tools/simple_code.py | 3 + .../providers/time/tools/current_time.py | 2 + .../time/tools/localtime_to_timestamp.py | 2 + .../time/tools/timestamp_to_localtime.py | 2 + .../time/tools/timezone_conversion.py | 2 + .../providers/time/tools/weekday.py | 3 + .../providers/webscraper/tools/webscraper.py | 3 + api/core/tools/custom_tool/tool.py | 2 + api/core/tools/mcp_tool/tool.py | 5 +- api/core/tools/plugin_tool/tool.py | 3 + .../application_services/workflow.py | 58 ++++- .../app/generation/adapters/agent_chat.py | 53 +---- api/services/app/generation/runtime.py | 4 +- api/services/app_generate_service.py | 1 + api/services/plugin/app_invocation.py | 1 + api/services/workflow/definition_gateway.py | 2 +- .../conftest.py | 7 +- .../services/test_app_generate_service.py | 87 ++++---- .../tools/test_tools_transform_service.py | 6 +- .../workflow/test_workflow_converter.py | 38 +++- .../commands/test_data_migration_commands.py | 6 +- api/tests/unit_tests/conftest.py | 6 +- api/tests/unit_tests/controllers/conftest.py | 8 +- .../console/agent/test_agent_controllers.py | 6 +- .../console/app/test_model_config_api.py | 6 + .../controllers/console/app/test_workflow.py | 8 +- .../console/app/test_workflow_admission.py | 8 +- .../test_workflow_human_input_debug_api.py | 6 +- .../controllers/console/app/test_wraps.py | 31 ++- .../test_rag_pipeline_workflow_apis.py | 15 +- .../test_datasets_document_download.py | 89 +++++--- .../explore/test_installed_app_completion.py | 4 +- .../explore/test_trial_preview_details.py | 24 ++- .../inner_api/plugin/test_plugin.py | 5 +- .../controllers/inner_api/test_agent_tools.py | 15 +- .../service_api/app/test_hitl_service_api.py | 15 +- .../controllers/trigger/test_webhook.py | 14 +- .../test_agent_chat_app_generator.py | 5 +- .../plugin/test_backwards_invocation_app.py | 34 +-- .../core/tools/workflow_as_tool/test_tool.py | 204 +++++++++++------- .../nodes/agent_v2/test_agent_node.py | 8 +- .../nodes/agent_v2/test_binding_resolver.py | 16 +- .../test_tool_node_invocation_behavior.py | 7 +- ...rameters_cache_when_sync_draft_workflow.py | 9 +- .../handlers/test_generate_handler.py | 56 +++-- .../test_workflow_node_execution_accessors.py | 17 +- .../test_agent_execution_repository.py | 1 + .../workflow/test_definition_repository.py | 80 ++++--- .../agent/chat/test_tool_invocation.py | 4 +- .../services/agent/test_log_service.py | 4 +- .../services/agent/test_retirement_service.py | 6 +- .../agent/test_tool_invocation_service.py | 4 +- .../app/test_agent_app_access_service.py | 8 +- .../app/test_agent_app_feature_service.py | 11 +- .../app/test_agent_chat_attachments.py | 150 +++++++++---- .../services/app/test_console_service.py | 35 ++- .../services/app/test_response_gateway.py | 6 +- .../data_migration/test_import_service.py | 50 ++--- .../test_runner_retrieval_boundary.py | 142 ++++++++---- .../services/test_app_dsl_service.py | 66 ++++-- .../services/test_app_generate_service.py | 70 +++--- .../test_app_preview_details_adapters.py | 20 +- .../test_installed_app_generation_adapters.py | 16 +- .../services/test_snippet_dsl_service.py | 27 ++- .../services/test_snippet_generate_service.py | 4 +- .../test_trial_app_generation_adapters.py | 9 +- .../tools/test_api_provider_runtime.py | 39 ++-- .../tools/test_builtin_provider_runtime.py | 102 +++++---- .../workflow/test_conversion_service.py | 26 ++- .../workflow/test_generation_read_boundary.py | 27 +-- .../workflow/test_human_input_persistence.py | 52 +++-- .../services/workflow/test_runtime_gateway.py | 73 ++++--- 75 files changed, 1256 insertions(+), 694 deletions(-) diff --git a/api/core/tools/__base/tool.py b/api/core/tools/__base/tool.py index 151733bd2cd4c7..e023ce117b1379 100644 --- a/api/core/tools/__base/tool.py +++ b/api/core/tools/__base/tool.py @@ -5,6 +5,8 @@ from copy import deepcopy from typing import TYPE_CHECKING, Any +from sqlalchemy.orm import Session + if TYPE_CHECKING: # pragma: no cover from models.model import File @@ -46,19 +48,20 @@ def tool_provider_type(self) -> ToolProviderType: def invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, app_id: str | None = None, message_id: str | None = None, ) -> Generator[ToolInvokeMessage]: - """Invoke a configured provider without carrying persistence resources.""" if self.runtime and self.runtime.runtime_parameters: tool_parameters.update(self.runtime.runtime_parameters) tool_parameters = self._transform_tool_parameters_type(tool_parameters) result = self._invoke( + session=session, user_id=user_id, tool_parameters=tool_parameters, conversation_id=conversation_id, @@ -97,6 +100,7 @@ def _transform_tool_parameters_type(self, tool_parameters: dict[str, Any]) -> di @abstractmethod def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/audio/tools/asr.py b/api/core/tools/builtin_tool/providers/audio/tools/asr.py index 79a345cfbe52f7..44543667815b50 100644 --- a/api/core/tools/builtin_tool/providers/audio/tools/asr.py +++ b/api/core/tools/builtin_tool/providers/audio/tools/asr.py @@ -2,6 +2,8 @@ from collections.abc import Generator from typing import Any, override +from sqlalchemy.orm import Session + from core.credit_usage import CreditUsageCreatedBy from core.model_context import with_credit_usage_created_by from core.model_manager import ModelManager @@ -20,6 +22,7 @@ class ASRTool(BuiltinTool): @with_credit_usage_created_by(CreditUsageCreatedBy.AUDIO) def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/audio/tools/tts.py b/api/core/tools/builtin_tool/providers/audio/tools/tts.py index 55a575a332de15..8350af1e040ead 100644 --- a/api/core/tools/builtin_tool/providers/audio/tools/tts.py +++ b/api/core/tools/builtin_tool/providers/audio/tools/tts.py @@ -1,6 +1,8 @@ from collections.abc import Generator from typing import Any, override +from sqlalchemy.orm import Session + from core.base.tts.audio_mime import get_model_audio_mime_type, inspect_audio_stream from core.credit_usage import CreditUsageCreatedBy from core.model_context import with_credit_usage_created_by @@ -18,6 +20,7 @@ class TTSTool(BuiltinTool): @with_credit_usage_created_by(CreditUsageCreatedBy.AUDIO) def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/code/tools/simple_code.py b/api/core/tools/builtin_tool/providers/code/tools/simple_code.py index dd041ef5eb9c97..78a01115b03636 100644 --- a/api/core/tools/builtin_tool/providers/code/tools/simple_code.py +++ b/api/core/tools/builtin_tool/providers/code/tools/simple_code.py @@ -1,6 +1,8 @@ from collections.abc import Generator from typing import Any, override +from sqlalchemy.orm import Session + from core.helper.code_executor.code_executor import CodeExecutor, CodeLanguage from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage @@ -11,6 +13,7 @@ class SimpleCode(BuiltinTool): @override def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/time/tools/current_time.py b/api/core/tools/builtin_tool/providers/time/tools/current_time.py index 1164cc11c628c1..e9530da9a92357 100644 --- a/api/core/tools/builtin_tool/providers/time/tools/current_time.py +++ b/api/core/tools/builtin_tool/providers/time/tools/current_time.py @@ -3,6 +3,7 @@ from typing import Any, override from pytz import timezone as pytz_timezone # type: ignore[import-untyped] +from sqlalchemy.orm import Session from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage @@ -12,6 +13,7 @@ class CurrentTimeTool(BuiltinTool): @override def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/time/tools/localtime_to_timestamp.py b/api/core/tools/builtin_tool/providers/time/tools/localtime_to_timestamp.py index e88c0198c070f6..b327c15e01b3b8 100644 --- a/api/core/tools/builtin_tool/providers/time/tools/localtime_to_timestamp.py +++ b/api/core/tools/builtin_tool/providers/time/tools/localtime_to_timestamp.py @@ -3,6 +3,7 @@ from typing import Any, cast, override import pytz # type: ignore[import-untyped] +from sqlalchemy.orm import Session from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage @@ -13,6 +14,7 @@ class LocaltimeToTimestampTool(BuiltinTool): @override def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/time/tools/timestamp_to_localtime.py b/api/core/tools/builtin_tool/providers/time/tools/timestamp_to_localtime.py index f1bf1471444a04..489f450e6f4154 100644 --- a/api/core/tools/builtin_tool/providers/time/tools/timestamp_to_localtime.py +++ b/api/core/tools/builtin_tool/providers/time/tools/timestamp_to_localtime.py @@ -3,6 +3,7 @@ from typing import Any, override import pytz # type: ignore[import-untyped] +from sqlalchemy.orm import Session from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage @@ -13,6 +14,7 @@ class TimestampToLocaltimeTool(BuiltinTool): @override def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/time/tools/timezone_conversion.py b/api/core/tools/builtin_tool/providers/time/tools/timezone_conversion.py index 6f00e0009e7ef1..6d9eb9e57f88a4 100644 --- a/api/core/tools/builtin_tool/providers/time/tools/timezone_conversion.py +++ b/api/core/tools/builtin_tool/providers/time/tools/timezone_conversion.py @@ -3,6 +3,7 @@ from typing import Any, override import pytz # type: ignore[import-untyped] +from sqlalchemy.orm import Session from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage @@ -13,6 +14,7 @@ class TimezoneConversionTool(BuiltinTool): @override def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/time/tools/weekday.py b/api/core/tools/builtin_tool/providers/time/tools/weekday.py index d6352fc23f4446..7cc589128ee41b 100644 --- a/api/core/tools/builtin_tool/providers/time/tools/weekday.py +++ b/api/core/tools/builtin_tool/providers/time/tools/weekday.py @@ -3,6 +3,8 @@ from datetime import datetime from typing import Any, override +from sqlalchemy.orm import Session + from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage @@ -11,6 +13,7 @@ class WeekdayTool(BuiltinTool): @override def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/builtin_tool/providers/webscraper/tools/webscraper.py b/api/core/tools/builtin_tool/providers/webscraper/tools/webscraper.py index 44c6f720d0e055..4c29ef3145b715 100644 --- a/api/core/tools/builtin_tool/providers/webscraper/tools/webscraper.py +++ b/api/core/tools/builtin_tool/providers/webscraper/tools/webscraper.py @@ -1,6 +1,8 @@ from collections.abc import Generator from typing import Any, override +from sqlalchemy.orm import Session + from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage from core.tools.errors import ToolInvokeError @@ -11,6 +13,7 @@ class WebscraperTool(BuiltinTool): @override def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/custom_tool/tool.py b/api/core/tools/custom_tool/tool.py index 8a5d59656e6a1d..0e1da00a3fc389 100644 --- a/api/core/tools/custom_tool/tool.py +++ b/api/core/tools/custom_tool/tool.py @@ -6,6 +6,7 @@ from urllib.parse import urlencode import httpx +from sqlalchemy.orm import Session from core.helper import ssrf_proxy from core.tools.__base.tool import Tool @@ -380,6 +381,7 @@ def _convert_body_property_type(self, property: dict[str, Any], value: Any): @override def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/core/tools/mcp_tool/tool.py b/api/core/tools/mcp_tool/tool.py index c21c34a6f20666..2042dec70677ed 100644 --- a/api/core/tools/mcp_tool/tool.py +++ b/api/core/tools/mcp_tool/tool.py @@ -68,6 +68,7 @@ def tool_provider_type(self) -> ToolProviderType: @override def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, @@ -289,8 +290,10 @@ def invoke_remote_mcp_tool( headers = self.headers.copy() if self.headers else {} tool_parameters = self._handle_none_parameter(tool_parameters) + from sqlalchemy.orm import Session + from extensions.ext_database import db - from services.tools.mcp_tools_manage_service import MCPToolManageService + from services.tools.legacy_mcp_tools_manage_service import MCPToolManageService # Step 1: Load provider entity and credentials in a short-lived session # This minimizes database connection hold time diff --git a/api/core/tools/plugin_tool/tool.py b/api/core/tools/plugin_tool/tool.py index ac17d542bc02bf..27362632c3ae09 100644 --- a/api/core/tools/plugin_tool/tool.py +++ b/api/core/tools/plugin_tool/tool.py @@ -3,6 +3,8 @@ from collections.abc import Generator from typing import Any, override +from sqlalchemy.orm import Session + from core.plugin.impl.tool import PluginToolManager from core.plugin.utils.converter import convert_parameters_to_plugin_format from core.tools.__base.tool import Tool @@ -27,6 +29,7 @@ def tool_provider_type(self) -> ToolProviderType: @override def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, diff --git a/api/extensions/application_services/workflow.py b/api/extensions/application_services/workflow.py index b3674d4ef3b8c1..7725a3dc20eacb 100644 --- a/api/extensions/application_services/workflow.py +++ b/api/extensions/application_services/workflow.py @@ -8,17 +8,22 @@ from sqlalchemy.orm import Session, sessionmaker +from core.app.entities.app_invoke_entities import InvokeFrom +from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler +from core.ops.ops_trace_manager import TraceQueueManager from core.tools.__base.tool import Tool -from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.entities.tool_entities import ToolInvokeMessage, ToolInvokeMeta from extensions.ext_redis import RedisClientWrapper from models.agent_runtime_contracts import WorkflowAgentRuntimeBindings from models.annotation_reply import AnnotationReplies from models.human_input_contracts import HumanInputFormFactory +from models.model import Message from models.tool_runtime_contracts import WorkflowToolQueries from repositories.knowledge.retrieval_repository import KnowledgeRetrievalRepository -from services.agent.chat.ports import AgentDatasetTools +from services.agent.chat.ports import AgentDatasetTools, AgentToolInvoker from services.app.generation.agent_config import AgentAppConfigurations +from services.app.generation.ports import MessageFileWriter from services.human_input.ports import HumanInputFormReader from services.knowledge.retrieval.ports import DatasetRetrievalFactory from services.tools.provider_queries import ToolProviders @@ -69,6 +74,47 @@ def __call__( ) +class _SessionBoundAgentToolInvoker(AgentToolInvoker): + """Own one short-lived Session for each eager Agent tool invocation.""" + + def __init__(self, sessions: sessionmaker[Session]) -> None: + self._sessions = sessions + + @override + def __call__( + self, + tool: Tool, + tool_parameters: str | dict[str, Any], + user_id: str, + tenant_id: str, + message: Message, + invoke_from: InvokeFrom, + agent_tool_callback: DifyAgentCallbackHandler, + trace_manager: TraceQueueManager | None = None, + conversation_id: str | None = None, + app_id: str | None = None, + message_id: str | None = None, + *, + records: MessageFileWriter, + ) -> tuple[str, list[str], ToolInvokeMeta]: + with self._sessions() as session: + return ToolEngine.agent_invoke( + session=session, + tool=tool, + tool_parameters=tool_parameters, + user_id=user_id, + tenant_id=tenant_id, + message=message, + invoke_from=invoke_from, + agent_tool_callback=agent_tool_callback, + trace_manager=trace_manager, + conversation_id=conversation_id, + app_id=app_id, + message_id=message_id, + records=records, + ) + + @dataclass(frozen=True) class WorkflowExecutionDependencies: history: ConversationHistory @@ -79,6 +125,7 @@ class WorkflowExecutionDependencies: tools: WorkflowToolQueries tool_providers: ToolProviders tool_invoker: WorkflowToolInvoker + agent_tool_invoker: AgentToolInvoker agent_bindings: WorkflowAgentRuntimeBindings chat_records: ChatflowRecords annotation_replies: AnnotationReplies @@ -139,6 +186,7 @@ def build_workflow_execution_dependencies(database_client: sessionmaker[Session] tools=WorkflowToolRepository(database_client), tool_providers=ToolProviderRepository(database_client), tool_invoker=_SessionBoundWorkflowToolInvoker(database_client), + agent_tool_invoker=_SessionBoundAgentToolInvoker(database_client), agent_bindings=WorkflowAgentBindingResolver(database_client), chat_records=records, annotation_replies=build_annotation_replies(database_client), @@ -189,6 +237,7 @@ def build_console_workflow_service( from services.workflow.console_service import ConsoleWorkflowService from services.workflow.conversion_service import WorkflowConversionService from services.workflow.runtime_gateway import WorkflowRuntimeGateway + from services.workflow.workflow_converter import WorkflowConverter from services.workflow_service import WorkflowService workflows = WorkflowService( @@ -201,7 +250,10 @@ def build_console_workflow_service( apps = ConsoleAppRepository(session_factory=database_client) return ConsoleWorkflowService( conversion=WorkflowConversionService( - apps, decrypt_token=decrypt_token, notify_created=AppService.notify_created_app + apps, + converter=WorkflowConverter(), + decrypt_token=decrypt_token, + notify_created=AppService.notify_created_app, ), agent_services=WorkflowAgentPublishService, definitions=definitions, diff --git a/api/services/app/generation/adapters/agent_chat.py b/api/services/app/generation/adapters/agent_chat.py index b45637f955d5b3..af61605782a2c7 100644 --- a/api/services/app/generation/adapters/agent_chat.py +++ b/api/services/app/generation/adapters/agent_chat.py @@ -7,7 +7,7 @@ from flask import Flask, current_app from pydantic import ValidationError -from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.orm import Session from configs import dify_config from constants import UUID_NIL @@ -20,75 +20,30 @@ from core.app.apps.exc import GenerateTaskStoppedError from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueManager from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity, InvokeFrom -from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler from core.helper.trace_id_helper import extract_trace_session_id_from_args from core.ops.ops_trace_manager import TraceQueueManager -from core.tools.__base.tool import Tool -from core.tools.entities.tool_entities import ToolInvokeMeta from factories import file_factory from graphon.model_runtime.errors.invoke import InvokeAuthorizationError from libs.flask_utils import preserve_flask_contexts from models import Account, App, EndUser from models.annotation_reply import AnnotationReplies -from models.model import Message from services.agent.chat.ports import AgentDatasetTools, AgentToolInvoker from services.app.generation.adapters.agent_chat_runner import AgentChatAppRunner from services.app.generation.message_records import MessageBasedAppGenerator -from services.app.generation.ports import ChatRecords, MessageFileWriter -from services.tools.tool_engine import ToolEngine +from services.app.generation.ports import ChatRecords from services.workflow.execution.ports import WorkflowRuntime logger = logging.getLogger(__name__) -class _SessionBoundAgentToolInvoker(AgentToolInvoker): - """Own one short-lived Session for each eager Agent tool invocation.""" - - def __init__(self, tool_sessions: sessionmaker[Session]) -> None: - self._tool_sessions = tool_sessions - - def __call__( - self, - tool: Tool, - tool_parameters: str | dict[str, Any], - user_id: str, - tenant_id: str, - message: Message, - invoke_from: InvokeFrom, - agent_tool_callback: DifyAgentCallbackHandler, - trace_manager: TraceQueueManager | None = None, - conversation_id: str | None = None, - app_id: str | None = None, - message_id: str | None = None, - *, - records: MessageFileWriter, - ) -> tuple[str, list[str], ToolInvokeMeta]: - with self._tool_sessions() as session: - return ToolEngine.agent_invoke( - session=session, - tool=tool, - tool_parameters=tool_parameters, - user_id=user_id, - tenant_id=tenant_id, - message=message, - invoke_from=invoke_from, - agent_tool_callback=agent_tool_callback, - trace_manager=trace_manager, - conversation_id=conversation_id, - app_id=app_id, - message_id=message_id, - records=records, - ) - - class AgentChatAppGenerator(MessageBasedAppGenerator): def __init__( self, *, dataset_tools: AgentDatasetTools, + tool_invoker: AgentToolInvoker, records: ChatRecords, annotations: AnnotationReplies, - tool_sessions: sessionmaker[Session], draft_variable_saver: Callable[[str, Account], DraftVariableSaverFactory] | None = None, workflow_runtime: WorkflowRuntime | None = None, ): @@ -99,7 +54,7 @@ def __init__( workflow_runtime=workflow_runtime, ) self._dataset_tools = dataset_tools - self._tool_invoker = _SessionBoundAgentToolInvoker(tool_sessions) + self._tool_invoker = tool_invoker @overload def generate( diff --git a/api/services/app/generation/runtime.py b/api/services/app/generation/runtime.py index 36761ab096f390..2233a022cbb59f 100644 --- a/api/services/app/generation/runtime.py +++ b/api/services/app/generation/runtime.py @@ -2,7 +2,7 @@ from typing import Protocol -from services.agent.chat.ports import AgentDatasetTools +from services.agent.chat.ports import AgentDatasetTools, AgentToolInvoker from services.app.generation.agent_config import AgentAppConfigurations from services.workflow.execution.chatflow_ports import ChatflowRuntime @@ -12,3 +12,5 @@ class AppGenerationRuntime(ChatflowRuntime, Protocol): def dataset_tools(self) -> AgentDatasetTools: ... @property def agent_configs(self) -> AgentAppConfigurations: ... + @property + def agent_tool_invoker(self) -> AgentToolInvoker: ... diff --git a/api/services/app_generate_service.py b/api/services/app_generate_service.py index 58b1b1916ab298..eb543c2bf85c81 100644 --- a/api/services/app_generate_service.py +++ b/api/services/app_generate_service.py @@ -330,6 +330,7 @@ def _dispatch_generate( convert_to_event_stream( AgentChatAppGenerator( dataset_tools=runtime.dataset_tools, + tool_invoker=runtime.agent_tool_invoker, annotations=runtime.annotation_replies, records=runtime.chat_records, draft_variable_saver=variables.saver_factory, diff --git a/api/services/plugin/app_invocation.py b/api/services/plugin/app_invocation.py index f7747e35a95b20..ee0ba817818d84 100644 --- a/api/services/plugin/app_invocation.py +++ b/api/services/plugin/app_invocation.py @@ -177,6 +177,7 @@ def invoke_chat_app( annotations=runtime.annotation_replies, records=runtime.chat_records, dataset_tools=runtime.dataset_tools, + tool_invoker=runtime.agent_tool_invoker, workflow_runtime=runtime, ).generate( app_model=app, diff --git a/api/services/workflow/definition_gateway.py b/api/services/workflow/definition_gateway.py index 49c273536c44c6..b150a33b70c928 100644 --- a/api/services/workflow/definition_gateway.py +++ b/api/services/workflow/definition_gateway.py @@ -43,7 +43,7 @@ def snapshot( class WorkflowDefinitionValidator(Protocol): - def validate_features_structure(self, app_model: App, features: dict[str, Any]) -> None: ... + def validate_features_structure(self, app_model: App, features: dict[str, Any]) -> dict[str, Any | None]: ... def validate_graph_structure(self, graph: Mapping[str, Any]) -> None: ... def validate_publication( self, app_model: App, draft_workflow: Workflow, *, credentials: CredentialQuery diff --git a/api/tests/test_containers_integration_tests/conftest.py b/api/tests/test_containers_integration_tests/conftest.py index 16e9bf4a01a24b..7efcf07cd1b024 100644 --- a/api/tests/test_containers_integration_tests/conftest.py +++ b/api/tests/test_containers_integration_tests/conftest.py @@ -596,8 +596,9 @@ def _fake_request(method: str, url: str, **_kwargs: object) -> httpx.Response: @pytest.fixture -def workflow_variables(app: Flask) -> WorkflowExecutionVariables: +def workflow_variables(db_session_with_containers: Session) -> WorkflowExecutionVariables: from extensions.application_services.workflow_variables import build_workflow_variable_service - with app.app_context(): - return build_workflow_variable_service(database_client=sessionmaker(bind=db.engine, expire_on_commit=False)) + return build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) diff --git a/api/tests/test_containers_integration_tests/services/test_app_generate_service.py b/api/tests/test_containers_integration_tests/services/test_app_generate_service.py index b0c0f6d315ddd3..88b8b7be42005d 100644 --- a/api/tests/test_containers_integration_tests/services/test_app_generate_service.py +++ b/api/tests/test_containers_integration_tests/services/test_app_generate_service.py @@ -1,6 +1,9 @@ +from __future__ import annotations + import json import uuid -from typing import Literal +from collections.abc import Generator +from typing import TYPE_CHECKING, Literal from unittest.mock import ANY, MagicMock, patch import pytest @@ -19,19 +22,24 @@ from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.test_containers_integration_tests.helpers import generate_valid_password +if TYPE_CHECKING: + from extensions.application_services.workflow import WorkflowExecutionDependencies + @pytest.fixture -def workflow_runtime(db_session_with_containers): +def workflow_runtime(db_session_with_containers: Session) -> WorkflowExecutionDependencies: from extensions.application_services.workflow import build_workflow_execution_dependencies - return build_workflow_execution_dependencies(sessionmaker(bind=db_session_with_containers.get_bind())) + return build_workflow_execution_dependencies( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) class TestAppGenerateService: """Integration tests for AppGenerateService using testcontainers.""" @pytest.fixture - def mock_external_service_dependencies(self): + def mock_external_service_dependencies(self) -> Generator[dict[str, MagicMock], None, None]: """Mock setup for external service dependencies.""" with ( patch("services.billing_service.BillingService", autospec=True) as mock_billing_service, @@ -54,9 +62,10 @@ def mock_external_service_dependencies(self): "services.app_generate_service.AdvancedChatAppGenerator", autospec=True ) as mock_advanced_chat_generator, patch("services.app_generate_service.WorkflowAppGenerator", autospec=True) as mock_workflow_generator, + patch("services.app_generate_service.convert_to_event_stream", autospec=True) as mock_event_stream, patch( - "services.app_generate_service.MessageBasedAppGenerator", autospec=True - ) as mock_message_based_generator, + "services.app_generate_service.WorkflowEventStream.retrieve_events", autospec=True + ) as mock_retrieve_events, patch( "services.account.login_adapters.SystemFeatureService", autospec=True ) as mock_account_feature_service, @@ -102,23 +111,17 @@ def mock_external_service_dependencies(self): mock_completion_generator_instance = mock_completion_generator.return_value mock_completion_generator_instance.generate.return_value = ["completion_response"] mock_completion_generator_instance.generate_more_like_this.return_value = ["more_like_this_response"] - mock_completion_generator.convert_to_event_stream.return_value = ["completion_stream"] mock_chat_generator_instance = mock_chat_generator.return_value mock_chat_generator_instance.generate.return_value = ["chat_response"] - mock_chat_generator.convert_to_event_stream.return_value = ["chat_stream"] mock_agent_chat_generator_instance = mock_agent_chat_generator.return_value mock_agent_chat_generator_instance.generate.return_value = ["agent_chat_response"] - mock_agent_chat_generator.convert_to_event_stream.return_value = ["agent_chat_stream"] mock_advanced_chat_generator_instance = mock_advanced_chat_generator.return_value mock_advanced_chat_generator_instance.generate.return_value = ["advanced_chat_response"] mock_advanced_chat_generator_instance.single_iteration_generate.return_value = ["single_iteration_response"] mock_advanced_chat_generator_instance.single_loop_generate.return_value = ["single_loop_response"] - mock_advanced_chat_generator_instance.retrieve_events.return_value = ["advanced_chat_events"] - mock_advanced_chat_generator_instance.convert_to_event_stream.return_value = ["advanced_chat_stream"] - mock_advanced_chat_generator.convert_to_event_stream.return_value = ["advanced_chat_stream"] mock_workflow_generator_instance = mock_workflow_generator.return_value mock_workflow_generator_instance.generate.return_value = ["workflow_response"] @@ -126,9 +129,8 @@ def mock_external_service_dependencies(self): "workflow_single_iteration_response" ] mock_workflow_generator_instance.single_loop_generate.return_value = ["workflow_single_loop_response"] - mock_workflow_generator.convert_to_event_stream.return_value = ["workflow_stream"] - - mock_message_based_generator.retrieve_events.return_value = ["workflow_events"] + mock_event_stream.side_effect = lambda events: events + mock_retrieve_events.return_value = ["workflow_events"] # Setup default mock returns for account service mock_account_feature_service.is_registration_allowed.return_value = True @@ -157,7 +159,8 @@ def mock_external_service_dependencies(self): "agent_chat_generator": mock_agent_chat_generator, "advanced_chat_generator": mock_advanced_chat_generator, "workflow_generator": mock_workflow_generator, - "message_based_generator": mock_message_based_generator, + "event_stream": mock_event_stream, + "retrieve_events": mock_retrieve_events, "account_feature_service": mock_account_feature_service, "dify_config": mock_dify_config, "quota_dify_config": mock_quota_dify_config, @@ -285,8 +288,9 @@ def test_generate_completion_mode_success( mock_external_service_dependencies["rate_limit"].return_value.generate.assert_called_once() # Verify completion generator was called - mock_external_service_dependencies["completion_generator"].return_value.generate.assert_called_once() - mock_external_service_dependencies["completion_generator"].convert_to_event_stream.assert_called_once() + generator = mock_external_service_dependencies["completion_generator"].return_value + generator.generate.assert_called_once() + mock_external_service_dependencies["event_stream"].assert_called_once_with(generator.generate.return_value) def test_generate_chat_mode_success( self, @@ -323,8 +327,9 @@ def test_generate_chat_mode_success( assert result == ["test_response"] # Verify chat generator was called - mock_external_service_dependencies["chat_generator"].return_value.generate.assert_called_once() - mock_external_service_dependencies["chat_generator"].convert_to_event_stream.assert_called_once() + generator = mock_external_service_dependencies["chat_generator"].return_value + generator.generate.assert_called_once() + mock_external_service_dependencies["event_stream"].assert_called_once_with(generator.generate.return_value) def test_generate_agent_chat_mode_success( self, @@ -361,8 +366,15 @@ def test_generate_agent_chat_mode_success( assert result == ["test_response"] # Verify agent chat generator was called - mock_external_service_dependencies["agent_chat_generator"].return_value.generate.assert_called_once() - mock_external_service_dependencies["agent_chat_generator"].convert_to_event_stream.assert_called_once() + generator_factory = mock_external_service_dependencies["agent_chat_generator"] + generator_factory.assert_called_once() + constructor_args = generator_factory.call_args.kwargs + assert constructor_args["tool_invoker"] is workflow_runtime.agent_tool_invoker + assert constructor_args["draft_variable_saver"] is workflow_variables.saver_factory + assert constructor_args["workflow_runtime"] is workflow_runtime + generator = generator_factory.return_value + generator.generate.assert_called_once() + mock_external_service_dependencies["event_stream"].assert_called_once_with(generator.generate.return_value) def test_generate_advanced_chat_mode_success( self, @@ -398,11 +410,10 @@ def test_generate_advanced_chat_mode_success( # Verify the result assert result == ["test_response"] - # Verify advanced chat generator was called - mock_external_service_dependencies["advanced_chat_generator"].return_value.retrieve_events.assert_called_once() - mock_external_service_dependencies[ - "advanced_chat_generator" - ].return_value.convert_to_event_stream.assert_called_once() + # Streaming workflows read the shared event stream instead of constructing a synchronous generator. + events = mock_external_service_dependencies["retrieve_events"] + events.assert_called_once_with(ANY, ANY, on_subscribe=ANY) + mock_external_service_dependencies["event_stream"].assert_called_once_with(events.return_value) def test_generate_workflow_mode_success( self, @@ -438,9 +449,10 @@ def test_generate_workflow_mode_success( # Verify the result assert result == ["test_response"] - # Verify workflow generator was called - mock_external_service_dependencies["message_based_generator"].retrieve_events.assert_called_once() - mock_external_service_dependencies["workflow_generator"].convert_to_event_stream.assert_called_once() + # Streaming workflows read the shared event stream instead of constructing a synchronous generator. + events = mock_external_service_dependencies["retrieve_events"] + events.assert_called_once_with(ANY, ANY, on_subscribe=ANY) + mock_external_service_dependencies["event_stream"].assert_called_once_with(events.return_value) def test_generate_with_specific_workflow_id( self, @@ -880,7 +892,7 @@ def test_generate_single_iteration_advanced_chat_success( ) # Verify the result - assert result == ["advanced_chat_stream"] + assert result == ["single_iteration_response"] # Verify advanced chat generator was called mock_external_service_dependencies[ @@ -919,7 +931,7 @@ def test_generate_single_iteration_workflow_success( ) # Verify the result - assert result == ["advanced_chat_stream"] + assert result == ["workflow_single_iteration_response"] # Verify workflow generator was called mock_external_service_dependencies[ @@ -993,7 +1005,7 @@ def test_generate_single_loop_advanced_chat_success( ) # Verify the result - assert result == ["advanced_chat_stream"] + assert result == ["single_loop_response"] # Verify advanced chat generator was called mock_external_service_dependencies[ @@ -1032,7 +1044,7 @@ def test_generate_single_loop_workflow_success( ) # Verify the result - assert result == ["advanced_chat_stream"] + assert result == ["workflow_single_loop_response"] # Verify workflow generator was called mock_external_service_dependencies["workflow_generator"].return_value.single_loop_generate.assert_called_once() @@ -1291,8 +1303,9 @@ def test_generate_with_agent_mode_detection( assert result == ["test_response"] # Verify agent chat generator was called instead of regular chat generator - mock_external_service_dependencies["agent_chat_generator"].return_value.generate.assert_called_once() - mock_external_service_dependencies["agent_chat_generator"].convert_to_event_stream.assert_called_once() + generator = mock_external_service_dependencies["agent_chat_generator"].return_value + generator.generate.assert_called_once() + mock_external_service_dependencies["event_stream"].assert_called_once_with(generator.generate.return_value) def test_generate_with_different_invoke_from_values( self, @@ -1395,7 +1408,7 @@ def test_generate_with_complex_args( assert call_kwargs["args"] == args # Verify workflow streaming event retrieval was used - mock_external_service_dependencies["message_based_generator"].retrieve_events.assert_called_once_with( + mock_external_service_dependencies["retrieve_events"].assert_called_once_with( ANY, mock_payload.workflow_run_id, on_subscribe=ANY, diff --git a/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py b/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py index de5c297b43400a..7541d158ca9e7e 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py @@ -1,6 +1,6 @@ from __future__ import annotations -from unittest.mock import Mock, patch +from unittest.mock import Mock, create_autospec, patch import pytest from faker import Faker @@ -830,7 +830,9 @@ def _make_controller(provider_id="provider_123", *, workflow_queries, **identity identity = ToolProviderIdentity(**defaults) entity = ToolProviderEntity(identity=identity) return WorkflowToolProviderController( - draft_variable_saver=Mock(return_value=Mock(spec=DraftVariableSaverFactory)), + draft_variable_saver=Mock( + return_value=create_autospec(DraftVariableSaverFactory, instance=True, spec_set=True) + ), entity=entity, provider_id=provider_id, queries=workflow_queries, diff --git a/api/tests/test_containers_integration_tests/services/workflow/test_workflow_converter.py b/api/tests/test_containers_integration_tests/services/workflow/test_workflow_converter.py index 623e371385e1e6..cf0dec1e8015f8 100644 --- a/api/tests/test_containers_integration_tests/services/workflow/test_workflow_converter.py +++ b/api/tests/test_containers_integration_tests/services/workflow/test_workflow_converter.py @@ -12,15 +12,21 @@ from models.model import App, AppMode, AppModelConfig, InstalledApp, Site from models.workflow import Workflow from repositories.app.console_repository import ConsoleAppRepository +from services.entities.app_entities import AppEvent from services.workflow.conversion_service import WorkflowConversionService +from services.workflow.workflow_converter import WorkflowConverter + +type ConversionSource = tuple[sessionmaker[Session], RequestContext, str] @pytest.fixture -def conversion_source(db_session_with_containers: Session): +def conversion_source(db_session_with_containers: Session) -> ConversionSource: class ConversionSession(Session): pass - sessions = sessionmaker(db_session_with_containers.get_bind(), class_=ConversionSession, expire_on_commit=False) + sessions: sessionmaker[Session] = sessionmaker[Session]( + db_session_with_containers.get_bind(), class_=ConversionSession, expire_on_commit=False + ) account_id, tenant_id, app_id, config_id = (str(uuid4()) for _ in range(4)) with sessions.begin() as session: session.add_all( @@ -65,15 +71,17 @@ class ConversionSession(Session): @pytest.mark.parametrize("mode", [AppMode.CHAT, AppMode.COMPLETION]) -def test_conversion_creates_complete_target_before_notification(conversion_source, mode: AppMode): +def test_conversion_creates_complete_target_before_notification( + conversion_source: ConversionSource, mode: AppMode +) -> None: sessions, context, app_id = conversion_source with sessions.begin() as session: app = session.get(App, app_id) assert app is not None app.mode = mode - observed = [] + observed: list[str] = [] - def created(*, event, account_id, backing_agent_id): + def created(*, event: AppEvent, account_id: str, backing_agent_id: str | None) -> None: assert (account_id, backing_agent_id) == (context.account_id, None) with sessions() as session: assert session.scalar(select(Workflow).where(Workflow.app_id == event.id)) is not None @@ -82,7 +90,10 @@ def created(*, event, account_id, backing_agent_id): observed.append(event.id) service = WorkflowConversionService( - ConsoleAppRepository(session_factory=sessions), decrypt_token=lambda *_: "key", notify_created=created + ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), + decrypt_token=lambda *_: "key", + notify_created=created, ) new_id = service.convert(context, app_id, {"name": "Converted", "icon": "🚀"}) assert observed == [new_id] @@ -104,19 +115,26 @@ def created(*, event, account_id, backing_agent_id): ] -def test_workflow_write_failure_rolls_back_target_and_required_records(conversion_source): +def test_workflow_write_failure_rolls_back_target_and_required_records( + conversion_source: ConversionSource, +) -> None: sessions, context, app_id = conversion_source - def reject_workflow(session, *_): + def reject_workflow(session: Session, *_: object) -> None: if any(isinstance(row, Workflow) for row in session.new): raise RuntimeError("workflow write failed") event.listen(sessions.class_, "before_flush", reject_workflow) - notifications = [] + notifications: list[dict[str, object]] = [] + + def notify_created(*, event: AppEvent, account_id: str, backing_agent_id: str | None) -> None: + notifications.append({"event": event, "account_id": account_id, "backing_agent_id": backing_agent_id}) + service = WorkflowConversionService( ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), decrypt_token=lambda *_: "key", - notify_created=lambda **kwargs: notifications.append(kwargs), + notify_created=notify_created, ) with pytest.raises(RuntimeError, match="workflow write failed"): service.convert(context, app_id, {}) diff --git a/api/tests/unit_tests/commands/test_data_migration_commands.py b/api/tests/unit_tests/commands/test_data_migration_commands.py index 8924ef4e5e58c9..8ef0d26261c8d4 100644 --- a/api/tests/unit_tests/commands/test_data_migration_commands.py +++ b/api/tests/unit_tests/commands/test_data_migration_commands.py @@ -6,7 +6,7 @@ import json from pathlib import Path -from unittest.mock import Mock +from unittest.mock import create_autospec from click.testing import CliRunner from sqlalchemy.engine import Engine @@ -152,8 +152,8 @@ def test_import_command_uses_cli_owned_session( } ) - publisher = Mock(spec=ConsoleWorkflowService) - services = Mock(spec=ext_application_services.ApplicationServices) + publisher = create_autospec(ConsoleWorkflowService, instance=True) + services = create_autospec(ext_application_services.ApplicationServices, instance=True) services.console_workflows = publisher services.tools = workflow_application.tools monkeypatch.setattr(data_migration, "application_services", lambda: services) diff --git a/api/tests/unit_tests/conftest.py b/api/tests/unit_tests/conftest.py index 7668c9cb7cc39b..13364fef584d29 100644 --- a/api/tests/unit_tests/conftest.py +++ b/api/tests/unit_tests/conftest.py @@ -351,7 +351,11 @@ def account_application_services( @pytest.fixture -def app_services(sqlite_session_factory: sessionmaker[Session], tool_providers, workflow_queries) -> AppServices: +def app_services( + sqlite_session_factory: sessionmaker[Session], + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, +) -> AppServices: from unittest.mock import Mock from extensions.application_services.app import build_app_services diff --git a/api/tests/unit_tests/controllers/conftest.py b/api/tests/unit_tests/controllers/conftest.py index 7b90a95ec4db05..d59e1a9d536efe 100644 --- a/api/tests/unit_tests/controllers/conftest.py +++ b/api/tests/unit_tests/controllers/conftest.py @@ -32,7 +32,9 @@ @pytest.fixture(autouse=True) def datasource_application_dependencies( - monkeypatch: pytest.MonkeyPatch, workflow_variables: WorkflowExecutionVariables, sqlite_session_factory + monkeypatch: pytest.MonkeyPatch, + workflow_variables: WorkflowExecutionVariables, + sqlite_session_factory: sessionmaker[Session], ) -> None: registry = DatasourceApplicationStub( tools=build_tool_services(sqlite_session_factory), @@ -85,7 +87,9 @@ class PipelineApplicationStub: @pytest.fixture def pipeline_application( - monkeypatch: pytest.MonkeyPatch, workflow_variables: WorkflowExecutionVariables, sqlite_session_factory + monkeypatch: pytest.MonkeyPatch, + workflow_variables: WorkflowExecutionVariables, + sqlite_session_factory: sessionmaker[Session], ) -> PipelineGenerator: from controllers.console.datasets.rag_pipeline import rag_pipeline_workflow as console_workflow from controllers.service_api.dataset.rag_pipeline import rag_pipeline_workflow as service_api_workflow diff --git a/api/tests/unit_tests/controllers/console/agent/test_agent_controllers.py b/api/tests/unit_tests/controllers/console/agent/test_agent_controllers.py index c2526f54a7e52c..38daffb9b5cb46 100644 --- a/api/tests/unit_tests/controllers/console/agent/test_agent_controllers.py +++ b/api/tests/unit_tests/controllers/console/agent/test_agent_controllers.py @@ -1603,7 +1603,11 @@ def test_workflow_impact_returns_empty_without_version(app: Flask) -> None: def test_agent_composer_routes_resolve_app_from_agent_id( - app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str, *, workflow_application + app: Flask, + monkeypatch: pytest.MonkeyPatch, + account_id: str, + *, + workflow_application: ApplicationServices, ) -> None: agent_id = "00000000-0000-0000-0000-000000000001" captured: dict[str, object] = {} diff --git a/api/tests/unit_tests/controllers/console/app/test_model_config_api.py b/api/tests/unit_tests/controllers/console/app/test_model_config_api.py index bc13892a4d6c66..1c91aa11cfcaad 100644 --- a/api/tests/unit_tests/controllers/console/app/test_model_config_api.py +++ b/api/tests/unit_tests/controllers/console/app/test_model_config_api.py @@ -863,6 +863,12 @@ def test_post_encrypts_agent_tool_parameters( }, }, ) + tool_services = SimpleNamespace(tool_providers=object(), workflow_queries=object()) + monkeypatch.setattr( + model_config_module, + "application_services", + lambda: SimpleNamespace(tools=tool_services), + ) monkeypatch.setattr(model_config_module.ToolManager, "get_agent_tool_runtime", lambda **_kwargs: object()) class _ParamManager: diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow.py b/api/tests/unit_tests/controllers/console/app/test_workflow.py index 388057fd89b769..97f4a83ba99a74 100644 --- a/api/tests/unit_tests/controllers/console/app/test_workflow.py +++ b/api/tests/unit_tests/controllers/console/app/test_workflow.py @@ -3,7 +3,7 @@ import inspect from dataclasses import dataclass from datetime import UTC, datetime -from unittest.mock import Mock +from unittest.mock import MagicMock, Mock, create_autospec from uuid import UUID import pytest @@ -33,8 +33,8 @@ class Services: @pytest.fixture(name="workflows") -def workflow_use_cases(monkeypatch: pytest.MonkeyPatch) -> Mock: - use_cases = Mock(spec=ConsoleWorkflowService) +def workflow_use_cases(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + use_cases = create_autospec(ConsoleWorkflowService, instance=True, spec_set=True) monkeypatch.setattr(controller, "application_services", lambda: Services(use_cases)) return use_cases @@ -159,7 +159,7 @@ def test_restore_domain_error_http_response(workflows: Mock, failure: Exception, api.error_handlers = console_api.error_handlers.copy() class Restore(Resource): - def post(self): + def post(self) -> object: return invoke(controller.DraftWorkflowRestoreApi, workflow_id="version") api.add_resource(Restore, "/restore") diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow_admission.py b/api/tests/unit_tests/controllers/console/app/test_workflow_admission.py index 886e433a8d9f4d..c5778edfb83c36 100644 --- a/api/tests/unit_tests/controllers/console/app/test_workflow_admission.py +++ b/api/tests/unit_tests/controllers/console/app/test_workflow_admission.py @@ -5,7 +5,7 @@ from collections.abc import Callable, Sequence from dataclasses import dataclass from inspect import unwrap -from unittest.mock import Mock +from unittest.mock import create_autospec from uuid import UUID import pytest @@ -113,7 +113,8 @@ def test_context_is_stable_and_app_mode_is_checked(app: Flask, monkeypatch: pyte monkeypatch.setattr(flask_admission, "get_request_id", lambda: "request") monkeypatch.setattr(flask_admission, "get_trace_id", lambda: None) monkeypatch.setattr(flask_admission, "enforce_rbac_checks", lambda **_kwargs: None) - apps, use_cases = Mock(spec=ConsoleAppService), Mock(spec=ConsoleWorkflowService) + apps = create_autospec(ConsoleAppService, instance=True) + use_cases = create_autospec(ConsoleWorkflowService, instance=True) apps.get_reference.return_value = AppReference(str(APP_ID), "App", mode, None) use_cases.published.return_value = None services = Services(Apps(apps), use_cases) @@ -139,7 +140,8 @@ def test_revision_conflict_is_translated_to_existing_409(app: Flask, monkeypatch lambda: AccountWithTenant(make_account(role=TenantAccountRole.EDITOR), "tenant"), ) monkeypatch.setattr(flask_admission, "enforce_rbac_checks", lambda **_kwargs: None) - apps, use_cases = Mock(spec=ConsoleAppService), Mock(spec=ConsoleWorkflowService) + apps = create_autospec(ConsoleAppService, instance=True) + use_cases = create_autospec(ConsoleWorkflowService, instance=True) apps.get_reference.return_value = AppReference(str(APP_ID), "App", "workflow", None) use_cases.publish.side_effect = WorkflowHashNotEqualError() services = Services(Apps(apps), use_cases) diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py b/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py index 76137cb7e90837..be760bd7c69863 100644 --- a/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py +++ b/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py @@ -1,6 +1,6 @@ """Human input transport delegates parsed payloads to application services.""" -from unittest.mock import Mock +from unittest.mock import Mock, create_autospec import pytest from flask import Flask @@ -19,8 +19,8 @@ @pytest.fixture(name="workflows") def debug_use_cases(monkeypatch: pytest.MonkeyPatch) -> Mock: - service = Mock(spec=HumanInputDebugService) - dependencies = Mock(spec=ApplicationServices) + service = create_autospec(HumanInputDebugService, instance=True) + dependencies = create_autospec(ApplicationServices, instance=True) dependencies.human_input_debug = service monkeypatch.setattr(controller, "application_services", lambda: dependencies) return service diff --git a/api/tests/unit_tests/controllers/console/app/test_wraps.py b/api/tests/unit_tests/controllers/console/app/test_wraps.py index f090f14a51b573..e4a1c855ece934 100644 --- a/api/tests/unit_tests/controllers/console/app/test_wraps.py +++ b/api/tests/unit_tests/controllers/console/app/test_wraps.py @@ -1,6 +1,8 @@ from __future__ import annotations +from collections.abc import Callable from inspect import getsource +from typing import cast from uuid import uuid4 import pytest @@ -8,7 +10,6 @@ from controllers.common.session import with_session from controllers.console.app import completion as completion_module -from controllers.console.app import workflow as workflow_module from controllers.console.app import wraps as wraps_module from controllers.console.app.error import AppNotFoundError from models.agent import Agent, AgentScope, AgentSource, AgentStatus @@ -35,10 +36,10 @@ def test_get_app_model_injects_model(monkeypatch: pytest.MonkeyPatch, sqlite_ses monkeypatch.setattr(wraps_module.db, "session", sqlite_session) @wraps_module.get_app_model - def handler(app_model): + def handler(app_model: App) -> str: return app_model.id - assert handler(app_id=app_model.id) == app_model.id + assert cast(Callable[..., str], handler)(app_id=app_model.id) == app_model.id def test_get_app_model_rejects_wrong_mode(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session) -> None: @@ -47,20 +48,20 @@ def test_get_app_model_rejects_wrong_mode(monkeypatch: pytest.MonkeyPatch, sqlit monkeypatch.setattr(wraps_module.db, "session", sqlite_session) @wraps_module.get_app_model(mode=[AppMode.COMPLETION]) - def handler(app_model): + def handler(app_model: App) -> str: return app_model.id with pytest.raises(AppNotFoundError): - handler(app_id=app_model.id) + cast(Callable[..., str], handler)(app_id=app_model.id) def test_get_app_model_requires_app_id() -> None: @wraps_module.get_app_model - def handler(app_model): + def handler(app_model: App) -> str: return app_model.id with pytest.raises(ValueError): - handler() + cast(Callable[..., str], handler)() def test_wraps_with_session_reexports_common_session_decorator() -> None: @@ -76,14 +77,15 @@ def test_get_app_model_prefers_injected_session( class Handler: @wraps_module.get_app_model - def get(self, _injected_session, app_model): + def get(self, _injected_session: Session, app_model: App) -> str: return app_model.id # An unbound real Session fails on query, so success proves the injected # request Session was preferred over the legacy scoped-session fallback. with Session() as scoped_session: monkeypatch.setattr(wraps_module.db, "session", scoped_session) - assert Handler().get(sqlite_session, app_id=app_model.id) == app_model.id + get = cast(Callable[..., str], Handler().get) + assert get(sqlite_session, app_id=app_model.id) == app_model.id @pytest.mark.parametrize("scope", [AgentScope.ROSTER, AgentScope.WORKFLOW_ONLY]) @@ -119,11 +121,12 @@ def get(self, *_args: object, app_model: App) -> str: return app_model.id args = (sqlite_session,) if inject_session else () + get = cast(Callable[..., str], Handler().get) if workflow_only: with pytest.raises(AppNotFoundError): - Handler().get(*args, app_id=app_model.id) + get(*args, app_id=app_model.id) else: - assert Handler().get(*args, app_id=app_model.id) == app_model.id + assert get(*args, app_id=app_model.id) == app_model.id @pytest.mark.parametrize( @@ -131,11 +134,7 @@ def get(self, *_args: object, app_model: App) -> str: [ completion_module.CompletionMessageApi, completion_module.ChatMessageApi, - workflow_module.AdvancedChatDraftWorkflowRunApi, - workflow_module.DraftWorkflowRunApi, - workflow_module.DraftWorkflowTriggerRunApi, - workflow_module.DraftWorkflowTriggerRunAllApi, ], ) -def test_migrated_handlers_open_session_before_app_lookup(resource: type) -> None: +def test_legacy_handlers_open_session_before_app_lookup(resource: type) -> None: assert "@with_session\n @get_app_model" in getsource(resource) diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow_apis.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow_apis.py index 20cc81c257137f..c6900e34f14036 100644 --- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow_apis.py +++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow_apis.py @@ -8,7 +8,7 @@ from datetime import datetime from inspect import unwrap from typing import TypedDict, Unpack -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, Mock, create_autospec, patch from uuid import uuid4 import pytest @@ -54,6 +54,7 @@ from models.dataset import Pipeline from models.enums import CreatorUserRole from models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom +from repositories.tools.provider_repository import ToolProviderRepository from repositories.workflow.definition_repository import workflow_record, workflow_snapshot from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError from services.errors.workflow_service import WorkflowInUseError @@ -319,7 +320,7 @@ def test_sync_hash_not_match(self, app: Flask, workflow_application: Application pipeline = make_pipeline() user = make_account() - service = Mock(spec=WorkflowDraftService) + service = create_autospec(WorkflowDraftService, instance=True) service.sync.side_effect = WorkflowHashNotEqualError() with ( @@ -357,7 +358,7 @@ def test_restore_published_workflow_to_draft_success( created_at=datetime(2024, 1, 1), ) - service = Mock(spec=WorkflowDraftService) + service = create_autospec(WorkflowDraftService, instance=True) service.restore.return_value = workflow_snapshot(workflow) with ( @@ -382,7 +383,7 @@ def test_restore_published_workflow_to_draft_not_found( pipeline = make_pipeline() user = make_account(id="account-1") - service = Mock(spec=WorkflowDraftService) + service = create_autospec(WorkflowDraftService, instance=True) service.restore.side_effect = WorkflowNotFoundError("Workflow not found") with ( @@ -405,7 +406,7 @@ def test_restore_published_workflow_to_draft_returns_400_for_draft_source( pipeline = make_pipeline() user = make_account(id="account-1") - service = Mock(spec=WorkflowDraftService) + service = create_autospec(WorkflowDraftService, instance=True) service.restore.side_effect = IsDraftWorkflowError("source workflow must be published") with ( @@ -759,7 +760,9 @@ def test_last_run_not_found(self, app: Flask) -> None: class TestRagPipelineWorkflowRunNodeExecutionListApi: - def test_get_node_executions_passes_current_user(self, app: Flask, *, tool_providers) -> None: + def test_get_node_executions_passes_current_user( + self, app: Flask, *, tool_providers: ToolProviderRepository + ) -> None: api = RagPipelineWorkflowRunNodeExecutionListApi() method = unwrap(api.get) diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py index 92b64d4875f89c..3ada7cecd7e02f 100644 --- a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py +++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py @@ -1,24 +1,31 @@ """Document downloads through the controller, application service, and SQLite adapter.""" import json +from collections.abc import Iterator from datetime import datetime from inspect import unwrap from io import BytesIO +from os import PathLike from pathlib import Path from types import SimpleNamespace +from typing import Never from unittest.mock import create_autospec from uuid import UUID from zipfile import ZipFile import pytest -from flask import send_file -from sqlalchemy import event +from flask import Flask, Response, send_file +from sqlalchemy import Engine, event +from sqlalchemy.orm import Session, sessionmaker from werkzeug.exceptions import NotFound from controllers.console.datasets import datasets_document as controller +from extensions.storage.storage_type import StorageType from machinery.context import RequestContext from models.dataset import Dataset, Document +from models.enums import CreatorUserRole from models.model import UploadFile +from services.errors.document import DocumentSourceNotFoundError from services.knowledge.dataset_access import AccessibleDataset, DatasetAccess from services.knowledge.documents.adapters import SQLAlchemyDocumentOperations from services.knowledge.documents.application import DatasetDocumentApplicationService @@ -28,7 +35,7 @@ FILE_ID = str(UUID(int=2)) -def _document(**values): +def _document(**values: object) -> Document: return Document( **{ "id": DOC_ID, @@ -47,19 +54,21 @@ def _document(**values): @pytest.fixture(autouse=True) -def downloads(monkeypatch, sqlite_session_factory): +def downloads( + monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session] +) -> DatasetDocumentApplicationService: with sqlite_session_factory.begin() as session: session.add(Dataset(id="dataset-1", tenant_id="tenant-1", name="Dataset", created_by="actor-1")) session.add(_document()) upload = UploadFile( tenant_id="tenant-1", - storage_type="local", + storage_type=StorageType.LOCAL, key="key-1", name="document.txt", size=5, extension="txt", mime_type="text/plain", - created_by_role="account", + created_by_role=CreatorUserRole.ACCOUNT, created_by="actor-1", created_at=datetime(2024, 1, 1), used=False, @@ -79,12 +88,12 @@ def downloads(monkeypatch, sqlite_session_factory): return documents -def test_document_download_returns_signed_url(app, monkeypatch): +def test_document_download_returns_signed_url(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: from services.knowledge import dataset_service - signed = [] + signed: list[dict[str, object]] = [] - def sign(**kwargs): + def sign(**kwargs: object) -> str: signed.append(kwargs) return "https://files.example/download" @@ -98,52 +107,62 @@ def sign(**kwargs): @pytest.mark.parametrize( - "values", + ("values", "expected_error"), [ - {"data_source_type": "website_crawl"}, - {"data_source_info": "{}"}, - {"data_source_info": json.dumps({"upload_file_id": "missing"})}, - {"tenant_id": "foreign"}, - {"dataset_id": "foreign"}, + ({"data_source_type": "website_crawl"}, DocumentSourceNotFoundError), + ({"data_source_info": "{}"}, DocumentSourceNotFoundError), + ({"data_source_info": json.dumps({"upload_file_id": "missing"})}, DocumentSourceNotFoundError), + ({"tenant_id": "foreign"}, NotFound), + ({"dataset_id": "foreign"}, NotFound), ], ) -def test_invalid_or_foreign_document_is_not_downloadable(app, sqlite_session_factory, values): +def test_invalid_or_foreign_document_is_not_downloadable( + app: Flask, + sqlite_session_factory: sessionmaker[Session], + values: dict[str, object], + expected_error: type[BaseException], +) -> None: with sqlite_session_factory.begin() as session: document = session.get(Document, DOC_ID) + assert document is not None for key, value in values.items(): setattr(document, key, value) - with app.test_request_context("/"), pytest.raises(NotFound): + with app.test_request_context("/"), pytest.raises(expected_error): unwrap(controller.DocumentDownloadApi.get)(controller.DocumentDownloadApi(), CONTEXT, "dataset-1", DOC_ID) -def test_cross_tenant_upload_file_is_not_signed(app, sqlite_session_factory): +def test_cross_tenant_upload_file_is_not_signed(app: Flask, sqlite_session_factory: sessionmaker[Session]) -> None: with sqlite_session_factory.begin() as session: - session.get(UploadFile, FILE_ID).tenant_id = "foreign" - with app.test_request_context("/"), pytest.raises(NotFound): + upload = session.get(UploadFile, FILE_ID) + assert upload is not None + upload.tenant_id = "foreign" + with app.test_request_context("/"), pytest.raises(DocumentSourceNotFoundError): unwrap(controller.DocumentDownloadApi.get)(controller.DocumentDownloadApi(), CONTEXT, "dataset-1", DOC_ID) -def test_zip_is_readable_and_cleanup_waits_for_response_close(app, monkeypatch, sqlite_engine): +def test_zip_is_readable_and_cleanup_waits_for_response_close( + app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine +) -> None: from services import file_service - paths = [] - checked_out = set() + paths: list[Path] = [] + checked_out: set[object] = set() - def checkout(_connection, record, _proxy): + def checkout(_connection: object, record: object, _proxy: object) -> None: checked_out.add(record) - def checkin(_connection, record): + def checkin(_connection: object, record: object) -> None: checked_out.discard(record) - def load(key, *, stream): + def load(key: str, *, stream: bool) -> Iterator[bytes]: assert not checked_out, "storage I/O must run after the read transaction closes" assert key == "key-1" assert stream is True return iter([b"hello"]) - def send(path, **kwargs): + def send(path: str | PathLike[str], *, mimetype: str, as_attachment: bool, download_name: str) -> Response: paths.append(Path(path)) - return send_file(path, **kwargs) + return send_file(path, mimetype=mimetype, as_attachment=as_attachment, download_name=download_name) monkeypatch.setattr(file_service.storage, "load", load) monkeypatch.setattr(controller, "send_file", send) @@ -165,13 +184,13 @@ def send(path, **kwargs): event.remove(sqlite_engine, "checkin", checkin) -def test_zip_cleanup_when_response_creation_fails(app, monkeypatch): +def test_zip_cleanup_when_response_creation_fails(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: from services import file_service - paths = [] + paths: list[Path] = [] monkeypatch.setattr(file_service.storage, "load", lambda *_args, **_kwargs: iter([b"hello"])) - def fail_send(path, **_kwargs): + def fail_send(path: str | PathLike[str], **_kwargs: object) -> Never: paths.append(Path(path)) raise RuntimeError("response failed") @@ -187,10 +206,12 @@ def fail_send(path, **_kwargs): assert not paths[0].exists() -def test_zip_rejects_non_upload_document(app, sqlite_session_factory): +def test_zip_rejects_non_upload_document(app: Flask, sqlite_session_factory: sessionmaker[Session]) -> None: with sqlite_session_factory.begin() as session: - session.get(Document, DOC_ID).data_source_type = "website_crawl" - with app.test_request_context("/", json={"document_ids": [DOC_ID]}), pytest.raises(NotFound): + document = session.get(Document, DOC_ID) + assert document is not None + document.data_source_type = "website_crawl" + with app.test_request_context("/", json={"document_ids": [DOC_ID]}), pytest.raises(DocumentSourceNotFoundError): unwrap(controller.DocumentBatchDownloadZipApi.post)( controller.DocumentBatchDownloadZipApi(), CONTEXT, "dataset-1" ) diff --git a/api/tests/unit_tests/controllers/console/explore/test_installed_app_completion.py b/api/tests/unit_tests/controllers/console/explore/test_installed_app_completion.py index 7203e4b1124741..a1c8e27302b2f6 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_installed_app_completion.py +++ b/api/tests/unit_tests/controllers/console/explore/test_installed_app_completion.py @@ -9,6 +9,7 @@ import controllers.console.explore.completion as completion_module import services.installed_app_generation_service as completion_service_module from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError +from extensions.application_services.workflow import WorkflowExecutionDependencies from graphon.model_runtime.errors.invoke import InvokeError from models import App, AppMode, InstalledApp, Tenant from repositories.installed_app_repository import SQLAlchemyInstalledAppRepository @@ -18,7 +19,6 @@ from services.errors.conversation import ConversationCompletedError, ConversationNotExistsError from services.installed_app_generation_adapters import AppGenerateServiceRuntime from services.installed_app_generation_service import GenerationResponse, InstalledAppGenerationService -from services.workflow.execution.ports import WorkflowRuntime from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.unit_tests.controllers.console.explore.test_installed_app_admission import ( _assert_json_response, @@ -395,7 +395,7 @@ def test_resource_removed_after_admission_is_revalidated_before_generation( deleted_resource: str, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: WorkflowExecutionDependencies, ) -> None: def delete_resource() -> None: with sqlite_session_factory.begin() as session: diff --git a/api/tests/unit_tests/controllers/console/explore/test_trial_preview_details.py b/api/tests/unit_tests/controllers/console/explore/test_trial_preview_details.py index dde68bb212326a..18134aeb9f6376 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_trial_preview_details.py +++ b/api/tests/unit_tests/controllers/console/explore/test_trial_preview_details.py @@ -124,8 +124,8 @@ def tool_runtime( app_id: str, agent_tool: AgentToolEntity, user_id: str, - tool_providers, - workflow_queries, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> Tool: del workflow_queries, tool_providers assert isinstance(self.engine.pool, QueuePool) @@ -244,8 +244,8 @@ def harness( sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], *, - tool_providers, - workflow_queries, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> _Harness: config_overrides( LOGIN_DISABLED=False, @@ -541,7 +541,13 @@ def hardcoded_provider(provider_id: str) -> object: raise KeyError(provider_id) def tool_runtime( - *, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str, tool_providers, workflow_queries + *, + tenant_id: str, + app_id: str, + agent_tool: AgentToolEntity, + user_id: str, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> Tool: if agent_tool.provider_id != "example": raise ValueError("Invalid historical provider ID") @@ -629,7 +635,13 @@ def get_runtime_parameters(self) -> list[ToolParameter]: ) def tool_runtime( - *, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str, tool_providers, workflow_queries + *, + tenant_id: str, + app_id: str, + agent_tool: AgentToolEntity, + user_id: str, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> Tool: if agent_tool.provider_id == "example": return harness.io.tool_runtime( diff --git a/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py b/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py index 8007a070d9efdc..0041918c7c545c 100644 --- a/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py +++ b/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py @@ -9,7 +9,7 @@ import inspect import json -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, Mock, create_autospec, patch import pytest from flask import Flask @@ -140,7 +140,8 @@ def test_plugin_invokes_persisted_workflow_tool_through_composed_runtime( # Any fallback to the global database instead of the composed repositories must fail. monkeypatch.setattr("core.db.session_factory._session_maker", sessionmaker()) monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", lambda *_args, **_kwargs: None) - monkeypatch.setattr(generator_module, "TraceQueueManager", Mock(return_value=Mock(spec=TraceQueueManager))) + trace_queue_manager = create_autospec(TraceQueueManager, instance=True, spec_set=True) + monkeypatch.setattr(generator_module, "TraceQueueManager", Mock(return_value=trace_queue_manager)) monkeypatch.setattr(generator_module.WorkflowAppGenerator, "_generate_worker", staticmethod(lambda **_kwargs: None)) handled = [] diff --git a/api/tests/unit_tests/controllers/inner_api/test_agent_tools.py b/api/tests/unit_tests/controllers/inner_api/test_agent_tools.py index fbbb324700d6f2..91d4800d2b899a 100644 --- a/api/tests/unit_tests/controllers/inner_api/test_agent_tools.py +++ b/api/tests/unit_tests/controllers/inner_api/test_agent_tools.py @@ -2,14 +2,13 @@ from collections.abc import Generator from contextlib import contextmanager +from dataclasses import dataclass from unittest.mock import Mock, create_autospec, patch import pytest from flask import Flask from controllers.inner_api import bp as inner_api_bp -from extensions.application_services.agent import AgentAppServices -from extensions.ext_application_services import ApplicationServices from services.agent.tool_invocation_service import AgentToolInnerService from services.entities.agent_tool_inner import AgentToolInvokeRequest, AgentToolInvokeResponse from services.errors.agent_tool_inner import AgentToolInnerServiceError @@ -58,10 +57,20 @@ def _agent_inner_auth() -> Generator[None]: yield +@dataclass(frozen=True) +class _AgentApps: + tools: AgentToolInnerService + + +@dataclass(frozen=True) +class _ApplicationServices: + agent_apps: _AgentApps + + @contextmanager def _tool_service() -> Generator[Mock]: service = create_autospec(AgentToolInnerService, instance=True, spec_set=True) - services = Mock(spec=ApplicationServices, agent_apps=Mock(spec=AgentAppServices, tools=service)) + services = _ApplicationServices(agent_apps=_AgentApps(tools=service)) with patch("controllers.inner_api.agent.tools.application_services", return_value=services): yield service.invoke diff --git a/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py b/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py index 66ac14e5facff5..6249da2a317a7b 100644 --- a/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py +++ b/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py @@ -47,8 +47,10 @@ from repositories.api_workflow_node_execution_repository import WorkflowNodeExecutionSnapshot from repositories.entities.workflow_pause import WorkflowPauseEntity from services.app.generation.ports import ConversationSnapshot, MessageSnapshot, WorkflowSnapshot +from services.app.generation.runtime import AppGenerationRuntime from services.app_generate_service import AppGenerateService from services.workflow.execution.adapters.response_converter import WorkflowResponseConverter +from services.workflow.variable_contracts import WorkflowExecutionVariables from services.workflow_event_snapshot_service import _build_snapshot_events from tests.unit_tests.config_override import apply_config_overrides @@ -417,6 +419,9 @@ def test_advanced_chat_blocking_injects_pause_state_config( self, monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, ) -> None: apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) monkeypatch.setattr(ags_module, "RateLimit", _DummyRateLimit) @@ -428,9 +433,9 @@ def test_advanced_chat_blocking_injects_pause_state_config( generator_instance = Mock() generator_instance.generate.return_value = {"result": "advanced-blocking"} - generator_instance.convert_to_event_stream.side_effect = lambda payload: payload generator_factory = Mock(return_value=generator_instance) monkeypatch.setattr(ags_module, "AdvancedChatAppGenerator", generator_factory) + monkeypatch.setattr(ags_module, "convert_to_event_stream", lambda payload: payload) app_model = _app(app_id="app-id", tenant_id="tenant-id", mode=AppMode.ADVANCED_CHAT) user = _end_user(user_id="user-id", app_id="app-id", tenant_id="tenant-id") @@ -443,10 +448,16 @@ def test_advanced_chat_blocking_injects_pause_state_config( args={"workflow_id": None, "query": "hi", "inputs": {}}, invoke_from=InvokeFrom.SERVICE_API, streaming=False, + variables=workflow_variables, + runtime=workflow_runtime, ) assert result == {"result": "advanced-blocking"} - generator_factory.assert_called_once_with() + generator_factory.assert_called_once_with( + runtime=workflow_runtime, + draft_variable_loader=workflow_variables.workflow_loader, + draft_variable_saver=workflow_variables.saver_factory, + ) call_kwargs = generator_instance.generate.call_args.kwargs assert call_kwargs["streaming"] is False assert call_kwargs["pause_state_config"] is not None diff --git a/api/tests/unit_tests/controllers/trigger/test_webhook.py b/api/tests/unit_tests/controllers/trigger/test_webhook.py index ad6f55805fe624..388d6f0d95520f 100644 --- a/api/tests/unit_tests/controllers/trigger/test_webhook.py +++ b/api/tests/unit_tests/controllers/trigger/test_webhook.py @@ -1,3 +1,4 @@ +from collections.abc import Callable from unittest.mock import patch import pytest @@ -5,7 +6,6 @@ from werkzeug.exceptions import RequestEntityTooLarge import controllers.trigger.webhook as module -from configs import dify_config from controllers.common.errors import NotFoundError from models.trigger import WorkflowWebhookTrigger from models.workflow import Workflow @@ -61,9 +61,11 @@ def _workflow() -> Workflow: @pytest.mark.parametrize("path", ["webhook", "webhook-debug"]) -def test_webhook_body_limit_remains_http_413(monkeypatch, path): +def test_webhook_body_limit_remains_http_413( + monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], path: str +) -> None: app = Flask(__name__) - monkeypatch.setattr(dify_config, "WEBHOOK_REQUEST_BODY_MAX_SIZE", 8) + config_overrides(WEBHOOK_REQUEST_BODY_MAX_SIZE=8) monkeypatch.setattr( module.WebhookService, "get_webhook_trigger_and_workflow", @@ -76,10 +78,12 @@ def test_webhook_body_limit_remains_http_413(monkeypatch, path): @pytest.mark.parametrize("path", ["webhook", "webhook-debug"]) -def test_framework_body_limit_is_not_translated_to_internal_error(monkeypatch, path): +def test_framework_body_limit_is_not_translated_to_internal_error( + monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], path: str +) -> None: app = Flask(__name__) app.config["MAX_CONTENT_LENGTH"] = 8 - monkeypatch.setattr(dify_config, "WEBHOOK_REQUEST_BODY_MAX_SIZE", 1024) + config_overrides(WEBHOOK_REQUEST_BODY_MAX_SIZE=1024) monkeypatch.setattr( module.WebhookService, "get_webhook_trigger_and_workflow", diff --git a/api/tests/unit_tests/core/app/apps/agent_chat/test_agent_chat_app_generator.py b/api/tests/unit_tests/core/app/apps/agent_chat/test_agent_chat_app_generator.py index e2d36d6fb37528..63ecf54aebbf4a 100644 --- a/api/tests/unit_tests/core/app/apps/agent_chat/test_agent_chat_app_generator.py +++ b/api/tests/unit_tests/core/app/apps/agent_chat/test_agent_chat_app_generator.py @@ -6,7 +6,7 @@ import pytest from pydantic import ValidationError from pytest_mock import MockerFixture -from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.orm import Session from core.app.apps.exc import GenerateTaskStoppedError from core.app.entities.app_invoke_entities import InvokeFrom @@ -84,16 +84,15 @@ def _message() -> Message: def generator( app_records, mocker: MockerFixture, - sqlite_session_factory: sessionmaker[Session], *, annotation_replies, workflow_runtime, ): gen = AgentChatAppGenerator( dataset_tools=workflow_runtime.dataset_tools, + tool_invoker=workflow_runtime.agent_tool_invoker, annotations=annotation_replies, records=app_records, - tool_sessions=sqlite_session_factory, ) mocker.patch( "services.app.generation.adapters.agent_chat.current_app", diff --git a/api/tests/unit_tests/core/plugin/test_backwards_invocation_app.py b/api/tests/unit_tests/core/plugin/test_backwards_invocation_app.py index 015420cfdca315..7e5b26d9372f51 100644 --- a/api/tests/unit_tests/core/plugin/test_backwards_invocation_app.py +++ b/api/tests/unit_tests/core/plugin/test_backwards_invocation_app.py @@ -13,8 +13,8 @@ from models.enums import EndUserType from models.model import App, AppMode, AppModelConfig, EndUser from models.workflow import Workflow +from services.app.generation.runtime import AppGenerationRuntime from services.plugin.app_invocation import PluginAppBackwardsInvocation -from services.workflow.execution.ports import WorkflowRuntime from tests.unit_tests.model_factories import make_app, make_end_user, make_workflow @@ -168,7 +168,7 @@ def test_fetch_app_info_model_config_path(self, mocker: MockerFixture): ], ) def test_invoke_app_routes_by_mode( - self, mocker: MockerFixture, mode, route_method, *, workflow_runtime: WorkflowRuntime + self, mocker: MockerFixture, mode, route_method, *, workflow_runtime: AppGenerationRuntime ): app = _app(mode=mode) user = _end_user() @@ -196,7 +196,7 @@ def test_invoke_app_routes_by_mode( assert route.call_count == 1 def test_invoke_app_uses_end_user_when_user_id_missing( - self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime ): app = _app(mode=AppMode.WORKFLOW) end_user = _end_user() @@ -226,7 +226,7 @@ def test_invoke_app_uses_end_user_when_user_id_missing( assert route.call_args.args[2] is end_user def test_invoke_app_missing_query_for_chat_raises( - self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime ): mocker.patch.object(PluginAppBackwardsInvocation, "_get_app", return_value=_app(mode=AppMode.CHAT)) mocker.patch.object(PluginAppBackwardsInvocation, "_get_user", return_value=_end_user()) @@ -246,7 +246,7 @@ def test_invoke_app_missing_query_for_chat_raises( runtime=workflow_runtime, ) - def test_invoke_app_unexpected_mode_raises(self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime): + def test_invoke_app_unexpected_mode_raises(self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime): mocker.patch.object( PluginAppBackwardsInvocation, "_get_app", @@ -277,10 +277,16 @@ def test_invoke_app_unexpected_mode_raises(self, mocker: MockerFixture, *, workf ], ) def test_invoke_chat_app_agent_and_chat( - self, mocker: MockerFixture, mode, generator_path, *, workflow_runtime: WorkflowRuntime + self, mocker: MockerFixture, mode, generator_path, *, workflow_runtime: AppGenerationRuntime ): app = _app(mode=mode) - spy = mocker.patch(generator_path, return_value={"result": "ok"}) + agent_generator = None + if mode is AppMode.AGENT_CHAT: + agent_generator = mocker.patch("services.plugin.app_invocation.AgentChatAppGenerator") + spy = agent_generator.return_value.generate + spy.return_value = {"result": "ok"} + else: + spy = mocker.patch(generator_path, return_value={"result": "ok"}) result = PluginAppBackwardsInvocation.invoke_chat_app( app=app, @@ -296,9 +302,11 @@ def test_invoke_chat_app_agent_and_chat( assert result == {"result": "ok"} assert spy.call_count == 1 + if agent_generator is not None: + assert agent_generator.call_args.kwargs["tool_invoker"] is workflow_runtime.agent_tool_invoker def test_invoke_chat_app_advanced_chat_injects_pause_state_config( - self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime ): workflow = _workflow() workflow.created_by = "owner-id" @@ -336,7 +344,7 @@ def test_invoke_chat_app_advanced_chat_injects_pause_state_config( assert pause_state_config.state_owner_user_id == "owner-id" def test_invoke_chat_app_advanced_chat_without_workflow_raises( - self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime ): app = _app(mode=AppMode.ADVANCED_CHAT) mocker.patch.object(PluginAppBackwardsInvocation, "_get_workflow", return_value=None) @@ -353,7 +361,7 @@ def test_invoke_chat_app_advanced_chat_without_workflow_raises( runtime=workflow_runtime, ) - def test_invoke_chat_app_unexpected_mode_raises(self, *, workflow_runtime: WorkflowRuntime): + def test_invoke_chat_app_unexpected_mode_raises(self, *, workflow_runtime: AppGenerationRuntime): app = _app(mode=cast(AppMode, "invalid")) with pytest.raises(ValueError, match="unexpected app type"): PluginAppBackwardsInvocation.invoke_chat_app( @@ -369,7 +377,7 @@ def test_invoke_chat_app_unexpected_mode_raises(self, *, workflow_runtime: Workf ) def test_invoke_workflow_app_injects_pause_state_config( - self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime ): workflow = _workflow() workflow.created_by = "owner-id" @@ -402,7 +410,7 @@ def test_invoke_workflow_app_injects_pause_state_config( assert pause_state_config.state_owner_user_id == "owner-id" def test_invoke_app_workflow_without_workflow_raises( - self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime ): app = _app(mode=AppMode.WORKFLOW) mocker.patch.object(PluginAppBackwardsInvocation, "_get_app", return_value=app) @@ -559,7 +567,7 @@ def test_get_user_raises_when_user_not_found(self): PluginAppBackwardsInvocation._get_user("uid", app) def test_invoke_app_creates_end_user_for_unknown_external_user_id( - self, mocker: MockerFixture, *, workflow_runtime: WorkflowRuntime + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime ): app = _app(mode=AppMode.WORKFLOW) end_user = _end_user(session_id="wecom-sender-1") diff --git a/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py b/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py index fac484fba26af2..aaeae6824be501 100644 --- a/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py +++ b/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py @@ -1,14 +1,15 @@ """Unit tests for workflow-as-tool behavior with real SQLite ORM boundaries.""" import json +import threading import uuid from collections.abc import Callable, Iterator from dataclasses import dataclass -from typing import Any -from unittest.mock import Mock, patch +from typing import Any, cast +from unittest.mock import Mock, create_autospec, patch import pytest -from sqlalchemy import Engine, inspect +from sqlalchemy import Engine, Table, inspect from sqlalchemy.orm import Session, sessionmaker from core.app.apps.draft_variable_saver import DraftVariableSaverFactory @@ -29,6 +30,7 @@ from models.enums import EndUserType from models.model import App, AppMode, EndUser from models.workflow import Workflow, WorkflowType +from repositories.tools.workflow_repository import WorkflowToolRepository from services.tools.workflow.tool import WorkflowTool from services.workflow.execution.ports import WorkflowRuntime @@ -68,7 +70,7 @@ def sqlite_tool_db( ) -> Iterator[SqliteToolDb]: """Bind service-owned sessions to SQLite.""" models = (App, Workflow, EndUser, Account, Tenant, TenantAccountJoin) - TypeBase.metadata.create_all(sqlite_engine, tables=[model.__table__ for model in models]) + TypeBase.metadata.create_all(sqlite_engine, tables=[cast(Table, model.__table__) for model in models]) session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False) with session_maker() as caller_session: yield SqliteToolDb(engine=sqlite_engine, session_maker=session_maker, caller_session=caller_session) @@ -175,7 +177,9 @@ def _build_tool( ) runtime = ToolRuntime(tenant_id=tenant_id, invoke_from=InvokeFrom.EXPLORE) return WorkflowTool( - draft_variable_saver=Mock(return_value=Mock(spec=DraftVariableSaverFactory)), + draft_variable_saver=Mock( + return_value=create_autospec(DraftVariableSaverFactory, instance=True, spec_set=True) + ), workflow_app_id=workflow_app_id, workflow_as_tool_id="wf-tool-1", version=version, @@ -190,7 +194,7 @@ def _build_tool( def test_workflow_tool_should_raise_tool_invoke_error_when_result_has_error_field( monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime -): +) -> None: """Ensure that WorkflowTool will throw a `ToolInvokeError` exception when `WorkflowAppGenerator.generate` returns a result with `error` key inside the `data` element. @@ -214,13 +218,13 @@ def test_workflow_tool_should_raise_tool_invoke_error_when_result_has_error_fiel with pytest.raises(ToolInvokeError) as exc_info: # WorkflowTool always returns a generator, so we need to iterate to # actually `run` the tool. - list(tool.invoke("test_user", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) assert exc_info.value.args == ("oops",) def test_workflow_tool_does_not_use_pause_state_config( monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime -): +) -> None: """Ensure pause_state_config is passed as None.""" tool = _build_tool(workflow_runtime=workflow_runtime) @@ -236,7 +240,7 @@ def test_workflow_tool_does_not_use_pause_state_config( ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke("test_user", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) call_kwargs = generate_calls[-1] assert "pause_state_config" in call_kwargs @@ -245,7 +249,7 @@ def test_workflow_tool_does_not_use_pause_state_config( def test_workflow_tool_passes_parent_trace_context_from_runtime( monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime -): +) -> None: """Ensure nested workflow runtime metadata is forwarded as parent trace context.""" tool = _build_tool(workflow_runtime=workflow_runtime) tool.set_parent_trace_context( @@ -265,7 +269,7 @@ def test_workflow_tool_passes_parent_trace_context_from_runtime( ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke("test_user", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) call_kwargs = generate_calls[-1] assert call_kwargs["args"]["parent_trace_context"].model_dump() == { @@ -276,7 +280,7 @@ def test_workflow_tool_passes_parent_trace_context_from_runtime( def test_workflow_tool_passes_parent_trace_session_id( monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime -): +) -> None: """Ensure nested workflows inherit the parent observability session ID.""" tool = _build_tool(workflow_runtime=workflow_runtime) tool.entity.parameters = [ @@ -301,7 +305,13 @@ def test_workflow_tool_passes_parent_trace_session_id( ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke("test_user", {"trace_session_id": "user-input-session"})) + list( + tool.invoke( + session=sqlite_tool_db.caller_session, + user_id="test_user", + tool_parameters={"trace_session_id": "user-input-session"}, + ) + ) call_kwargs = generate_calls[-1] assert call_kwargs["args"]["inputs"]["trace_session_id"] == "user-input-session" @@ -310,7 +320,7 @@ def test_workflow_tool_passes_parent_trace_session_id( def test_workflow_tool_keeps_user_inputs_named_like_trace_runtime_keys( monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime -): +) -> None: """Ensure private trace context does not overwrite same-named workflow inputs.""" tool = _build_tool(workflow_runtime=workflow_runtime) tool.entity.parameters = [ @@ -346,8 +356,9 @@ def test_workflow_tool_keeps_user_inputs_named_like_trace_runtime_keys( list( tool.invoke( - "test_user", - { + session=sqlite_tool_db.caller_session, + user_id="test_user", + tool_parameters={ "outer_workflow_run_id": "user-workflow-input", "outer_node_execution_id": "user-node-input", }, @@ -365,7 +376,7 @@ def test_workflow_tool_keeps_user_inputs_named_like_trace_runtime_keys( def test_workflow_tool_can_clear_parent_trace_context( monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime -): +) -> None: """Ensure reused WorkflowTool instances do not keep stale parent trace context.""" tool = _build_tool(workflow_runtime=workflow_runtime) tool.set_parent_trace_context( @@ -386,7 +397,7 @@ def test_workflow_tool_can_clear_parent_trace_context( ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke("test_user", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) call_kwargs = generate_calls[-1] assert "parent_trace_context" not in call_kwargs["args"] @@ -394,7 +405,7 @@ def test_workflow_tool_can_clear_parent_trace_context( def test_workflow_tool_can_clear_trace_session_id( monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime -): +) -> None: """Ensure reused WorkflowTool instances do not keep stale trace session IDs.""" tool = _build_tool(workflow_runtime=workflow_runtime) tool.set_trace_session_id("session-1") @@ -412,7 +423,7 @@ def test_workflow_tool_can_clear_trace_session_id( ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke("test_user", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) call_kwargs = generate_calls[-1] assert "trace_session_id" not in call_kwargs["args"] @@ -433,7 +444,7 @@ def test_workflow_tool_omits_parent_trace_context_when_runtime_is_incomplete( sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime, -): +) -> None: """Ensure incomplete runtime metadata does not leak parent trace context into generator args.""" tool = _build_tool(workflow_runtime=workflow_runtime) tool.runtime.runtime_parameters = runtime_parameters @@ -450,7 +461,7 @@ def test_workflow_tool_omits_parent_trace_context_when_runtime_is_incomplete( ) monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke("test_user", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) call_kwargs = generate_calls[-1] assert "parent_trace_context" not in call_kwargs["args"] @@ -458,7 +469,7 @@ def test_workflow_tool_omits_parent_trace_context_when_runtime_is_incomplete( def test_workflow_tool_should_generate_variable_messages_for_outputs( monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime -): +) -> None: """Test that WorkflowTool should generate variable messages when there are outputs""" tool = _build_tool(workflow_runtime=workflow_runtime) @@ -481,32 +492,32 @@ def test_workflow_tool_should_generate_variable_messages_for_outputs( monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) # Execute tool invocation - messages = list(tool.invoke("test_user", {})) + messages = list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) # Verify variable messages - variable_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.VARIABLE] + variable_messages = [msg.message for msg in messages if isinstance(msg.message, ToolInvokeMessage.VariableMessage)] assert len(variable_messages) == 3 # Verify content of each variable message - variable_dict = {msg.message.variable_name: msg.message.variable_value for msg in variable_messages} + variable_dict = {message.variable_name: message.variable_value for message in variable_messages} assert variable_dict["result"] == "success" assert variable_dict["count"] == 42 assert variable_dict["data"] == {"key": "value"} # Verify text message - text_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.TEXT] + text_messages = [msg.message for msg in messages if isinstance(msg.message, ToolInvokeMessage.TextMessage)] assert len(text_messages) == 1 - assert json.loads(text_messages[0].message.text) == mock_outputs + assert json.loads(text_messages[0].text) == mock_outputs # Verify JSON message - json_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.JSON] + json_messages = [msg.message for msg in messages if isinstance(msg.message, ToolInvokeMessage.JsonMessage)] assert len(json_messages) == 1 - assert json_messages[0].message.json_object == mock_outputs + assert json_messages[0].json_object == mock_outputs def test_workflow_tool_should_handle_empty_outputs( monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime -): +) -> None: """Test that WorkflowTool should handle empty outputs correctly""" tool = _build_tool(workflow_runtime=workflow_runtime) @@ -526,25 +537,25 @@ def test_workflow_tool_should_handle_empty_outputs( monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) # Execute tool invocation - messages = list(tool.invoke("test_user", {})) + messages = list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) # Verify generated messages # Should contain: 0 variable messages + 1 text message + 1 JSON message = 2 messages assert len(messages) == 2 # Verify no variable messages - variable_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.VARIABLE] + variable_messages = [msg.message for msg in messages if isinstance(msg.message, ToolInvokeMessage.VariableMessage)] assert len(variable_messages) == 0 # Verify text message - text_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.TEXT] + text_messages = [msg.message for msg in messages if isinstance(msg.message, ToolInvokeMessage.TextMessage)] assert len(text_messages) == 1 - assert text_messages[0].message.text == "{}" + assert text_messages[0].text == "{}" # Verify JSON message - json_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.JSON] + json_messages = [msg.message for msg in messages if isinstance(msg.message, ToolInvokeMessage.JsonMessage)] assert len(json_messages) == 1 - assert json_messages[0].message.json_object == {} + assert json_messages[0].json_object == {} @pytest.mark.parametrize( @@ -558,19 +569,20 @@ def test_workflow_tool_should_handle_empty_outputs( ("dict_var", {"key": "value"}), ], ) -def test_create_variable_message(var_name, var_value, *, workflow_runtime: WorkflowRuntime): +def test_create_variable_message(var_name: str, var_value: object, *, workflow_runtime: WorkflowRuntime) -> None: """Create variable messages for multiple value types.""" tool = _build_tool(workflow_runtime=workflow_runtime) message = tool.create_variable_message(var_name, var_value) assert message.type == ToolInvokeMessage.MessageType.VARIABLE + assert isinstance(message.message, ToolInvokeMessage.VariableMessage) assert message.message.variable_name == var_name assert message.message.variable_value == var_value assert message.message.stream is False -def test_create_file_message_should_include_file_marker(*, workflow_runtime: WorkflowRuntime): +def test_create_file_message_should_include_file_marker(*, workflow_runtime: WorkflowRuntime) -> None: """Ensure file message includes marker and meta payload.""" tool = _build_tool(workflow_runtime=workflow_runtime) @@ -578,11 +590,14 @@ def test_create_file_message_should_include_file_marker(*, workflow_runtime: Wor message = tool.create_file_message(file_obj) # type: ignore[arg-type] assert message.type == ToolInvokeMessage.MessageType.FILE + assert isinstance(message.message, ToolInvokeMessage.FileMessage) assert message.message.file_marker == "file_marker" assert message.meta == {"file": file_obj} -def test_resolve_tenant_user_falls_back_to_end_user(sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime): +def test_resolve_tenant_user_falls_back_to_end_user( + sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Ensure worker context can resolve EndUser when Account is missing.""" _persist_tenant(sqlite_tool_db) end_user = _persist_end_user(sqlite_tool_db) @@ -606,7 +621,7 @@ def test_resolve_tenant_user_falls_back_to_end_user(sqlite_tool_db: SqliteToolDb def test_resolve_tenant_user_returns_none_when_no_tenant( sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime -): +) -> None: """Return None if tenant cannot be found in worker context.""" tool = _build_tool(tenant_id=OTHER_TENANT_ID, workflow_runtime=workflow_runtime) tool.runtime.invoke_from = InvokeFrom.SERVICE_API @@ -616,7 +631,7 @@ def test_resolve_tenant_user_returns_none_when_no_tenant( assert resolved_user is None -def test_workflow_tool_provider_type_and_fork_runtime(*, workflow_runtime: WorkflowRuntime): +def test_workflow_tool_provider_type_and_fork_runtime(*, workflow_runtime: WorkflowRuntime) -> None: """Verify provider type and forked runtime behavior.""" tool = _build_tool(workflow_runtime=workflow_runtime) assert tool.tool_provider_type() == ToolProviderType.WORKFLOW @@ -628,19 +643,19 @@ def test_workflow_tool_provider_type_and_fork_runtime(*, workflow_runtime: Workf assert forked.runtime.tenant_id == "tenant-2" -def test_derive_usage_from_top_level_usage_key(): +def test_derive_usage_from_top_level_usage_key() -> None: """Derive usage from top-level usage dict.""" usage = WorkflowTool._derive_usage_from_result({"usage": {"total_tokens": 12, "total_price": "0.2"}}) assert usage.total_tokens == 12 -def test_derive_usage_from_metadata_usage(): +def test_derive_usage_from_metadata_usage() -> None: """Derive usage from metadata usage dict.""" metadata_usage = WorkflowTool._derive_usage_from_result({"metadata": {"usage": {"total_tokens": 7}}}) assert metadata_usage.total_tokens == 7 -def test_derive_usage_from_totals(): +def test_derive_usage_from_totals() -> None: """Derive usage from top-level totals fields.""" totals_usage = WorkflowTool._derive_usage_from_result( {"total_tokens": "9", "total_price": "1.3", "currency": "USD"} @@ -649,13 +664,13 @@ def test_derive_usage_from_totals(): assert str(totals_usage.total_price) == "1.3" -def test_derive_usage_from_empty(): +def test_derive_usage_from_empty() -> None: """Default usage values when result is empty.""" empty_usage = WorkflowTool._derive_usage_from_result({}) assert empty_usage.total_tokens == 0 -def test_extract_usage_from_nested(): +def test_extract_usage_from_nested() -> None: """Extract nested usage dict from result payloads.""" nested = WorkflowTool._extract_usage_dict({"nested": [{"data": {"usage": {"total_tokens": 3}}}]}) assert nested == {"total_tokens": 3} @@ -663,7 +678,7 @@ def test_extract_usage_from_nested(): def test_invoke_raises_when_user_not_found( monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime -): +) -> None: """Raise ToolInvokeError when user resolution fails.""" tool = _build_tool(workflow_runtime=workflow_runtime) monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) @@ -671,10 +686,12 @@ def test_invoke_raises_when_user_not_found( monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: None) with pytest.raises(ToolInvokeError, match="User not found"): - list(tool.invoke("missing", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="missing", tool_parameters={})) -def test_resolve_tenant_user_returns_account(sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime): +def test_resolve_tenant_user_returns_account( + sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Resolve Account and set tenant in worker context.""" tenant = _persist_tenant(sqlite_tool_db) account = _persist_account(sqlite_tool_db) @@ -687,7 +704,9 @@ def test_resolve_tenant_user_returns_account(sqlite_tool_db: SqliteToolDb, *, wo assert inspect(resolved).detached is True -def test_get_workflow_and_get_app_db_branches(sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime): +def test_get_workflow_and_get_app_db_branches( + sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Cover workflow/app retrieval branches and error cases.""" app = _persist_app(sqlite_tool_db) specific_workflow = _persist_workflow(sqlite_tool_db, version="1") @@ -734,7 +753,7 @@ def _setup_transform_args_tool(monkeypatch: pytest.MonkeyPatch, *, workflow_runt return tool -def test_transform_args_valid_files(monkeypatch: pytest.MonkeyPatch, *, workflow_runtime: WorkflowRuntime): +def test_transform_args_valid_files(monkeypatch: pytest.MonkeyPatch, *, workflow_runtime: WorkflowRuntime) -> None: """Transform args into parameters and files payloads.""" tool = _setup_transform_args_tool(monkeypatch, workflow_runtime=workflow_runtime) build_file_from_stored_mapping, build_file_calls = _record_calls( @@ -795,7 +814,7 @@ def test_transform_args_valid_files(monkeypatch: pytest.MonkeyPatch, *, workflow assert all(call["tenant_id"] == "test_tool" for call in build_file_calls) -def test_transform_args_invalid_files(monkeypatch: pytest.MonkeyPatch, *, workflow_runtime: WorkflowRuntime): +def test_transform_args_invalid_files(monkeypatch: pytest.MonkeyPatch, *, workflow_runtime: WorkflowRuntime) -> None: """Ignore invalid file entries while keeping params.""" tool = _setup_transform_args_tool(monkeypatch, workflow_runtime=workflow_runtime) invalid_params, invalid_files = tool._transform_args({"query": "hello", "files": [{"invalid": True}]}) @@ -806,7 +825,7 @@ def test_transform_args_invalid_files(monkeypatch: pytest.MonkeyPatch, *, workfl @pytest.mark.parametrize("empty_value", [None, "", [], [None], [""]]) def test_transform_args_normalizes_optional_files_parameter( monkeypatch: pytest.MonkeyPatch, empty_value: Any, *, workflow_runtime: WorkflowRuntime -): +) -> None: """Pass optional workflow file-list inputs as an empty list when no files were provided.""" tool = _build_tool(workflow_runtime=workflow_runtime) images_param = ToolParameter.get_simple_instance( @@ -826,7 +845,7 @@ def test_transform_args_normalizes_optional_files_parameter( def test_workflow_tool_invocation_normalizes_optional_files_parameter( monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime -): +) -> None: """Ensure casted empty FILES values do not reach workflow input validation as [None].""" tool = _build_tool(workflow_runtime=workflow_runtime) images_param = ToolParameter.get_simple_instance( @@ -848,13 +867,19 @@ def test_workflow_tool_invocation_normalizes_optional_files_parameter( "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate ) - list(tool.invoke("test_user", {"images": None})) + list( + tool.invoke( + session=sqlite_tool_db.caller_session, + user_id="test_user", + tool_parameters={"images": None}, + ) + ) call_kwargs = generate_calls[-1] assert call_kwargs["args"]["inputs"]["images"] == [] -def test_extract_files(*, workflow_runtime: WorkflowRuntime): +def test_extract_files(*, workflow_runtime: WorkflowRuntime) -> None: """Extract file outputs into result and file list.""" tool = _build_tool(workflow_runtime=workflow_runtime) built_files = [ @@ -893,7 +918,7 @@ def test_extract_files(*, workflow_runtime: WorkflowRuntime): assert len(extracted_files) == 2 -def test_update_file_mapping(*, workflow_runtime: WorkflowRuntime): +def test_update_file_mapping(*, workflow_runtime: WorkflowRuntime) -> None: """Map tool/local file transfer methods into output shape.""" tool = _build_tool(workflow_runtime=workflow_runtime) tool_file = tool._update_file_mapping({"transfer_method": "tool_file", "related_id": "tool-1"}) @@ -911,7 +936,7 @@ def test_composed_tool_fork_runs_real_generator_with_injected_draft_saver( entry: str, workflow_runtime: WorkflowRuntime, *, - workflow_queries, + workflow_queries: WorkflowToolRepository, ) -> None: from flask import Flask from sqlalchemy import select @@ -999,7 +1024,11 @@ def test_composed_tool_fork_runs_real_generator_with_injected_draft_saver( assert tool._draft_variable_saver is saver monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", lambda *_args, **_kwargs: None) - monkeypatch.setattr(generator_module, "TraceQueueManager", Mock(return_value=Mock(spec=TraceQueueManager))) + monkeypatch.setattr( + generator_module, + "TraceQueueManager", + Mock(return_value=create_autospec(TraceQueueManager, instance=True, spec_set=True)), + ) monkeypatch.setattr(WorkflowAppGenerator, "_generate_worker", staticmethod(lambda **_kwargs: None)) outputs = {"result": "nested result"} @@ -1016,7 +1045,7 @@ def handle_response( generator_module.WorkflowAppGenerateResponseConverter, "convert", lambda *, response, invoke_from: response ) with Flask(__name__).app_context(): - messages = list(tool._invoke(user_id=account.id, tool_parameters={})) + messages = list(tool._invoke(session=sqlite_tool_db.caller_session, user_id=account.id, tool_parameters={})) assert messages with sqlite_tool_db.session_maker() as session: saved = session.scalar(select(WorkflowDraftVariable).where(WorkflowDraftVariable.node_id == "nested-node")) @@ -1031,12 +1060,12 @@ def handle_response( @pytest.mark.parametrize("invoke_from", [InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API]) def test_nested_workflow_preserves_admitted_end_user_and_resolves_agent_binding( - sqlite_tool_db, - workflow_runtime, - monkeypatch, - invoke_from, - config_overrides, -): + sqlite_tool_db: SqliteToolDb, + workflow_runtime: WorkflowRuntime, + monkeypatch: pytest.MonkeyPatch, + invoke_from: InvokeFrom, + config_overrides: Callable[..., None], +) -> None: """Exercise tool admission, generator, worker, runner and node construction across apps.""" from flask import Flask @@ -1044,6 +1073,7 @@ def test_nested_workflow_preserves_admitted_end_user_and_resolves_agent_binding( from enums.agent import WorkflowAgentBindingType from models.tools import WorkflowToolProvider from services.tools.workflow.provider import WorkflowToolProviderController + from services.workflow.execution.adapters.agent_v2.agent_node import DifyAgentNode from services.workflow.execution.adapters.workflow import app_generator as generator_module from services.workflow.execution.adapters.workflow_entry import WorkflowEntry from tests.unit_tests.core.workflow.nodes.agent_v2.test_binding_resolver import _agent, _binding, _snapshot @@ -1107,28 +1137,54 @@ def test_nested_workflow_preserves_admitted_end_user_and_resolves_agent_binding( ToolRuntime(tenant_id=child.tenant_id, invoke_from=invoke_from) ) monkeypatch.setattr(OpsTraceManager, "get_ops_trace_instance", lambda *_args: None) - resolved = [] + resolved: list[tuple[str, str]] = [] - def probe_engine(entry): + def probe_engine(entry: WorkflowEntry) -> Iterator[object]: engine = entry.graph_engine pool = engine._graph_runtime_state.variable_pool node = engine._graph.nodes["agent-node"] - bundle = node._binding_resolver.resolve(**ids) - resolved.append((pool.get(["sys", "user_id"]).value, bundle.snapshot.id)) + assert isinstance(node, DifyAgentNode) + bundle = node._binding_resolver.resolve( + tenant_id=child.tenant_id, + app_id=child.id, + workflow_id=workflow.id, + node_id="agent-node", + ) + user_id = pool.get(["sys", "user_id"]) + assert user_id is not None + assert isinstance(user_id.value, str) + resolved.append((user_id.value, bundle.snapshot.id)) return iter(()) monkeypatch.setattr(WorkflowEntry, "run", probe_engine) - def execute_worker(*, worker, respond): + def execute_worker( + *, worker: threading.Thread, respond: Callable[[], object] + ) -> dict[str, dict[str, dict[str, str]]]: + del respond worker.run() assert resolved == [("customer-42", snapshot.id)] return {"data": {"outputs": {"caller": resolved[0][0]}}} monkeypatch.setattr(generator_module, "generate_response", execute_worker) with Flask(__name__).app_context(): - result = list(tool._invoke(user_id=user.id, tool_parameters={}, app_id=outer.id)) + result = list( + tool._invoke( + session=sqlite_tool_db.caller_session, + user_id=user.id, + tool_parameters={}, + app_id=outer.id, + ) + ) assert any(item.type == ToolInvokeMessage.MessageType.VARIABLE for item in result) # Tool admission must still reject an external identity from another tenant. foreign_user = _persist_end_user(sqlite_tool_db, end_user_id=str(uuid.uuid4()), tenant_id=OTHER_TENANT_ID) with pytest.raises(ToolInvokeError, match="User not found"): - list(tool._invoke(user_id=foreign_user.id, tool_parameters={}, app_id=outer.id)) + list( + tool._invoke( + session=sqlite_tool_db.caller_session, + user_id=foreign_user.id, + tool_parameters={}, + app_id=outer.id, + ) + ) diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py index e5a0454e7a3471..5360e26ad66ee2 100644 --- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py @@ -2,7 +2,7 @@ from datetime import UTC, datetime from types import SimpleNamespace from typing import cast -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, create_autospec, patch from uuid import UUID, uuid4 import pytest @@ -474,7 +474,8 @@ def is_accessible_file_mapping(self, *, file_id: str, tenant_id: str, transfer_m runtime_request_builder=runtime_request_builder or WorkflowAgentRuntimeRequestBuilder( dify_tools_builder=WorkflowAgentDifyToolsBuilder( - tool_providers=tool_providers, workflow_queries=Mock(spec=WorkflowToolQueries) + tool_providers=tool_providers, + workflow_queries=create_autospec(WorkflowToolQueries, instance=True), ) ), agent_backend_client=client, @@ -630,7 +631,8 @@ def test_agent_node_passes_execution_id_to_session_store_and_runtime_request_bui store = FakeSessionStore() request_builder = WorkflowAgentRuntimeRequestBuilder( dify_tools_builder=WorkflowAgentDifyToolsBuilder( - tool_providers=tool_providers, workflow_queries=Mock(spec=WorkflowToolQueries) + tool_providers=tool_providers, + workflow_queries=create_autospec(WorkflowToolQueries, instance=True), ) ) node = _node(tool_providers=tool_providers, session_store=store, runtime_request_builder=request_builder) diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py index 70f99c94d42891..6ad03c72f02e4f 100644 --- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py +++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py @@ -1,5 +1,5 @@ from collections.abc import Iterator -from unittest.mock import MagicMock, Mock +from unittest.mock import MagicMock, Mock, create_autospec from uuid import uuid4 import pytest @@ -7,7 +7,7 @@ from flask import Flask from sqlalchemy import create_engine, event, inspect from sqlalchemy.orm import ORMExecuteState, Session, sessionmaker -from sqlalchemy.sql import Executable +from sqlalchemy.sql import ClauseElement, Executable from core.workflow.nodes.agent_v2.session_store import WorkflowAgentSessionScope, WorkflowAgentWorkspaceStore from enums.agent import WorkflowAgentBindingType @@ -61,7 +61,7 @@ def test_node_binding_resolution_uses_injected_database_through_nested_factories sessions = sessionmaker(bind=engine, expire_on_commit=False) monkeypatch.setattr( "clients.agent_backend.factory.create_agent_backend_run_client", - Mock(return_value=Mock(spec=AgentBackendRunClient)), + Mock(return_value=create_autospec(AgentBackendRunClient, instance=True, spec_set=True)), ) try: TypeBase.metadata.create_all( @@ -90,7 +90,7 @@ def test_node_binding_resolution_uses_injected_database_through_nested_factories database_client=sessions, deployment_edition=DeploymentEdition.COMMUNITY, initialization_password="", - redis=Mock(spec=RedisClientWrapper), + redis=create_autospec(RedisClientWrapper, instance=True, spec_set=True), ) resolver = WorkflowAgentBindingResolver(sessions) if explicit_worker else services.workflow_agent_bindings monkeypatch.delitem(app.extensions, "application_services", raising=False) @@ -580,8 +580,12 @@ def test_binding_resolver_uses_pinned_snapshot_for_existing_node_execution( ) assert bundle.snapshot.id == pinned_snapshot.id - assert binding.id in orm_statements[0].compile().params.values() - assert pinned_snapshot.id in orm_statements[-1].compile().params.values() + first_statement = orm_statements[0] + last_statement = orm_statements[-1] + assert isinstance(first_statement, ClauseElement) + assert isinstance(last_statement, ClauseElement) + assert binding.id in first_statement.compile().params.values() + assert pinned_snapshot.id in last_statement.compile().params.values() @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) diff --git a/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_invocation_behavior.py b/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_invocation_behavior.py index 52b94514843104..ea0f4de55df0ca 100644 --- a/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_invocation_behavior.py +++ b/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_invocation_behavior.py @@ -15,10 +15,11 @@ from graphon.runtime import GraphRuntimeState, VariablePool from services.workflow.execution.adapters.node_factory import DifyNodeFactory from services.workflow.execution.adapters.node_runtime import DifyToolFileManager, DifyToolNodeRuntime +from services.workflow.execution.ports import WorkflowRuntime from tests.workflow_test_utils import build_test_graph_init_params -def init_tool_node(config: dict[str, object], *, workflow_runtime) -> ToolNode: +def init_tool_node(config: dict[str, object], *, workflow_runtime: WorkflowRuntime) -> ToolNode: graph_config = { "edges": [ { @@ -73,7 +74,7 @@ def init_tool_node(config: dict[str, object], *, workflow_runtime) -> ToolNode: return node -def test_tool_variable_invoke(workflow_runtime) -> None: +def test_tool_variable_invoke(workflow_runtime: WorkflowRuntime) -> None: node = init_tool_node( workflow_runtime=workflow_runtime, config={ @@ -109,7 +110,7 @@ def test_tool_variable_invoke(workflow_runtime) -> None: assert item.node_run_result.outputs.get("text") is not None -def test_tool_mixed_invoke(workflow_runtime) -> None: +def test_tool_mixed_invoke(workflow_runtime: WorkflowRuntime) -> None: node = init_tool_node( workflow_runtime=workflow_runtime, config={ diff --git a/api/tests/unit_tests/events/event_handlers/test_delete_tool_parameters_cache_when_sync_draft_workflow.py b/api/tests/unit_tests/events/event_handlers/test_delete_tool_parameters_cache_when_sync_draft_workflow.py index 7b13f33761183d..026414304ebeda 100644 --- a/api/tests/unit_tests/events/event_handlers/test_delete_tool_parameters_cache_when_sync_draft_workflow.py +++ b/api/tests/unit_tests/events/event_handlers/test_delete_tool_parameters_cache_when_sync_draft_workflow.py @@ -1,4 +1,5 @@ import logging +from types import SimpleNamespace import pytest @@ -12,7 +13,7 @@ def test_missing_tool_provider_does_not_log_error_traceback( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, -): +) -> None: app = App( id="workflow-id", tenant_id="tenant-id", @@ -55,6 +56,12 @@ def test_missing_tool_provider_does_not_log_error_traceback( }, ) monkeypatch.setattr(handler_module.ToolEntity, "model_validate", lambda _data: tool_entity) + tool_services = SimpleNamespace(tool_providers=object(), workflow_queries=object()) + monkeypatch.setattr( + handler_module, + "application_services", + lambda: SimpleNamespace(tools=tool_services), + ) monkeypatch.setattr( handler_module.ToolManager, "get_tool_runtime", diff --git a/api/tests/unit_tests/extensions/otel/decorators/handlers/test_generate_handler.py b/api/tests/unit_tests/extensions/otel/decorators/handlers/test_generate_handler.py index c00fbaae36ceca..ae060329de2615 100644 --- a/api/tests/unit_tests/extensions/otel/decorators/handlers/test_generate_handler.py +++ b/api/tests/unit_tests/extensions/otel/decorators/handlers/test_generate_handler.py @@ -6,11 +6,17 @@ 2. Verify span attribute mapping correctness """ -from unittest.mock import MagicMock +from unittest.mock import MagicMock, create_autospec + +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from core.app.entities.app_invoke_entities import InvokeFrom from extensions.otel.decorators.handlers.generate_handler import AppGenerateHandler from extensions.otel.semconv import DifySpanAttributes, GenAIAttributes +from models import Account, App +from services.app.generation.runtime import AppGenerationRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.unit_tests.config_override import config_overrides_context @@ -19,8 +25,11 @@ class TestAppGenerateHandler: @config_overrides_context(ENABLE_OTEL=True) def test_compatible_with_real_function_signature( - self, tracer_provider_with_memory_exporter, mock_app_model, mock_account_user - ): + self, + tracer_provider_with_memory_exporter: TracerProvider, + mock_app_model: App, + mock_account_user: Account, + ) -> None: """ Verify handler compatibility with real AppGenerateService.generate signature. @@ -31,17 +40,18 @@ def test_compatible_with_real_function_signature( handler = AppGenerateHandler() - kwargs = { - "session": MagicMock(), - "app_model": mock_app_model, - "user": mock_account_user, - "args": {"workflow_id": "test-wf-123"}, - "invoke_from": InvokeFrom.DEBUGGER, - "streaming": True, - "root_node_id": None, - } - - arguments = handler._extract_arguments(AppGenerateService.generate, **kwargs) + arguments = handler._extract_arguments( + AppGenerateService.generate, + session=MagicMock(), + app_model=mock_app_model, + user=mock_account_user, + args={"workflow_id": "test-wf-123"}, + invoke_from=InvokeFrom.DEBUGGER, + variables=create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True), + runtime=create_autospec(AppGenerationRuntime, instance=True, spec_set=True), + streaming=True, + root_node_id=None, + ) assert arguments is not None, "Failed to extract arguments from AppGenerateService.generate" assert "app_model" in arguments, "Handler uses app_model but parameter is missing" @@ -51,8 +61,12 @@ def test_compatible_with_real_function_signature( @config_overrides_context(ENABLE_OTEL=True) def test_all_span_attributes_set_correctly( - self, tracer_provider_with_memory_exporter, memory_span_exporter, mock_app_model, mock_account_user - ): + self, + tracer_provider_with_memory_exporter: TracerProvider, + memory_span_exporter: InMemorySpanExporter, + mock_app_model: App, + mock_account_user: Account, + ) -> None: """Verify all span attributes are mapped correctly""" handler = AppGenerateHandler() tracer = tracer_provider_with_memory_exporter.get_tracer(__name__) @@ -66,7 +80,14 @@ def test_all_span_attributes_set_correctly( mock_app_model.tenant_id = test_tenant_id mock_account_user.id = test_user_id - def dummy_func(app_model, user, args, invoke_from, streaming=True): + def dummy_func( + app_model: App, + user: Account, + args: dict[str, object], + invoke_from: InvokeFrom, + streaming: bool = True, + ) -> str: + del app_model, user, args, invoke_from, streaming return "result" handler.wrapper( @@ -82,6 +103,7 @@ def dummy_func(app_model, user, args, invoke_from, streaming=True): spans = memory_span_exporter.get_finished_spans() assert len(spans) == 1 attrs = spans[0].attributes + assert attrs is not None assert attrs[DifySpanAttributes.APP_ID] == test_app_id assert attrs[DifySpanAttributes.TENANT_ID] == test_tenant_id diff --git a/api/tests/unit_tests/models/test_workflow_node_execution_accessors.py b/api/tests/unit_tests/models/test_workflow_node_execution_accessors.py index 3295d4a47a8c60..f0fb7bb7a7c7e3 100644 --- a/api/tests/unit_tests/models/test_workflow_node_execution_accessors.py +++ b/api/tests/unit_tests/models/test_workflow_node_execution_accessors.py @@ -13,12 +13,14 @@ import pytest from sqlalchemy.orm import Session +from core.tools.entities.tool_entities import ToolProviderType from fields.workflow_run_fields import WorkflowRunNodeExecutionResponse, node_execution_response_source from graphon.enums import BuiltinNodeTypes from models.account import Account from models.enums import CreatorUserRole, EndUserType from models.model import EndUser from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom +from services.tools.provider_queries import ToolProviderIcons from services.tools.tool_manager import ToolManager @@ -101,8 +103,13 @@ def test_returns_none_when_role_is_account(self, sqlite_session: Session) -> Non class TestNodeExecutionResponseSource: @pytest.mark.parametrize("node_type", [BuiltinNodeTypes.TOOL, BuiltinNodeTypes.DATASOURCE]) def test_provider_icons_survive_response_serialization( - self, sqlite_session, monkeypatch, node_type, *, tool_providers - ): + self, + sqlite_session: Session, + monkeypatch: pytest.MonkeyPatch, + node_type: str, + *, + tool_providers: ToolProviderIcons, + ) -> None: execution = _execution(CreatorUserRole.ACCOUNT, created_by="account") execution.id = "execution" execution.node_type = node_type @@ -113,7 +120,9 @@ def test_provider_icons_survive_response_serialization( } ) - def tool_icon(*, tenant_id, provider_type, provider_id, tool_providers): + def tool_icon( + *, tenant_id: str, provider_type: ToolProviderType, provider_id: str, tool_providers: ToolProviderIcons + ) -> str: assert tool_providers is not None assert (tenant_id, provider_type, provider_id) == (execution.tenant_id, "builtin", "provider") return "tool-icon" @@ -125,7 +134,7 @@ def tool_icon(*, tenant_id, provider_type, provider_id, tool_providers): assert response.extras == {"icon": "tool-icon" if node_type == BuiltinNodeTypes.TOOL else "source-icon"} def test_accessors_resolve_via_wrapped_session_and_other_attributes_proxy( - self, sqlite_session: Session, *, tool_providers + self, sqlite_session: Session, *, tool_providers: ToolProviderIcons ) -> None: account = Account(name="Test Account", email="test@example.com") sqlite_session.add(account) diff --git a/api/tests/unit_tests/repositories/test_agent_execution_repository.py b/api/tests/unit_tests/repositories/test_agent_execution_repository.py index 33646b8d7c1993..d6d6bde20b17bf 100644 --- a/api/tests/unit_tests/repositories/test_agent_execution_repository.py +++ b/api/tests/unit_tests/repositories/test_agent_execution_repository.py @@ -123,6 +123,7 @@ def test_execution_finish_keeps_retry_candidates_including_cancelled_reservation if cancel: reservation = debug_lease(session, "run") assert reservation is not None + assert reservation.expires_at is not None assert reservation.expires_at <= naive_utc_now() assert finish(cancel=cancel) == WorkflowExecutionStatus.STOPPED assert repository.finished_agent_ids( diff --git a/api/tests/unit_tests/repositories/workflow/test_definition_repository.py b/api/tests/unit_tests/repositories/workflow/test_definition_repository.py index ac290f4f364f8d..8fa864801fde1e 100644 --- a/api/tests/unit_tests/repositories/workflow/test_definition_repository.py +++ b/api/tests/unit_tests/repositories/workflow/test_definition_repository.py @@ -4,7 +4,7 @@ from collections.abc import Callable from dataclasses import replace from typing import cast -from unittest.mock import Mock, patch +from unittest.mock import Mock, create_autospec, patch from uuid import uuid4 import pytest @@ -50,6 +50,7 @@ from services.data_migration.import_service import MigrationImportService from services.errors.app import TriggerNodeLimitExceededError, WorkflowHashNotEqualError from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError +from services.tools.workflow_tools_manage_service import WorkflowToolManageService from services.workflow import definition_gateway as gateway_module from services.workflow.console_service import ( ConsoleWorkflowService, @@ -137,7 +138,7 @@ def check(credential_id: str, *_args: object, **_kwargs: object) -> None: definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=workflows, credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) event.listen(seeded, "after_begin", opened) try: @@ -291,7 +292,9 @@ def wrap(**_kwargs: object) -> str: database_client=seeded, variables=build_workflow_variable_service(database_client=seeded) ) with seeded.begin() as session: - session.get(App, "app-1").mode = AppMode.ADVANCED_CHAT + app = session.get(App, "app-1") + assert app is not None + app.mode = AppMode.ADVANCED_CHAT workflows = WorkflowService(seeded) monkeypatch.setattr(workflows, "validate_graph_structure", lambda **_kwargs: None) definitions = WorkflowDefinitionRepository(session_factory=seeded) @@ -367,7 +370,9 @@ def encrypt(*, tenant_id: str, token: str) -> str: database_client=seeded, variables=build_workflow_variable_service(database_client=seeded) ) with seeded.begin() as session: - session.get(App, "app-1").mode = AppMode.ADVANCED_CHAT + app = session.get(App, "app-1") + assert app is not None + app.mode = AppMode.ADVANCED_CHAT workflows = WorkflowService(seeded) monkeypatch.setattr(workflows, "validate_graph_structure", lambda *_args, **_kwargs: None) monkeypatch.setattr(workflows, "validate_features_structure", lambda *_args: None) @@ -644,20 +649,20 @@ def test_queries_and_updates_keep_complete_owner_chain( def application(lifecycle: WorkflowDefinitionGateway) -> ConsoleWorkflowService[WorkflowAgentBindingStore]: return ConsoleWorkflowService( - conversion=Mock(spec=WorkflowConversion), + conversion=create_autospec(WorkflowConversion, instance=True), agent_services=WorkflowAgentPublishService, definitions=cast(WorkflowDefinitionRepository, lifecycle._definitions), drafts=WorkflowDraftService( agent_services=WorkflowAgentPublishService, definitions=cast(WorkflowDraftRepository, lifecycle._drafts), lifecycle=lifecycle, - retirement=Mock(spec=DraftAgentRetirement), + retirement=create_autospec(DraftAgentRetirement, instance=True), ), lifecycle=lifecycle, - runtime=Mock(spec=WorkflowRuntime), - apps=Mock(spec=WorkflowAppLookup), - presence=Mock(spec=WorkflowPresence), - access=Mock(spec=WorkflowAccess), + runtime=create_autospec(WorkflowRuntime, instance=True), + apps=create_autospec(WorkflowAppLookup, instance=True), + presence=create_autospec(WorkflowPresence, instance=True), + access=create_autospec(WorkflowAccess, instance=True), ) @@ -677,7 +682,7 @@ def test_publication_and_app_pointer_commit_atomically(seeded: sessionmaker[Sess definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) with seeded() as session: draft = session.get(Workflow, "draft") @@ -743,7 +748,7 @@ def test_sync_prepares_patch_without_replacing_server_environment(seeded: sessio definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) ).sync(CONTEXT, "app-1", command) with seeded() as session: @@ -758,7 +763,7 @@ def test_sync_prepares_patch_without_replacing_server_environment(seeded: sessio definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) ).sync(CONTEXT, "app-1", replace(command, environment_upserts=None)) with seeded() as session: @@ -791,24 +796,24 @@ def test_use_case_notifies_only_after_real_transaction_commits( definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) - runtime = Mock(spec=WorkflowRuntime) + runtime = create_autospec(WorkflowRuntime, instance=True) service = ConsoleWorkflowService( - conversion=Mock(spec=WorkflowConversion), + conversion=create_autospec(WorkflowConversion, instance=True), agent_services=WorkflowAgentPublishService, definitions=repository, drafts=WorkflowDraftService( agent_services=WorkflowAgentPublishService, definitions=WorkflowDraftRepository(seeded), lifecycle=lifecycle, - retirement=Mock(spec=DraftAgentRetirement), + retirement=create_autospec(DraftAgentRetirement, instance=True), ), lifecycle=lifecycle, runtime=runtime, - apps=Mock(spec=WorkflowAppLookup), - presence=Mock(spec=WorkflowPresence), - access=Mock(spec=WorkflowAccess), + apps=create_autospec(WorkflowAppLookup, instance=True), + presence=create_autospec(WorkflowPresence, instance=True), + access=create_autospec(WorkflowAccess, instance=True), ) if operation in {"sync", "restore"}: with seeded.begin() as session: @@ -935,7 +940,7 @@ def test_publish_rejects_revision_modified_after_validation(seeded: sessionmaker definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) with pytest.raises(WorkflowHashNotEqualError): publish(lifecycle, validated) @@ -970,7 +975,7 @@ def test_trigger_limit_rejects_publish_before_version_or_pointer_changes( definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) with pytest.raises(ValueError, match="maximum.*limit"): application(lifecycle).publish(CONTEXT, "app-1", marked_name="Rejected", marked_comment="") @@ -991,7 +996,7 @@ def test_publication_versions_increase_without_reusing_deleted_version(seeded: s definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) service = application(lifecycle) for expected in [2, 3, 4]: @@ -1085,7 +1090,7 @@ def test_publication_retains_blocking_domain_validations( definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) with pytest.raises(expected_error): application(lifecycle).publish(CONTEXT, "app-1", marked_name="Rejected", marked_comment="") @@ -1117,7 +1122,7 @@ def test_restore_preserves_historical_features_without_normalizing_persisted_sou definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) application(lifecycle).restore(CONTEXT, "app-1", "published") with seeded() as session: @@ -1145,7 +1150,7 @@ def capture(execution: ORMExecuteState) -> None: definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) event.listen(seeded, "do_orm_execute", capture) try: @@ -1158,7 +1163,7 @@ def capture(execution: ORMExecuteState) -> None: @pytest.mark.parametrize("over_limit", [False, True]) def test_migration_uses_the_same_atomic_publication_boundary( - seeded: sessionmaker[Session], over_limit: bool, *, workflow_tools + seeded: sessionmaker[Session], over_limit: bool, *, workflow_tools: WorkflowToolManageService ) -> None: app_id = str(uuid4()) with seeded.begin() as session: @@ -1181,7 +1186,7 @@ def test_migration_uses_the_same_atomic_publication_boundary( definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) migration = MigrationImportService(workflows=application(lifecycle), app_dsl=Mock(), workflow_tools=workflow_tools) with seeded() as session: @@ -1284,7 +1289,7 @@ def test_redis_outage_cannot_leave_a_successful_publication_without_runtime_conf definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) def unavailable() -> None: @@ -1307,7 +1312,7 @@ def test_trigger_storage_failure_rolls_back_version_pointer_and_every_projection definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) service = application(lifecycle) service.publish(CONTEXT, "app-1", marked_name="A", marked_comment="") @@ -1343,7 +1348,7 @@ def test_republication_replaces_all_trigger_configuration( definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) for name in ["A", "B"]: configure_runtime_draft(seeded, name) @@ -1372,7 +1377,7 @@ def test_draft_webhook_receiver_uses_injected_database( definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) lifecycle.draft_synced(CONTEXT, WorkflowOwner("app-1"), snapshot) with seeded() as session: @@ -1455,10 +1460,12 @@ def test_variable_update_does_not_commit_an_ambient_session(seeded: sessionmaker repository = WorkflowDraftRepository(seeded) with seeded() as caller: caller.add(make_account(account_id="uncommitted", email="uncommitted@example.com")) + snapshot = repository.snapshot(CONTEXT, WorkflowOwner("app-1")) + assert snapshot is not None repository.update_draft_variables( CONTEXT, "app-1", - expected=repository.snapshot(CONTEXT, WorkflowOwner("app-1")), + expected=snapshot, environment_variables="{}", conversation_variables=None, ) @@ -1503,6 +1510,7 @@ def fail_commit(_session: Session) -> None: def test_variable_update_rejects_another_tenant(seeded: sessionmaker[Session]) -> None: repository = WorkflowDraftRepository(seeded) snapshot = repository.snapshot(CONTEXT, WorkflowOwner("app-1")) + assert snapshot is not None with pytest.raises(ConsoleAppNotFoundError): repository.update_draft_variables( CONTEXT._replace(active_workspace_id="other-tenant"), @@ -1523,7 +1531,7 @@ def test_republish_without_schedule_removes_the_runtime_plan(seeded: sessionmake definitions=WorkflowDefinitionRepository(session_factory=seeded), workflows=WorkflowService(session_maker=seeded), credentials=CredentialQueryRepository(session_factory=seeded), - skills=Mock(spec=AgentSkillReader), + skills=create_autospec(AgentSkillReader, instance=True), ) service = application(lifecycle) service.publish(CONTEXT, "app-1", marked_name="Scheduled", marked_comment="") @@ -1785,7 +1793,9 @@ def load(key: str) -> bytes: database_client=seeded, variables=build_workflow_variable_service(database_client=seeded) ) with seeded.begin() as session: - session.get(App, "app-1").mode = AppMode.ADVANCED_CHAT + app = session.get(App, "app-1") + assert app is not None + app.mode = AppMode.ADVANCED_CHAT workflows = WorkflowService(seeded) monkeypatch.setattr(workflows, "validate_graph_structure", lambda **_kwargs: None) definitions = WorkflowDefinitionRepository(session_factory=seeded) diff --git a/api/tests/unit_tests/services/agent/chat/test_tool_invocation.py b/api/tests/unit_tests/services/agent/chat/test_tool_invocation.py index 6bae46af85066e..431ef479d1fa7c 100644 --- a/api/tests/unit_tests/services/agent/chat/test_tool_invocation.py +++ b/api/tests/unit_tests/services/agent/chat/test_tool_invocation.py @@ -21,11 +21,11 @@ ToolInvokeMeta, ToolProviderType, ) +from extensions.application_services.workflow import build_workflow_execution_dependencies from graphon.model_runtime.entities import PromptMessage from models.model import Message from services.agent.chat.cot_runner import CotAgentRunner from services.agent.chat.ports import AgentToolInvoker -from services.app.generation.adapters.agent_chat import _SessionBoundAgentToolInvoker from services.app.generation.ports import AgentMessageRecords, MessageFileWriter from tests.unit_tests.model_factories import make_message @@ -98,7 +98,7 @@ def test_session_bound_invoker_consumes_lazy_messages_before_closing_session( fail_after_first_message: bool, ) -> None: tool_sessions: sessionmaker[Session] = sessionmaker(class_=_TrackingSession) - invoker = _SessionBoundAgentToolInvoker(tool_sessions) + invoker = build_workflow_execution_dependencies(tool_sessions).agent_tool_invoker tool = _LazyTool(fail_after_first_message=fail_after_first_message) text, files, meta = invoker( diff --git a/api/tests/unit_tests/services/agent/test_log_service.py b/api/tests/unit_tests/services/agent/test_log_service.py index 25713b05f9130a..1460b6e2632925 100644 --- a/api/tests/unit_tests/services/agent/test_log_service.py +++ b/api/tests/unit_tests/services/agent/test_log_service.py @@ -87,9 +87,7 @@ def _provide_app_context() -> None: @pytest.fixture -def store( - monkeypatch: pytest.MonkeyPatch, workflow_variables: WorkflowExecutionVariables -) -> Iterator[AgentLogStore]: +def store(monkeypatch: pytest.MonkeyPatch, workflow_variables: WorkflowExecutionVariables) -> Iterator[AgentLogStore]: engine = create_engine("sqlite://", poolclass=QueuePool) tables: list[Table] = [] for model in ( diff --git a/api/tests/unit_tests/services/agent/test_retirement_service.py b/api/tests/unit_tests/services/agent/test_retirement_service.py index 0eb38ffba9647f..9aa11238fcba77 100644 --- a/api/tests/unit_tests/services/agent/test_retirement_service.py +++ b/api/tests/unit_tests/services/agent/test_retirement_service.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock +from unittest.mock import MagicMock, create_autospec import pytest from sqlalchemy.orm import Session, sessionmaker @@ -26,7 +26,7 @@ def test_retire_unowned_failure_propagates() -> None: - repository = MagicMock(spec=WorkflowAgentRetirementRepository) + repository = create_autospec(WorkflowAgentRetirementRepository, instance=True, spec_set=True) error = RuntimeError("retirement failed") repository.retire.side_effect = error with pytest.raises(RuntimeError) as exc_info: @@ -231,7 +231,7 @@ def test_retire_unowned_archives_orphan_and_retires_resources( def test_hidden_app_enqueue_failure_prevents_agent_purge_enqueue(monkeypatch: pytest.MonkeyPatch) -> None: - repository = MagicMock(spec=WorkflowAgentRetirementRepository) + repository = create_autospec(WorkflowAgentRetirementRepository, instance=True, spec_set=True) repository.retire.return_value = AgentRetirement( ["agent-1", "agent-2"], ["hidden-app-1", "hidden-app-2"], [], [], [] ) diff --git a/api/tests/unit_tests/services/agent/test_tool_invocation_service.py b/api/tests/unit_tests/services/agent/test_tool_invocation_service.py index 0c7691db529039..fa5ea6d0ae4e82 100644 --- a/api/tests/unit_tests/services/agent/test_tool_invocation_service.py +++ b/api/tests/unit_tests/services/agent/test_tool_invocation_service.py @@ -143,13 +143,13 @@ def test_app_lookup_releases_connection_before_tool_setup_and_stream_consumption ) calls = [] - def resolve_runtime(**_kwargs): + def resolve_runtime(**_kwargs: object) -> Tool: assert not active calls.append("setup") return tool class PluginTransport: - def invoke(self, **kwargs): + def invoke(self, **kwargs: object) -> Generator[ToolInvokeMessage, None, None]: assert not active assert kwargs["tenant_id"] == TENANT_ID assert kwargs["app_id"] == APP_ID diff --git a/api/tests/unit_tests/services/app/test_agent_app_access_service.py b/api/tests/unit_tests/services/app/test_agent_app_access_service.py index fe70e80fe05eba..5609a0b6151058 100644 --- a/api/tests/unit_tests/services/app/test_agent_app_access_service.py +++ b/api/tests/unit_tests/services/app/test_agent_app_access_service.py @@ -1,6 +1,6 @@ """Published Agent App references use the canonical roster query.""" -from unittest.mock import Mock +from unittest.mock import create_autospec import pytest from sqlalchemy.orm import Session, sessionmaker @@ -59,7 +59,8 @@ def test_access_only_lists_current_published_workflow_references( ) sqlite_session.commit() service = build_agent_app_services( - database_client=sqlite_session_factory, variables=Mock(spec=WorkflowExecutionVariables) + database_client=sqlite_session_factory, + variables=create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True), ).access context = RequestContext("request", None, "account", "tenant-1") [reference] = service.list_referencing_workflows(context, "agent") @@ -96,7 +97,8 @@ def test_access_requires_public_agent_app( sqlite_session.add_all([make_app(app_id="agent-app", mode=AppMode.AGENT), agent]) sqlite_session.commit() service = build_agent_app_services( - database_client=sqlite_session_factory, variables=Mock(spec=WorkflowExecutionVariables) + database_client=sqlite_session_factory, + variables=create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True), ).access with pytest.raises(AgentAppNotFoundError): service.list_referencing_workflows(RequestContext("request", None, "account", "tenant-1"), "agent") diff --git a/api/tests/unit_tests/services/app/test_agent_app_feature_service.py b/api/tests/unit_tests/services/app/test_agent_app_feature_service.py index 5ee43a0e8ec017..3de7ae0e42c0da 100644 --- a/api/tests/unit_tests/services/app/test_agent_app_feature_service.py +++ b/api/tests/unit_tests/services/app/test_agent_app_feature_service.py @@ -7,7 +7,7 @@ touching model / prompt / agent_mode. """ -from unittest.mock import Mock +from unittest.mock import create_autospec import pytest from sqlalchemy import select @@ -113,7 +113,8 @@ def test_feature_save_is_atomic( agent_app: App, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session] ) -> None: service = build_agent_app_services( - database_client=sqlite_session_factory, variables=Mock(spec=WorkflowExecutionVariables) + database_client=sqlite_session_factory, + variables=create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True), ).features context = RequestContext("request", None, ACCOUNT_ID, TENANT_ID) calls: list[str] = [] @@ -149,7 +150,8 @@ def test_feature_validation_failure_does_not_write( agent_app: App, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session] ) -> None: service = build_agent_app_services( - database_client=sqlite_session_factory, variables=Mock(spec=WorkflowExecutionVariables) + database_client=sqlite_session_factory, + variables=create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True), ).features with pytest.raises(ValueError, match="opening_statement"): service.update_features( @@ -178,7 +180,8 @@ def test_features_require_owned_active_agent_app( agent.status = AgentStatus.ARCHIVED sqlite_session.commit() service = build_agent_app_services( - database_client=sqlite_session_factory, variables=Mock(spec=WorkflowExecutionVariables) + database_client=sqlite_session_factory, + variables=create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True), ).features with pytest.raises(AgentAppNotFoundError): service.update_features(RequestContext("request", None, ACCOUNT_ID, TENANT_ID), "agent", {}) diff --git a/api/tests/unit_tests/services/app/test_agent_chat_attachments.py b/api/tests/unit_tests/services/app/test_agent_chat_attachments.py index 42d7133f92a648..91eab0e83cb483 100644 --- a/api/tests/unit_tests/services/app/test_agent_chat_attachments.py +++ b/api/tests/unit_tests/services/app/test_agent_chat_attachments.py @@ -1,33 +1,46 @@ """Agent Chat reads image metadata through its injected database before accessing storage.""" import contextvars +from collections.abc import Iterator from dataclasses import dataclass +from typing import cast +from unittest.mock import create_autospec from uuid import uuid4 import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker +from flask import Flask +from sqlalchemy import Engine, Table, create_engine +from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import QueuePool +from core.agent.entities import AgentEntity from core.app.app_config.entities import DatasetRetrieveConfigEntity +from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.entities.app_invoke_entities import ( AgentChatAppGenerateEntity, InvokeFrom, ModelConfigWithCredentialsEntity, ) from core.db import session_factory +from core.model_manager import ModelInstance +from core.tools.__base.tool import Tool from core.tools.__base.tool_runtime import ToolRuntime from core.tools.entities.tool_entities import ToolInvokeMeta from extensions.ext_database import db -from graphon.file import file_manager +from graphon.file import File, file_manager +from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent from graphon.model_runtime.entities.model_entities import AIModelEntity, ModelFeature +from models.annotation_reply import AnnotationReplies from models.base import TypeBase from models.model import App, Conversation, Message, MessageAgentThought, MessageFile, UploadFile from repositories.app.generation_repository import AppGenerationRepository from services.agent.chat.function_call_runner import FunctionCallAgentRunner +from services.agent.chat.ports import AgentDatasetTools, AgentToolInvoker from services.app.generation.adapters import agent_chat as worker_module from services.app.generation.adapters.agent_chat import AgentChatAppGenerator -from services.tools.dataset.tool import DatasetRetrieverTool +from services.app.generation.ports import AgentMessageRecords +from services.knowledge.retrieval.adapters.resource_events import DatasetIndexToolCallbackHandler +from services.tools.dataset.tool import DatasetRetrieverTool, DatasetToolRetrieval from services.tools.tool_engine import ToolEngine from tests.unit_tests.core.agent.test_base_agent_runner import ( _app_config, @@ -41,7 +54,7 @@ @dataclass class LLM: - def get_model_schema(self, _name, _credentials): + def get_model_schema(self, _name: str, _credentials: dict[str, object] | None) -> AIModelEntity: return AIModelEntity.model_construct(features=[ModelFeature.VISION]) @@ -49,29 +62,27 @@ def get_model_schema(self, _name, _credentials): class Model: model_type_instance: LLM model_name: str = "model" - credentials: dict | None = None - - -@dataclass -class Queue: - def publish_error(self, error, _source): - raise AssertionError("worker failed") from error + credentials: dict[str, object] | None = None @pytest.fixture(params=[False, True], ids=["retain-after-commit", "expire-after-commit"]) -def agent_database(request, monkeypatch): +def agent_database( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> Iterator[tuple[sessionmaker[Session], AppGenerationRepository, Engine]]: engine = create_engine("sqlite://", poolclass=QueuePool) TypeBase.metadata.create_all( engine, tables=[ - model.__table__ for model in (App, Conversation, Message, MessageAgentThought, MessageFile, UploadFile) + cast(Table, model.__table__) + for model in (App, Conversation, Message, MessageAgentThought, MessageFile, UploadFile) ], ) + assert isinstance(request.param, bool) sessions = sessionmaker(engine, expire_on_commit=request.param) with sessions.begin() as session: session.add_all([make_app(app_id="app1", tenant_id="tenant"), _conversation(), _message()]) - def forbidden(*_args, **_kwargs): + def forbidden(*_args: object, **_kwargs: object) -> None: pytest.fail("Agent Chat accessed a global database") monkeypatch.setattr(session_factory, "create_session", forbidden) @@ -84,7 +95,10 @@ def forbidden(*_args, **_kwargs): engine.dispose() -def test_dataset_images_are_owned_ordered_and_materialized_before_storage(agent_database, monkeypatch): +def test_dataset_images_are_owned_ordered_and_materialized_before_storage( + agent_database: tuple[sessionmaker[Session], AppGenerationRepository, Engine], + monkeypatch: pytest.MonkeyPatch, +) -> None: sessions, records, engine = agent_database ids = [str(uuid4()) for _ in range(5)] with sessions.begin() as session: @@ -116,23 +130,25 @@ def test_dataset_images_are_owned_ordered_and_materialized_before_storage(agent_ runner = FunctionCallAgentRunner( records=records, dataset_tools=lambda **_kwargs: [], + tool_invoker=create_autospec(AgentToolInvoker, instance=True, spec_set=True), tenant_id="tenant", application_generate_entity=_app_generate(app_config=config), conversation=_conversation(), app_config=config, model_config=ModelConfigWithCredentialsEntity.model_construct(), - config=None, - queue_manager=Queue(), + config=AgentEntity(provider="provider", model="model", strategy=AgentEntity.Strategy.FUNCTION_CALLING), + queue_manager=create_autospec(AppQueueManager, instance=True, spec_set=True), message=_message(), user_id="user", - model_instance=Model(LLM()), + model_instance=cast(ModelInstance, Model(LLM())), ) reads = [] class FileRuntime: multimodal_send_format = "base64" - def load_file_bytes(self, *, file): + def load_file_bytes(self, *, file: File) -> bytes: + assert isinstance(engine.pool, QueuePool) assert engine.pool.checkedout() == 0 reads.append(file.filename) return b"image bytes" @@ -140,62 +156,108 @@ def load_file_bytes(self, *, file): monkeypatch.setattr(file_manager, "get_workflow_file_runtime", FileRuntime) class Retrieval: - def retrieve_dataset(self, *, query, **_kwargs): + def retrieve_dataset( + self, + *, + tenant_id: str, + app_id: str, + user_id: str, + dataset_id: str, + query: str, + config: DatasetRetrieveConfigEntity, + top_k: int, + inputs: dict[str, object], + invoke_from: InvokeFrom, + return_resource: bool, + hit_callback: DatasetIndexToolCallbackHandler, + ) -> str: + del tenant_id, app_id, user_id, dataset_id, config, top_k + del inputs, invoke_from, return_resource, hit_callback + assert isinstance(engine.pool, QueuePool) assert engine.pool.checkedout() == 0 return query tool = DatasetRetrieverTool( entity=_tool_entity("dataset"), runtime=ToolRuntime(tenant_id="tenant"), - retrieval=Retrieval(), + retrieval=cast(DatasetToolRetrieval, Retrieval()), dataset_id="dataset", config=DatasetRetrieveConfigEntity(retrieve_strategy="single"), top_k=2, inputs={}, invoke_from=InvokeFrom.DEBUGGER, return_resource=False, - hit_callback=None, + hit_callback=create_autospec(DatasetIndexToolCallbackHandler, instance=True, spec_set=True), app_id="app1", ) response = "\n".join(f"![image](/files/{id}/file-preview)" for id in [ids[1], ids[0], *ids]) - text, files, meta = ToolEngine.agent_invoke( - records=records, - tool=tool, - tool_parameters={"query": response}, - user_id="user", - tenant_id="tenant", - message=_message(), - invoke_from=InvokeFrom.DEBUGGER, - agent_tool_callback=runner.agent_callback, - ) + with sessions() as tool_session: + text, files, meta = ToolEngine.agent_invoke( + session=tool_session, + records=records, + tool=tool, + tool_parameters={"query": response}, + user_id="user", + tenant_id="tenant", + message=_message(), + invoke_from=InvokeFrom.DEBUGGER, + agent_tool_callback=runner.agent_callback, + ) assert files == [] assert isinstance(meta, ToolInvokeMeta) assert meta.error is None contents = runner._build_dataset_tool_image_contents(text, tool) assert reads == ["second.png", "first.png"] - assert all(content.base64_data == "aW1hZ2UgYnl0ZXM=" for content in contents) + assert all( + isinstance(content, ImagePromptMessageContent) and content.base64_data == "aW1hZ2UgYnl0ZXM=" + for content in contents + ) + assert isinstance(engine.pool, QueuePool) assert engine.pool.checkedout() == 0 def test_worker_passes_records_and_dataset_factory_without_opening_session( - agent_database, monkeypatch, app, annotation_replies -): + agent_database: tuple[sessionmaker[Session], AppGenerationRepository, Engine], + monkeypatch: pytest.MonkeyPatch, + app: Flask, + annotation_replies: AnnotationReplies, +) -> None: _, records, engine = agent_database - calls = [] + calls: list[str] = [] + expected_records = records + expected_tool_invoker = create_autospec(AgentToolInvoker, instance=True, spec_set=True) - def datasets(**_kwargs): + def datasets(**_kwargs: object) -> list[Tool]: return [] - class Runner(worker_module.AgentChatAppRunner): - def run(self, *, application_generate_entity, **kwargs): - assert self._records is records - assert self._dataset_tools is datasets + class Runner: + def __init__( + self, *, records: AgentMessageRecords, dataset_tools: AgentDatasetTools, tool_invoker: AgentToolInvoker + ) -> None: + assert records is expected_records + assert dataset_tools is datasets + assert tool_invoker is expected_tool_invoker + + def run(self, application_generate_entity: AgentChatAppGenerateEntity, **kwargs: object) -> None: assert "session" not in kwargs + assert isinstance(engine.pool, QueuePool) assert engine.pool.checkedout() == 0 calls.append(application_generate_entity.task_id) monkeypatch.setattr(worker_module, "AgentChatAppRunner", Runner) - generator = AgentChatAppGenerator(dataset_tools=datasets, records=records, annotations=annotation_replies) + generator = AgentChatAppGenerator( + dataset_tools=datasets, + tool_invoker=expected_tool_invoker, + records=records, + annotations=annotation_replies, + ) entity = AgentChatAppGenerateEntity.model_construct(task_id="task", app_config=_app_config()) - generator._generate_worker(app, contextvars.copy_context(), entity, Queue(), "conv1", "msg_current") + generator._generate_worker( + app, + contextvars.copy_context(), + entity, + create_autospec(AppQueueManager, instance=True, spec_set=True), + "conv1", + "msg_current", + ) assert calls == ["task"] diff --git a/api/tests/unit_tests/services/app/test_console_service.py b/api/tests/unit_tests/services/app/test_console_service.py index b1d2224ea35aa5..1821245158e8f2 100644 --- a/api/tests/unit_tests/services/app/test_console_service.py +++ b/api/tests/unit_tests/services/app/test_console_service.py @@ -26,6 +26,8 @@ from models.provider_ids import GenericProviderID from models.tools import ApiToolProvider from repositories.app.console_repository import ConsoleAppRepository +from repositories.tools.provider_repository import ToolProviderRepository +from repositories.tools.workflow_repository import WorkflowToolRepository from services.agent.errors import AgentAccessNotReadyError from services.agent.roster_package_entities import RosterAgentPackageExport from services.agent.roster_service import AgentRosterService @@ -474,8 +476,8 @@ def test_lifecycle_agent_creation_seeds_workspace_default_model( provider_name: str | None, expected_model: tuple[str, str, str] | None, *, - tool_providers, - workflow_queries, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) with factory.begin() as session: @@ -537,7 +539,12 @@ def test_lifecycle_agent_creation_seeds_workspace_default_model( @pytest.mark.parametrize("failure", [None, "persist", "external"]) @pytest.mark.parametrize("mode", [AppMode.WORKFLOW, AppMode.AGENT]) def test_lifecycle_runs_after_atomic_creation_and_session_close( - sqlite_engine: Engine, failure: str | None, mode: AppMode, *, tool_providers, workflow_queries + sqlite_engine: Engine, + failure: str | None, + mode: AppMode, + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) with factory.begin() as session: @@ -661,7 +668,10 @@ def deleted(self, context: RequestContext, deleted: AppDeletion) -> None: def test_app_deletion_failure_rolls_back_backing_agent_changes( - sqlite_engine: Engine, *, tool_providers, workflow_queries + sqlite_engine: Engine, + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) with factory.begin() as session: @@ -712,8 +722,8 @@ def test_detail_tool_enrichment_releases_database_before_plugin_io( operation: str, remote_fails: bool, *, - tool_providers, - workflow_queries, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) missing_api = str(uuid4()) @@ -880,7 +890,12 @@ def read_result() -> AppRecord: @pytest.mark.parametrize("operation", ["rename", "icon", "site"]) @pytest.mark.parametrize("commit_fails", [False, True]) def test_app_mutations_publish_only_after_commit( - sqlite_engine: Engine, operation: str, commit_fails: bool, *, tool_providers, workflow_queries + sqlite_engine: Engine, + operation: str, + commit_fails: bool, + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) context = RequestContext("request", None, str(uuid4()), str(uuid4())) @@ -970,7 +985,11 @@ def update() -> AppRecord: @pytest.mark.parametrize("surface", ["site", "api"]) def test_unpublished_agent_cannot_enable_access_through_console_service( - sqlite_engine: Engine, surface: str, *, tool_providers, workflow_queries + sqlite_engine: Engine, + surface: str, + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False) context = RequestContext("request", None, str(uuid4()), str(uuid4())) diff --git a/api/tests/unit_tests/services/app/test_response_gateway.py b/api/tests/unit_tests/services/app/test_response_gateway.py index b7ebc0638df2c9..52a72ad4d8321b 100644 --- a/api/tests/unit_tests/services/app/test_response_gateway.py +++ b/api/tests/unit_tests/services/app/test_response_gateway.py @@ -2,11 +2,15 @@ from machinery.context import RequestContext from models.model import AppMode +from repositories.tools.provider_repository import ToolProviderRepository +from repositories.tools.workflow_repository import WorkflowToolRepository from services.app.response_gateway import AppResponseGateway from services.entities.app_entities import AppRecord -def test_mask_record_omits_unverified_tool_parameters_without_mutating_record(tool_providers, workflow_queries) -> None: +def test_mask_record_omits_unverified_tool_parameters_without_mutating_record( + tool_providers: ToolProviderRepository, workflow_queries: WorkflowToolRepository +) -> None: agent_mode = { "enabled": True, "tools": [ diff --git a/api/tests/unit_tests/services/data_migration/test_import_service.py b/api/tests/unit_tests/services/data_migration/test_import_service.py index 550ebd780f9619..39494de29e9160 100644 --- a/api/tests/unit_tests/services/data_migration/test_import_service.py +++ b/api/tests/unit_tests/services/data_migration/test_import_service.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from typing import cast -from unittest.mock import Mock +from unittest.mock import Mock, create_autospec import pytest import yaml @@ -42,6 +42,10 @@ class Database: session: Session +def _workflow_publisher() -> WorkflowPublisher: + return create_autospec(WorkflowPublisher, instance=True, spec_set=True) + + @pytest.fixture def database(sqlite_session: Session) -> Database: database = Database(engine=cast(Engine, sqlite_session.get_bind()), session=sqlite_session) @@ -279,7 +283,7 @@ def _import_workflows( CapturingImportService( app_dsl=Mock(), - workflows=Mock(spec=WorkflowPublisher), + workflows=_workflow_publisher(), target_resolver=StubResolver(), workflow_tools=workflow_tools, ).import_package( @@ -291,9 +295,7 @@ def _import_workflows( def test_only_preserve_id_strategy_reuses_source_app_id(*, workflow_tools): - service = MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools - ) + service = MigrationImportService(app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools) assert service._should_preserve_source_app_id(ImportOptions(id_strategy=IdStrategy.PRESERVE_ID)) is True assert service._should_preserve_source_app_id(ImportOptions(id_strategy=IdStrategy.GENERATE_NEW_ID)) is False @@ -303,7 +305,7 @@ def test_find_existing_app_ignores_invalid_uuid(database: Database, *, workflow_ _persist_app(database.session, app_id="app-other", tenant_id="tenant-2") assert ( MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._find_existing_app("not-a-uuid", "tenant-1", session=database.session) is None ) @@ -319,7 +321,7 @@ def capture_statement(orm_execute_state) -> None: event.listen(database.session, "do_orm_execute", capture_statement) try: result = MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._find_existing_workflow_tool("tenant-1", "not-a-uuid", "tool-name", "app-id", session=database.session) finally: event.remove(database.session, "do_orm_execute", capture_statement) @@ -339,7 +341,7 @@ def capture_statement(orm_execute_state) -> None: event.listen(database.session, "do_orm_execute", capture_statement) try: result = MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._find_existing_mcp_tool("tenant-1", "my-test-mcp", "my-test-mcp", session=database.session) finally: event.remove(database.session, "do_orm_execute", capture_statement) @@ -367,7 +369,7 @@ def import_app(self, **kwargs): database.session.begin() imported_app_id = MigrationImportService( - app_dsl=StubAppDslService, workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=StubAppDslService, workflows=_workflow_publisher(), workflow_tools=workflow_tools )._import_workflow_app( account=object(), workflow_data={"name": "main_chatflow"}, @@ -417,7 +419,7 @@ def test_rewrite_workflow_dsl_replaces_tool_provider_ids(*, workflow_tools): ) rewritten = MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._rewrite_workflow_dsl_provider_ids( dsl_content, { @@ -463,7 +465,7 @@ def test_source_api_provider_ids_are_discovered_from_workflow_dsl(*, workflow_to ) assert MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._source_api_provider_ids_by_name(package) == {"weather": {"source-api-provider-id"}} @@ -489,7 +491,7 @@ def create_workflow_tool(**kwargs) -> None: monkeypatch.setattr(workflow_tools, "create_workflow_tool", create_workflow_tool) PublishingImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._import_workflow_tools( MigrationPackage.from_mapping( { @@ -512,7 +514,7 @@ def test_ensure_workflow_app_is_published_updates_current_workflow(database: Dat _, account = _persist_tenant_account(database.session) app_id = "00000000-0000-0000-0000-000000000001" _persist_app(database.session, app_id=app_id) - publisher = Mock(spec=WorkflowPublisher) + publisher = create_autospec(WorkflowPublisher, instance=True, spec_set=True) MigrationImportService( app_dsl=Mock(), workflows=publisher, workflow_tools=workflow_tools )._ensure_workflow_app_is_published( @@ -556,7 +558,7 @@ def create_workflow_tool(**kwargs) -> None: monkeypatch.setattr(workflow_tools, "create_workflow_tool", create_workflow_tool) StrategyImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._import_workflow_tools( MigrationPackage.from_mapping( { @@ -599,7 +601,7 @@ def _ensure_workflow_app_is_published(self, target, account, app_id, session): return None SkipImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._import_workflow_tools( MigrationPackage.from_mapping( { @@ -636,7 +638,7 @@ def test_api_tool_existing_provider_records_id_mapping( monkeypatch.setattr(import_service.ApiToolManageService, "update_api_tool_provider", lambda **kwargs: None) MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._import_api_tools( MigrationPackage.from_mapping( { @@ -681,7 +683,7 @@ def create_api_tool_provider(**_kwargs) -> None: monkeypatch.setattr(import_service.ApiToolManageService, "create_api_tool_provider", create_api_tool_provider) MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._import_api_tools( MigrationPackage.from_mapping( { @@ -722,7 +724,7 @@ def update_provider(self, **kwargs): monkeypatch.setattr(import_service, "MCPToolManageService", StubMCPToolManageService) MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._import_mcp_tools( MigrationPackage.from_mapping( { @@ -782,7 +784,7 @@ def update_provider(self, **kwargs): monkeypatch.setattr(import_service, "MCPToolManageService", StubMCPToolManageService) MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._import_mcp_tools( MigrationPackage.from_mapping( { @@ -838,7 +840,7 @@ def create_provider(self, **kwargs): monkeypatch.setattr(import_service, "MCPToolManageService", StubMCPToolManageService) MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._import_mcp_tools( MigrationPackage.from_mapping( { @@ -912,7 +914,7 @@ def test_dependency_only_mcp_preflight_reports_missing_target_provider_with_work ) MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._preflight_dependency_only_mcp( package, ImportTarget( @@ -949,7 +951,7 @@ def capture_statement(orm_execute_state) -> None: event.listen(database.session, "do_orm_execute", capture_statement) try: result = MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._find_dependency_only_mcp_provider( "tenant-1", "my-test-mcp-server", @@ -986,7 +988,7 @@ def test_dependency_only_mcp_preflight_reports_available_target_provider(databas ) MigrationImportService( - app_dsl=Mock(), workflows=Mock(spec=WorkflowPublisher), workflow_tools=workflow_tools + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools )._preflight_dependency_only_mcp( package, ImportTarget( @@ -1087,7 +1089,7 @@ def _import_mcp_tools( OrderedImportService( app_dsl=Mock(), - workflows=Mock(spec=WorkflowPublisher), + workflows=_workflow_publisher(), target_resolver=StubResolver(), workflow_tools=workflow_tools, ).import_package(ImportRequest(package=package), session=database.session) diff --git a/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py b/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py index e6f618ee70f1bd..54cc37ea571afb 100644 --- a/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py +++ b/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py @@ -1,11 +1,10 @@ -from repositories.knowledge.retrieval_repository import KnowledgeRetrievalRepository -from services.knowledge.retrieval.adapters.resource_events import DatasetIndexToolCallbackHandler -from services.tools.dataset.tool import DatasetRetrieverTool - """Chat and Completion release retrieval reads before model and HTTP calls.""" import json +from collections.abc import Iterator, Mapping from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, NoReturn, Self, override from uuid import uuid4 import httpx @@ -16,6 +15,7 @@ from sqlalchemy.pool import QueuePool from core.app.app_config.entities import DatasetEntity, DatasetRetrieveConfigEntity, ModelConfig +from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom from core.app.apps.chat.app_config_manager import ChatAppConfig from core.app.apps.completion.app_config_manager import CompletionAppConfig from core.app.entities.app_invoke_entities import ( @@ -24,11 +24,15 @@ InvokeFrom, ModelConfigWithCredentialsEntity, ) +from core.app.entities.queue_entities import AppQueueEvent from core.db import session_factory from core.model_manager import ModelManager +from core.tools.entities.tool_entities import ToolInvokeMessage from extensions.application_services.retrieval import build_dataset_retrieval from graphon.model_runtime.entities.llm_entities import LLMResultChunk, LLMResultChunkDelta, LLMUsage from graphon.model_runtime.entities.message_entities import AssistantPromptMessage +from graphon.model_runtime.entities.model_entities import ModelFeature +from models.annotation_reply import AnnotationReply from models.base import TypeBase from models.dataset import ( Dataset, @@ -38,51 +42,81 @@ ExternalKnowledgeApis, ExternalKnowledgeBindings, ) +from models.enums import ConversationFromSource from models.model import App, AppMode, Conversation, Message from repositories.app.generation_repository import AppGenerationRepository +from repositories.knowledge.retrieval_repository import KnowledgeRetrievalRepository from services.app.generation.adapters.chat_runner import ChatAppRunner from services.app.generation.adapters.completion_runner import CompletionAppRunner +from services.entities.external_knowledge_entities.external_knowledge_entities import ExternalKnowledgeApiSetting from services.knowledge.external.service import ExternalDatasetService +from services.knowledge.retrieval.adapters.resource_events import DatasetIndexToolCallbackHandler from services.knowledge.retrieval.dataset_retrieval import DatasetRetrieval +from services.tools.dataset.tool import DatasetRetrieverTool from tests.unit_tests.model_factories import make_dataset @dataclass -class Queue: - events: list = field(default_factory=list) +class Queue(AppQueueManager): + events: list[AppQueueEvent] = field(default_factory=list) - def publish(self, event, _source): + @override + def publish(self, event: AppQueueEvent, pub_from: PublishFrom) -> None: + del pub_from + self.events.append(event) + + @override + def _publish(self, event: AppQueueEvent, pub_from: PublishFrom) -> None: + del pub_from self.events.append(event) class NoAnnotations: - def query(self, **_kwargs): + def query( + self, + *, + tenant_id: str, + app_id: str, + message_id: str, + query: str, + user_id: str, + from_source: ConversationFromSource, + ) -> AnnotationReply | None: + del tenant_id, app_id, message_id, query, user_id, from_source return None @pytest.mark.parametrize("mode", [AppMode.CHAT, AppMode.COMPLETION, "agent-tool"]) @pytest.mark.parametrize("strategy", ["single", "multiple"]) -def test_runner_releases_connections_during_metadata_and_external_retrieval(tmp_path, monkeypatch, mode, strategy): +def test_runner_releases_connections_during_metadata_and_external_retrieval( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + mode: AppMode | Literal["agent-tool"], + strategy: Literal["single", "multiple"], +) -> None: engine = create_engine( f"sqlite:///{tmp_path / 'retrieval.db'}", poolclass=QueuePool, pool_size=1, max_overflow=0, ) + pool = engine.pool + assert isinstance(pool, QueuePool) sessions = sessionmaker(engine) # Production default: expire_on_commit=True. + tables = [ + TypeBase.metadata.tables[model.__tablename__] + for model in ( + Dataset, + Document, + DatasetMetadata, + DatasetQuery, + ExternalKnowledgeApis, + ExternalKnowledgeBindings, + ) + ] TypeBase.metadata.create_all( engine, - tables=[ - model.__table__ - for model in ( - Dataset, - Document, - DatasetMetadata, - DatasetQuery, - ExternalKnowledgeApis, - ExternalKnowledgeBindings, - ) - ], + tables=tables, ) tenant_id, dataset_id, user_id, app_id = [str(uuid4()) for _ in range(4)] with sessions.begin() as session: @@ -118,34 +152,34 @@ def test_runner_releases_connections_during_metadata_and_external_retrieval(tmp_ ), ] ) - calls = [] + calls: list[str] = [] - def no_global_database(*_args, **_kwargs): + def no_global_database(*_args: object, **_kwargs: object) -> NoReturn: raise AssertionError("retrieval used the global database") monkeypatch.setattr(session_factory, "create_session", no_global_database) @dataclass class Schema: - features: list = field(default_factory=list) + features: list[ModelFeature] = field(default_factory=list) class Model: model_name = "model" - credentials = {} + credentials: dict[str, object] = {} provider_model_bundle = object() @property - def model_type_instance(self): + def model_type_instance(self) -> Self: return self - def get_model_schema(self, **_kwargs): + def get_model_schema(self, **_kwargs: object) -> Schema: return Schema() - def get_model_instance(self, **_kwargs): + def get_model_instance(self, **_kwargs: object) -> Self: return self - def invoke_llm(self, **_kwargs): - assert engine.pool.checkedout() == 0 + def invoke_llm(self, **_kwargs: object) -> Iterator[LLMResultChunk]: + assert pool.checkedout() == 0 calls.append("metadata-model") yield LLMResultChunk( model="model", @@ -182,28 +216,36 @@ def invoke_llm(self, **_kwargs): monkeypatch.setattr(ModelManager, "for_tenant", lambda **_kwargs: model) monkeypatch.setattr(DatasetRetrieval, "_fetch_model_config", lambda *_args, **_kwargs: (model, model_config)) - def metadata_prompt(_self, **kwargs): + def metadata_prompt(_self: DatasetRetrieval, **kwargs: object) -> tuple[list[object], list[object]]: assert kwargs["metadata_fields"] == ["category"] # A real database read happened before model I/O. return [], [] monkeypatch.setattr(DatasetRetrieval, "_get_prompt_template", metadata_prompt) class Router: - def __init__(self, **_kwargs): + def __init__(self, **_kwargs: object) -> None: pass - def invoke(self, *_args, **_kwargs): - assert engine.pool.checkedout() == 0 + def invoke(self, *_args: object, **_kwargs: object) -> tuple[str, LLMUsage]: + assert pool.checkedout() == 0 calls.append("router") return dataset_id, LLMUsage.empty_usage() monkeypatch.setattr("services.knowledge.retrieval.dataset_retrieval.ReactMultiDatasetRouter", Router) - def external_request(request, _files): - assert engine.pool.checkedout() == 0 + def external_request(request: ExternalKnowledgeApiSetting, _files: dict[str, object] | None) -> httpx.Response: + assert pool.checkedout() == 0 assert request.url == "https://knowledge.example/retrieval" + assert request.headers is not None + assert request.params is not None assert request.headers["Authorization"] == "Bearer secret" - assert request.params["metadata_condition"]["conditions"][0]["value"] == "docs" + metadata_condition = request.params["metadata_condition"] + assert isinstance(metadata_condition, Mapping) + conditions = metadata_condition["conditions"] + assert isinstance(conditions, list) + condition = conditions[0] + assert isinstance(condition, Mapping) + assert condition["value"] == "docs" calls.append("external-knowledge") return httpx.Response( 200, json={"records": [{"content": "retrieved context", "title": "Docs", "score": 0.9, "metadata": {}}]} @@ -242,10 +284,12 @@ def external_request(request, _files): ) runner_type = ChatAppRunner if mode == AppMode.CHAT else CompletionAppRunner runner = runner_type(records=AppGenerationRepository(sessions), retrieval=build_dataset_retrieval(sessions)) - prompts = [] + prompts: list[str | None] = [] - def organize_prompt(**kwargs): - prompts.append(kwargs.get("context")) + def organize_prompt(**kwargs: object) -> tuple[list[object], list[object]]: + context = kwargs.get("context") + assert context is None or isinstance(context, str) + prompts.append(context) return [], [] monkeypatch.setattr(runner, "organize_prompt_messages", organize_prompt) @@ -258,6 +302,7 @@ def organize_prompt(**kwargs): try: with Flask(__name__).app_context(): if mode == "agent-tool": + assert config.dataset is not None tools = DatasetRetrieverTool.get_dataset_tools( records=KnowledgeRetrievalRepository(sessions), retrieval=build_dataset_retrieval(sessions), @@ -271,12 +316,23 @@ def organize_prompt(**kwargs): user_id=user_id, inputs={}, ) - prompts.append( - list(tools[0].invoke(user_id=user_id, tool_parameters={"query": "question"}))[0].message.text - ) + with sessions() as tool_session: + tool_message = list( + tools[0].invoke( + session=tool_session, + user_id=user_id, + tool_parameters={"query": "question"}, + ) + )[0] + assert isinstance(tool_message.message, ToolInvokeMessage.TextMessage) + prompts.append(tool_message.message.text) elif mode == AppMode.CHAT: + assert isinstance(runner, ChatAppRunner) + assert isinstance(entity, ChatAppGenerateEntity) runner.run(entity, queue, Conversation(), message, app, NoAnnotations()) else: + assert isinstance(runner, CompletionAppRunner) + assert isinstance(entity, CompletionAppGenerateEntity) runner.run(entity, queue, message, app) assert calls == [ "metadata-model", @@ -290,6 +346,6 @@ def organize_prompt(**kwargs): assert audit is not None assert audit.dataset_id == dataset_id assert audit.source_app_id == app_id - assert engine.pool.checkedout() == 0 + assert pool.checkedout() == 0 finally: engine.dispose() diff --git a/api/tests/unit_tests/services/test_app_dsl_service.py b/api/tests/unit_tests/services/test_app_dsl_service.py index 8caf53a2ad9811..7272093e7f8119 100644 --- a/api/tests/unit_tests/services/test_app_dsl_service.py +++ b/api/tests/unit_tests/services/test_app_dsl_service.py @@ -1,7 +1,7 @@ from collections.abc import Callable from types import SimpleNamespace from typing import cast -from unittest.mock import Mock +from unittest.mock import Mock, create_autospec import pytest import yaml @@ -193,7 +193,9 @@ def test_import_app_rejects_oversized_yaml_content_before_parsing( ) -> None: monkeypatch.setattr("services.app_dsl_service.DSL_MAX_SIZE", 3) service = AppDslService( - session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), ) account = _account() @@ -213,7 +215,9 @@ def test_import_app_rejects_oversized_yaml_url_bytes_before_decode( response.content = b"\xff\xff" monkeypatch.setattr("services.app_dsl_service.remote_fetcher.make_request", Mock(return_value=response)) service = AppDslService( - session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), ) result = service.import_app( @@ -235,7 +239,9 @@ def test_import_app_returns_decode_error_for_invalid_yaml_url_bytes( response.content = b"\xff" monkeypatch.setattr("services.app_dsl_service.remote_fetcher.make_request", Mock(return_value=response)) service = AppDslService( - session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), ) result = service.import_app( @@ -349,7 +355,9 @@ def test_pending_import_is_scoped_to_its_owner(monkeypatch: pytest.MonkeyPatch, lambda key, _expiry, value: pending_imports.__setitem__(key, value), ) service = AppDslService( - session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), ) creator = _account() @@ -397,7 +405,9 @@ def test_pending_import_requires_current_tenant(monkeypatch: pytest.MonkeyPatch, account._current_tenant = None result = AppDslService( - session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), ).import_app( account=account, import_mode="yaml-content", @@ -549,7 +559,9 @@ def test_create_or_update_app_rejects_null_required_site_setting_before_mutation with pytest.raises(ValueError, match="Required Site settings cannot be null"): AppDslService( - unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), )._create_or_update_app( app=app, data={"app": {"mode": AppMode.CHAT.value}, "site": {"title": None}}, @@ -563,7 +575,9 @@ def test_import_app_resolves_site_entitlement_before_database_writes( monkeypatch: pytest.MonkeyPatch, unbound_session: Session ) -> None: service = AppDslService( - unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), ) create_or_update = Mock(return_value=_app()) entitlement = Mock(return_value=False) @@ -618,7 +632,9 @@ def test_confirm_import_rechecks_site_entitlement(monkeypatch: pytest.MonkeyPatc entitlement = Mock(return_value=False) monkeypatch.setattr("services.app_dsl_service.FeatureService.can_import_premium_site_settings", entitlement) service = AppDslService( - unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), ) create_or_update = Mock(return_value=_app()) monkeypatch.setattr(service, "_create_or_update_app", create_or_update) @@ -762,8 +778,12 @@ def test_chat_dsl_import_preserves_valid_feature_fields(sqlite_session: Session) def test_create_or_update_app_removes_imported_workflow_viewport(unbound_session: Session) -> None: - drafts = Mock(spec=WorkflowDraftService) - service = AppDslService(session=unbound_session, drafts=drafts, overwrites=Mock(spec=AppDslOverwriteStore)) + drafts = create_autospec(WorkflowDraftService, instance=True, spec_set=True) + service = AppDslService( + session=unbound_session, + drafts=drafts, + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) app = make_app(mode=AppMode.WORKFLOW) imported_graph: dict[str, object] = { "nodes": [], @@ -831,7 +851,9 @@ def test_agent_import_of_new_agent_checks_function_scope( account = _account() AppDslService( - session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), )._ensure_agent_import_permission(account, app=None) check.assert_called_once_with( @@ -851,7 +873,9 @@ def test_create_or_update_app_gates_agent_mode_before_creation( config_overrides(RBAC_ENABLED=True) monkeypatch.setattr("services.app_dsl_service.RBACService.CheckAccess.check", Mock(return_value=False)) service = AppDslService( - session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), ) with pytest.raises(NoPermissionError): @@ -872,7 +896,9 @@ def test_import_app_reraises_permission_denial_instead_of_failed_result( config_overrides(RBAC_ENABLED=True) monkeypatch.setattr("services.app_dsl_service.RBACService.CheckAccess.check", Mock(return_value=False)) service = AppDslService( - session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), ) with pytest.raises(NoPermissionError): @@ -973,9 +999,13 @@ def append_model(export_data: dict[str, object], *_args: object, **_kwargs: obje def test_overwrite_rejects_incompatible_nodes_before_mutation(mode: AppMode, node_type: str) -> None: session = Mock() target = App(id="target", tenant_id="tenant", mode=mode, name="Original") - overwrites = Mock(spec=AppDslOverwriteStore) + overwrites = create_autospec(AppDslOverwriteStore, instance=True, spec_set=True) overwrites.snapshot.return_value = AppDslOverwriteTarget(mode, None) - service = AppDslService(session, drafts=Mock(spec=WorkflowDraftService), overwrites=overwrites) + service = AppDslService( + session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=overwrites, + ) result = service.import_app( account=_account(), import_mode="yaml-content", @@ -1010,7 +1040,9 @@ def test_missing_app_section_names_the_keys_that_were_present( unbound_session: Session, content: dict[str, object], found: str ) -> None: result = AppDslService( - session=unbound_session, drafts=Mock(spec=WorkflowDraftService), overwrites=Mock(spec=AppDslOverwriteStore) + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), ).import_app(account=_account(), import_mode="yaml-content", yaml_content=yaml.safe_dump(content, sort_keys=False)) assert result.status == ImportStatus.FAILED assert result.error is not None diff --git a/api/tests/unit_tests/services/test_app_generate_service.py b/api/tests/unit_tests/services/test_app_generate_service.py index 4f5e6df3c5bd46..9b807a30ca6cc1 100644 --- a/api/tests/unit_tests/services/test_app_generate_service.py +++ b/api/tests/unit_tests/services/test_app_generate_service.py @@ -30,13 +30,13 @@ from models.enums import AppStatus from models.model import App, AppMode from models.workflow import Workflow, WorkflowType +from services.app.generation.runtime import AppGenerationRuntime from services.app_generate_service import AppGenerateService from services.errors.app import ( TriggerWorkflowServiceModeUnavailableError, WorkflowIdFormatError, WorkflowNotFoundError, ) -from services.workflow.execution.ports import WorkflowRuntime from services.workflow.variable_contracts import WorkflowExecutionVariables @@ -318,7 +318,7 @@ def test_debug_generators_use_injected_loader_and_saver( mode: AppMode, entry: str, *, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ) -> None: from graphon.enums import BuiltinNodeTypes from services.app.generation.input_adapter import AppInputAdapter @@ -390,7 +390,7 @@ def test_completion_mode( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): gen_spy = mocker.patch( "services.app_generate_service.CompletionAppGenerator.generate", @@ -419,12 +419,11 @@ def test_agent_chat_mode( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): - gen_spy = mocker.patch( - "services.app_generate_service.AgentChatAppGenerator.generate", - return_value={"result": "agent"}, - ) + generator = mocker.patch("services.app_generate_service.AgentChatAppGenerator") + gen_spy = generator.return_value.generate + gen_spy.return_value = {"result": "agent"} mocker.patch( "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, @@ -441,6 +440,7 @@ def test_agent_chat_mode( ) assert result == {"result": "agent"} gen_spy.assert_called_once() + assert generator.call_args.kwargs["tool_invoker"] is workflow_runtime.agent_tool_invoker # -- AGENT_CHAT via is_agent flag (non-AGENT_CHAT mode) ----------------- def test_agent_via_is_agent_flag( @@ -448,12 +448,11 @@ def test_agent_via_is_agent_flag( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): - gen_spy = mocker.patch( - "services.app_generate_service.AgentChatAppGenerator.generate", - return_value={"result": "agent-via-flag"}, - ) + generator = mocker.patch("services.app_generate_service.AgentChatAppGenerator") + gen_spy = generator.return_value.generate + gen_spy.return_value = {"result": "agent-via-flag"} mocker.patch( "services.app_generate_service.convert_to_event_stream", side_effect=lambda x: x, @@ -473,6 +472,7 @@ def test_agent_via_is_agent_flag( ) assert result == {"result": "agent-via-flag"} gen_spy.assert_called_once() + assert generator.call_args.kwargs["tool_invoker"] is workflow_runtime.agent_tool_invoker is_agent.assert_called_once_with(session=session) # -- AGENT -------------------------------------------------------------- @@ -481,7 +481,7 @@ def test_agent_mode_passes_session( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): gen_spy = mocker.patch( "services.app_generate_service.AgentAppGenerator.generate", @@ -512,7 +512,7 @@ def test_chat_mode( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): gen_spy = mocker.patch( "services.app_generate_service.ChatAppGenerator.generate", @@ -542,7 +542,7 @@ def test_advanced_chat_blocking( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) @@ -581,7 +581,7 @@ def test_advanced_chat_streaming( config_overrides: Callable[..., None], *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): config_overrides(PUBSUB_REDIS_CHANNEL_TYPE="streams") workflow = _make_workflow() @@ -623,7 +623,7 @@ def test_workflow_blocking( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) @@ -664,7 +664,7 @@ def test_trigger_workflow_rejects_manual_service_surfaces( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ) -> None: workflow = _make_workflow(node_types=(node_type,)) mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) @@ -689,7 +689,7 @@ def test_trigger_workflow_allows_trigger_execution( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ) -> None: workflow = _make_workflow(node_types=("trigger-webhook",)) mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) @@ -721,7 +721,7 @@ def test_specific_start_workflow_version_remains_runnable( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ) -> None: workflow_id = str(uuid.uuid4()) workflow = _make_workflow(workflow_id=workflow_id, node_types=("start",)) @@ -758,7 +758,7 @@ def test_workflow_streaming( config_overrides: Callable[..., None], *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): config_overrides(PUBSUB_REDIS_CHANNEL_TYPE="streams") workflow = _make_workflow() @@ -800,7 +800,7 @@ def test_invalid_mode_raises( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): app = _make_app("invalid-mode") with pytest.raises(ValueError, match="Invalid app mode"): @@ -834,7 +834,7 @@ def test_cloud_edition_consumes_quota( config_overrides: Callable[..., None], *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) quota_charge = MagicMock() @@ -870,7 +870,7 @@ def test_billing_quota_exceeded_raises_rate_limit_error( config_overrides: Callable[..., None], *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): from services.errors.app import QuotaExceededError from services.errors.llm import InvokeRateLimitError @@ -899,7 +899,7 @@ def test_exception_refunds_quota_and_exits_rate_limit( config_overrides: Callable[..., None], *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) quota_charge = MagicMock() @@ -935,7 +935,7 @@ def test_rate_limit_exit_called_in_finally_for_blocking( config_overrides: Callable[..., None], *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): """For non-streaming (blocking) calls, rate_limit.exit should be called in finally.""" config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) @@ -975,7 +975,7 @@ def test_blocking_failure_exits_rate_limit_once( config_overrides: Callable[..., None], *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) quota_charge = MagicMock() @@ -1016,7 +1016,7 @@ def test_streaming_failure_exits_rate_limit_once( config_overrides: Callable[..., None], *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) quota_charge = MagicMock() @@ -1126,7 +1126,7 @@ def test_advanced_chat_mode( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) @@ -1157,7 +1157,7 @@ def test_workflow_mode( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) @@ -1189,7 +1189,7 @@ def test_invalid_mode_raises( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): app = _make_app(AppMode.CHAT) with pytest.raises(ValueError, match="Invalid app mode"): @@ -1213,7 +1213,7 @@ def test_advanced_chat_mode( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) @@ -1244,7 +1244,7 @@ def test_workflow_mode( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): workflow = _make_workflow() mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) @@ -1276,7 +1276,7 @@ def test_invalid_mode_raises( mocker: MockerFixture, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ): app = _make_app(AppMode.COMPLETION) with pytest.raises(ValueError, match="Invalid app mode"): diff --git a/api/tests/unit_tests/services/test_app_preview_details_adapters.py b/api/tests/unit_tests/services/test_app_preview_details_adapters.py index 8fad993c761d17..205714e752c08f 100644 --- a/api/tests/unit_tests/services/test_app_preview_details_adapters.py +++ b/api/tests/unit_tests/services/test_app_preview_details_adapters.py @@ -31,6 +31,8 @@ from models.tools import ApiToolProvider from models.workflow import Workflow, WorkflowType from repositories.app_preview_query_repository import AppPreviewQueryRepository +from repositories.tools.provider_repository import ToolProviderRepository +from repositories.tools.workflow_repository import WorkflowToolRepository from services.account_errors import AccountNotFoundError from services.app_definition_query_service import AppDefinitionUnavailableError from services.app_preview_details_adapters import AppPreviewDetailsRuntime @@ -84,7 +86,11 @@ def assert_read_only(self) -> None: @pytest.fixture def harness( - sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], *, tool_providers, workflow_queries + sqlite_engine: Engine, + sqlite_session_factory: sessionmaker[Session], + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> Iterator[_Harness]: target = App( id=str(uuid4()), tenant_id=str(uuid4()), name="Preview", mode="chat", enable_site=True, enable_api=True @@ -563,8 +569,8 @@ def test_agent_tool_masking_returns_a_copy_without_dirtying_or_flushing_models( monkeypatch: pytest.MonkeyPatch, failure: Literal["none", "runtime", "decrypt"], *, - tool_providers, - workflow_queries, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> None: provider_id = str(uuid4()) agent_mode = { @@ -598,7 +604,13 @@ def test_agent_tool_masking_returns_a_copy_without_dirtying_or_flushing_models( checkedout_during_io: list[int] = [] def runtime( - *, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str, tool_providers, workflow_queries + *, + tenant_id: str, + app_id: str, + agent_tool: AgentToolEntity, + user_id: str, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> Tool: del workflow_queries, tool_providers assert isinstance(harness.engine.pool, QueuePool) diff --git a/api/tests/unit_tests/services/test_installed_app_generation_adapters.py b/api/tests/unit_tests/services/test_installed_app_generation_adapters.py index 59f3c2ba6e5490..46f724de7f364a 100644 --- a/api/tests/unit_tests/services/test_installed_app_generation_adapters.py +++ b/api/tests/unit_tests/services/test_installed_app_generation_adapters.py @@ -16,6 +16,7 @@ from core.app.entities.app_invoke_entities import InvokeFrom from core.app.features.rate_limiting.rate_limit import RateLimit, RateLimitGenerator from enums import DeploymentEdition +from extensions.application_services.workflow import WorkflowExecutionDependencies from extensions.ext_database import db from extensions.ext_redis import RedisClientWrapper from libs.broadcast_channel.redis.streams_channel import ( @@ -24,9 +25,12 @@ _StreamsSubscription, ) from models import Account, App, AppMode, AppModelConfig, Conversation, Message, Workflow +from models.annotation_reply import AnnotationReplies from models.enums import ConversationFromSource from models.workflow import WorkflowType from services.account_errors import AccountNotFoundError +from services.app.generation.ports import ChatRecords +from services.app.generation.runtime import AppGenerationRuntime from services.app_definition_query_service import AppDefinitionUnavailableError from services.errors.app import MoreLikeThisDisabledError from services.errors.app_model_config import AppModelConfigBrokenError @@ -34,8 +38,8 @@ from services.errors.message import MessageNotExistsError from services.installed_app_generation_adapters import AppGenerateServiceRuntime from services.installed_app_generation_service import GenerationResponse +from services.knowledge.retrieval.ports import DatasetRetrievalFactory from services.workflow.execution.adapters import response_stream -from services.workflow.execution.ports import WorkflowRuntime from services.workflow.variable_contracts import WorkflowExecutionVariables _ARGS: dict[str, object] = {"inputs": {"count": 0}, "query": "hello", "auto_generate_name": False} @@ -57,7 +61,7 @@ def harness( sqlite_session_factory: sessionmaker[Session], *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: WorkflowExecutionDependencies, ) -> _RuntimeHarness: with sqlite_session_factory.begin() as session: app = App( @@ -130,7 +134,7 @@ def assert_context( def boundary_generate( *, variables: WorkflowExecutionVariables, - runtime: WorkflowRuntime, + runtime: AppGenerationRuntime, session: Session, app_model: App, user: Account, @@ -147,9 +151,9 @@ def boundary_generate( def boundary_generate_more_like_this( *, - retrieval, - records, - annotations, + retrieval: DatasetRetrievalFactory, + records: ChatRecords, + annotations: AnnotationReplies, session: Session, app_model: App, user: Account, diff --git a/api/tests/unit_tests/services/test_snippet_dsl_service.py b/api/tests/unit_tests/services/test_snippet_dsl_service.py index f15a89ddd40a13..5756bdc7f3854f 100644 --- a/api/tests/unit_tests/services/test_snippet_dsl_service.py +++ b/api/tests/unit_tests/services/test_snippet_dsl_service.py @@ -1,6 +1,6 @@ import json from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import Mock, create_autospec import pytest import yaml @@ -548,10 +548,12 @@ def test_confirm_import_returns_failed_when_create_or_update_raises( def test_check_dependencies_returns_empty_without_draft_workflow( service: SnippetDslService, monkeypatch: pytest.MonkeyPatch ): + snippets = create_autospec(SnippetService, instance=True) + snippets.get_draft_workflow.return_value = None monkeypatch.setattr( service, "_snippets", - Mock(spec=SnippetService, get_draft_workflow=Mock(return_value=None)), + snippets, ) result = service.check_dependencies(_snippet()) @@ -567,10 +569,12 @@ def test_check_dependencies_returns_generated_dependencies(service: SnippetDslSe "value": {"marketplace_plugin_unique_identifier": "langgenius/openai:0.0.1"}, } ] + snippets = create_autospec(SnippetService, instance=True) + snippets.get_draft_workflow.return_value = workflow monkeypatch.setattr( service, "_snippets", - Mock(spec=SnippetService, get_draft_workflow=Mock(return_value=workflow)), + snippets, ) monkeypatch.setattr(service, "_extract_dependencies_from_workflow", Mock(return_value=["langgenius/openai"])) monkeypatch.setattr( @@ -678,10 +682,12 @@ def test_create_or_update_snippet_creates_new_snippet_and_flushes( def test_export_snippet_dsl_raises_without_draft_workflow(service: SnippetDslService, monkeypatch: pytest.MonkeyPatch): + snippets = create_autospec(SnippetService, instance=True) + snippets.get_draft_workflow.return_value = None monkeypatch.setattr( service, "_snippets", - Mock(spec=SnippetService, get_draft_workflow=Mock(return_value=None)), + snippets, ) with pytest.raises(ValueError, match="Missing draft workflow"): @@ -696,10 +702,12 @@ def test_export_snippet_dsl_returns_yaml(service: SnippetDslService, monkeypatch icon_info=None, input_fields=[{"variable": "query"}], ) + snippets = create_autospec(SnippetService, instance=True) + snippets.get_draft_workflow.return_value = workflow monkeypatch.setattr( service, "_snippets", - Mock(spec=SnippetService, get_draft_workflow=Mock(return_value=workflow)), + snippets, ) monkeypatch.setattr( "services.snippet_dsl_service.DependenciesAnalysisService.generate_dependencies", @@ -720,14 +728,13 @@ def test_export_snippet_dsl_uses_requested_published_workflow( snippet = _snippet(name="Exported") get_published_workflow_by_id = Mock(return_value=workflow) get_draft_workflow = Mock() + snippets = create_autospec(SnippetService, instance=True) + snippets.get_draft_workflow = get_draft_workflow + snippets.get_published_workflow_by_id = get_published_workflow_by_id monkeypatch.setattr( service, "_snippets", - Mock( - spec=SnippetService, - get_draft_workflow=get_draft_workflow, - get_published_workflow_by_id=get_published_workflow_by_id, - ), + snippets, ) monkeypatch.setattr( "services.snippet_dsl_service.DependenciesAnalysisService.generate_dependencies", diff --git a/api/tests/unit_tests/services/test_snippet_generate_service.py b/api/tests/unit_tests/services/test_snippet_generate_service.py index 0598f7be0edbc1..b977d4c9e389bf 100644 --- a/api/tests/unit_tests/services/test_snippet_generate_service.py +++ b/api/tests/unit_tests/services/test_snippet_generate_service.py @@ -1,6 +1,6 @@ import json from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import Mock, create_autospec import pytest from sqlalchemy.orm import Session, sessionmaker @@ -273,7 +273,7 @@ def test_run_draft_node_delegates_to_workflow_service( snippet = _snippet() account = _account("account-1") execution = WorkflowNodeExecutionModel(id="execution-1") - workflow_service = Mock(spec=WorkflowService) + workflow_service = create_autospec(WorkflowService, instance=True, spec_set=True) workflow_service.run_draft_workflow_node.return_value = execution snippet_reader.workflow = workflow diff --git a/api/tests/unit_tests/services/test_trial_app_generation_adapters.py b/api/tests/unit_tests/services/test_trial_app_generation_adapters.py index 8be7abd003c1bb..c7c2d38c4a05d4 100644 --- a/api/tests/unit_tests/services/test_trial_app_generation_adapters.py +++ b/api/tests/unit_tests/services/test_trial_app_generation_adapters.py @@ -9,6 +9,7 @@ from sqlalchemy.pool import QueuePool from core.app.entities.app_invoke_entities import InvokeFrom +from extensions.application_services.workflow import WorkflowExecutionDependencies from models import Account, App, AppMode, Conversation from models.enums import ConversationFromSource from models.model import AccountTrialAppRecord @@ -16,6 +17,7 @@ from services.account_errors import AccountNotFoundError from services.app.generation.adapters.chat import ChatAppGenerator from services.app.generation.response import convert_to_event_stream +from services.app.generation.runtime import AppGenerationRuntime from services.app_definition_query_service import AppDefinitionUnavailableError from services.app_generate_service import AppGenerateService from services.errors.app_model_config import AppModelConfigBrokenError @@ -23,7 +25,6 @@ from services.trial_app_access_service import TrialAppRef from services.trial_app_generation_adapters import AppGenerateServiceRuntime from services.trial_app_generation_service import GenerationResponse, TrialAppGenerationService -from services.workflow.execution.ports import WorkflowRuntime from services.workflow.variable_contracts import WorkflowExecutionVariables _ARGS: dict[str, object] = {"inputs": {"count": 0}, "query": "hello", "auto_generate_name": False} @@ -44,7 +45,7 @@ def harness( sqlite_session_factory: sessionmaker[Session], *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: WorkflowExecutionDependencies, ) -> _Harness: with sqlite_session_factory.begin() as session: app = App(tenant_id=str(uuid4()), name="Trial app", mode=AppMode.CHAT, enable_site=True, enable_api=False) @@ -117,7 +118,7 @@ def _patch_generation( def boundary_generate( *, variables: WorkflowExecutionVariables, - runtime: WorkflowRuntime, + runtime: AppGenerationRuntime, session: Session, app_model: App, user: Account, @@ -369,7 +370,7 @@ def _patch_chat_generation(monkeypatch: pytest.MonkeyPatch, harness: _Harness) - def boundary_generate( *, variables: WorkflowExecutionVariables, - runtime: WorkflowRuntime, + runtime: AppGenerationRuntime, session: Session, app_model: App, user: Account, diff --git a/api/tests/unit_tests/services/tools/test_api_provider_runtime.py b/api/tests/unit_tests/services/tools/test_api_provider_runtime.py index cefbf81d3da6f4..d64f362dd2c0b9 100644 --- a/api/tests/unit_tests/services/tools/test_api_provider_runtime.py +++ b/api/tests/unit_tests/services/tools/test_api_provider_runtime.py @@ -1,14 +1,17 @@ """Exercise Agent API-tool invocation through real provider and credentials reads.""" import json +from collections.abc import Iterator +from typing import cast import httpx import pytest -from sqlalchemy import create_engine, event -from sqlalchemy.orm import sessionmaker +from sqlalchemy import Table, create_engine, event +from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import QueuePool from core.db import session_factory +from core.tools.custom_tool.provider import ApiToolProviderController from core.tools.entities.tool_bundle import ApiToolBundle from core.tools.entities.tool_entities import ApiProviderSchemaType from core.tools.errors import ToolInvokeError @@ -29,19 +32,26 @@ _service, ) +type Databases = tuple[sessionmaker[Session], Session] + @pytest.fixture -def databases(monkeypatch): +def databases(monkeypatch: pytest.MonkeyPatch) -> Iterator[Databases]: engines = [create_engine("sqlite://", poolclass=QueuePool) for _ in range(2)] for engine in engines: - TypeBase.metadata.create_all(engine, tables=[model.__table__ for model in (App, Account, ApiToolProvider)]) + tables: list[Table] = [] + for model in (App, Account, ApiToolProvider): + table = model.__table__ + assert isinstance(table, Table) + tables.append(table) + TypeBase.metadata.create_all(engine, tables=tables) sessions, global_sessions = [sessionmaker(engine, expire_on_commit=False) for engine in engines] with global_sessions() as global_session: monkeypatch.setattr(db, "session", global_session) monkeypatch.setattr(type(db), "engine", property(lambda _db: engines[1])) monkeypatch.setattr(session_factory, "create_session", global_sessions) - def reject_global(*_args): + def reject_global(*_args: object) -> None: pytest.fail("Tool runtime queried the global database") event.listen(engines[1], "before_cursor_execute", reject_global) @@ -54,7 +64,9 @@ def reject_global(*_args): @pytest.mark.parametrize("http_failure", [False, True]) -def test_api_tool_provider_read_finishes_before_decryption_and_http(databases, monkeypatch, http_failure): +def test_api_tool_provider_read_finishes_before_decryption_and_http( + databases: Databases, monkeypatch: pytest.MonkeyPatch, http_failure: bool +) -> None: sessions, global_session = databases with sessions() as session: _persist_app(session) @@ -85,29 +97,30 @@ def test_api_tool_provider_read_finishes_before_decryption_and_http(databases, m session.add(provider) session.commit() engine = sessions.kw["bind"] - calls = [] + calls: list[str] = [] - def released(): + def released() -> None: assert engine.pool.checkedout() == 0 assert not global_session.in_transaction() class Decrypter: - def decrypt(self, credentials): + def decrypt(self, credentials: dict[str, object]) -> dict[str, object]: released() assert credentials["api_key_value"] == "encrypted" calls.append("decrypt") return {**credentials, "api_key_value": "clear-secret"} - def encryption(*, tenant_id, controller): + def encryption(*, tenant_id: str, controller: ApiToolProviderController) -> tuple[Decrypter, None]: released() assert tenant_id == TENANT_ID assert controller.entity.identity.author == "Injected author" return Decrypter(), None - def http_get(url, **kwargs): + def http_get(url: str, **kwargs: object) -> httpx.Response: released() assert url == "https://api.example.com/search" - assert kwargs["headers"]["Authorization"] == "clear-secret" + headers = cast(dict[str, str], kwargs["headers"]) + assert headers["Authorization"] == "clear-secret" calls.append("http") if http_failure: raise ToolInvokeError("remote unavailable") @@ -131,7 +144,7 @@ def http_get(url, **kwargs): released() -def test_api_provider_lookup_is_tenant_scoped(databases): +def test_api_provider_lookup_is_tenant_scoped(databases: Databases) -> None: sessions, _ = databases with sessions.begin() as session: provider = ApiToolProvider( diff --git a/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py b/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py index e9fd8e55002c9a..3397415c0cb838 100644 --- a/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py +++ b/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py @@ -1,11 +1,14 @@ """Exercise selected builtin credentials through real repositories and real tool construction.""" import json +from collections.abc import Generator, Iterator +from typing import cast +from unittest.mock import create_autospec from uuid import uuid4 import pytest -from sqlalchemy import create_engine, event -from sqlalchemy.orm import sessionmaker +from sqlalchemy import Engine, Table, create_engine, event +from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import QueuePool from core.db import session_factory @@ -23,35 +26,42 @@ from services.tools.agent_invocation_gateway import AgentToolInvocationGateway from services.tools.builtin import credentials as credentials_module from services.tools.tool_manager import ToolManager +from services.workflow.variable_contracts import WorkflowExecutionVariables TENANT = "tenant" PROVIDER = "vendor/search/search" @pytest.fixture -def databases(monkeypatch): - engines = [create_engine("sqlite://", poolclass=QueuePool) for _ in range(2)] +def databases( + monkeypatch: pytest.MonkeyPatch, +) -> Generator[tuple[sessionmaker[Session], Session], None, None]: + engines: list[Engine] = [create_engine("sqlite://", poolclass=QueuePool) for _ in range(2)] for engine in engines: TypeBase.metadata.create_all( engine, - tables=[ - model.__table__ - for model in ( - Account, - BuiltinToolProvider, - ToolOAuthTenantClient, - ToolOAuthSystemClient, - MCPToolProvider, - ) - ], + tables=cast( + list[Table], + [ + model.__table__ + for model in ( + Account, + BuiltinToolProvider, + ToolOAuthTenantClient, + ToolOAuthSystemClient, + MCPToolProvider, + ) + ], + ), ) - sessions, global_sessions = [sessionmaker(engine, expire_on_commit=False) for engine in engines] + sessions: sessionmaker[Session] = sessionmaker(engines[0], expire_on_commit=False) + global_sessions: sessionmaker[Session] = sessionmaker(engines[1], expire_on_commit=False) with global_sessions() as global_session: monkeypatch.setattr(db, "session", global_session) monkeypatch.setattr(type(db), "engine", property(lambda _db: engines[1])) monkeypatch.setattr(session_factory, "create_session", global_sessions) - def reject_global(*_args): + def reject_global(*_args: object) -> None: pytest.fail("Tool runtime read the global database") event.listen(engines[1], "before_cursor_execute", reject_global) @@ -63,7 +73,12 @@ def reject_global(*_args): engine.dispose() -def seed(sessions, *, expires_at=1, oauth_client="tenant"): +def seed( + sessions: sessionmaker[Session], + *, + expires_at: int = 1, + oauth_client: str = "tenant", +) -> str: with sessions.begin() as session: credential = BuiltinToolProvider( tenant_id=TENANT, @@ -86,7 +101,7 @@ def seed(sessions, *, expires_at=1, oauth_client="tenant"): return credential.id -def controller(): +def controller() -> PluginToolProviderController: entity = ToolProviderEntityWithPlugin.model_validate( { "identity": { @@ -112,43 +127,48 @@ def controller(): @pytest.mark.parametrize("client", ["tenant", "system"]) @pytest.mark.parametrize("outcome", ["success", "refresh-error", "concurrent-edit", "concurrent-delete"]) -def test_selected_oauth_credential_releases_connections_across_full_invocation(databases, monkeypatch, client, outcome): +def test_selected_oauth_credential_releases_connections_across_full_invocation( + databases: tuple[sessionmaker[Session], Session], + monkeypatch: pytest.MonkeyPatch, + client: str, + outcome: str, +) -> None: sessions, global_session = databases credential_id = seed(sessions, oauth_client=client) runtime = build_workflow_execution_dependencies(sessions) - calls = [] + calls: list[str] = [] - def released(): + def released() -> None: assert sessions.kw["bind"].pool.checkedout() == 0 assert not global_session.in_transaction() class Codec: - def decrypt(self, values): + def decrypt(self, values: dict[str, str]) -> dict[str, str]: released() calls.append("decrypt") return {k: v.removeprefix("encrypted-") for k, v in values.items()} - def encrypt(self, values): + def encrypt(self, values: dict[str, str]) -> dict[str, str]: released() calls.append("encrypt") return {k: "encrypted-" + v for k, v in values.items()} class Cache: - def delete(self): + def delete(self) -> None: released() calls.append("invalidate") - def encrypter(**_kwargs): + def encrypter(**_kwargs: object) -> tuple[Codec, Cache]: released() return Codec(), Cache() - def policy(**kwargs): + def policy(**kwargs: object) -> None: released() assert kwargs["check_existence"] is False assert kwargs["credential_id"] == credential_id calls.append("policy") - def refresh(_self, **kwargs): + def refresh(_self: object, **kwargs: object) -> PluginOAuthCredentialsResponse: released() calls.append("refresh") assert kwargs["credentials"] == {"token": "old"} @@ -165,18 +185,18 @@ def refresh(_self, **kwargs): record.encrypted_credentials = '{"token":"manual-edit"}' return PluginOAuthCredentialsResponse(credentials={"token": "new"}, expires_at=4_000_000_000) - def invoke(_self, **kwargs): + def invoke(_self: object, **kwargs: object) -> Iterator[object]: released() calls.append("invoke") assert kwargs["credentials"] == {"token": "new"} return iter(()) - def verified(*_args): + def verified(*_args: object) -> bool: released() calls.append("verified") return True - def decrypt_system(params): + def decrypt_system(params: str) -> dict[str, str]: released() assert params == "system" return {"client": "client"} @@ -189,9 +209,8 @@ def decrypt_system(params): monkeypatch.setattr(credentials_module, "decrypt_system_params", decrypt_system) monkeypatch.setattr("core.plugin.impl.tool.PluginToolManager.invoke", invoke) - class Variables: - def saver_factory(self, *_args, **_kwargs): - raise AssertionError("No workflow variables for this tool") + variables = create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True) + variables.saver_factory.side_effect = AssertionError("No workflow variables for this tool") request = AgentToolInvokeRequest.model_validate( { @@ -210,7 +229,7 @@ def saver_factory(self, *_args, **_kwargs): }, } ) - gateway = AgentToolInvocationGateway(variables=Variables(), runtime=runtime) + gateway = AgentToolInvocationGateway(variables=variables, runtime=runtime) if outcome == "success": gateway.invoke(request) assert calls[-1] == "invoke" @@ -234,11 +253,16 @@ def saver_factory(self, *_args, **_kwargs): @pytest.mark.parametrize("wrong_scope", ["tenant", "provider"]) -def test_selected_credential_cannot_cross_owner_scope(databases, monkeypatch, wrong_scope): +def test_selected_credential_cannot_cross_owner_scope( + databases: tuple[sessionmaker[Session], Session], + monkeypatch: pytest.MonkeyPatch, + wrong_scope: str, +) -> None: sessions, _ = databases credential_id = seed(sessions, expires_at=-1) with sessions.begin() as session: record = session.get(BuiltinToolProvider, credential_id) + assert record is not None if wrong_scope == "tenant": record.tenant_id = "other" else: @@ -258,7 +282,11 @@ def test_selected_credential_cannot_cross_owner_scope(databases, monkeypatch, wr @pytest.mark.parametrize("reference", ["id", "identifier", "uuid-identifier", "missing", "other-tenant"]) -def test_mcp_runtime_resolves_injected_provider_before_decryption(databases, monkeypatch, reference): +def test_mcp_runtime_resolves_injected_provider_before_decryption( + databases: tuple[sessionmaker[Session], Session], + monkeypatch: pytest.MonkeyPatch, + reference: str, +) -> None: sessions, _ = databases with sessions.begin() as session: record = MCPToolProvider( @@ -272,7 +300,7 @@ def test_mcp_runtime_resolves_injected_provider_before_decryption(databases, mon ) session.add(record) - def build(provider): + def build(provider: MCPToolProvider) -> str: assert sessions.kw["bind"].pool.checkedout() == 0 assert provider.id == record.id return "controller" diff --git a/api/tests/unit_tests/services/workflow/test_conversion_service.py b/api/tests/unit_tests/services/workflow/test_conversion_service.py index ab1e667185ca81..13843a9a0c0f4c 100644 --- a/api/tests/unit_tests/services/workflow/test_conversion_service.py +++ b/api/tests/unit_tests/services/workflow/test_conversion_service.py @@ -1,6 +1,8 @@ """Real conversion, isolated persistence, and failures at the atomic write boundary.""" import json +from collections.abc import Mapping +from typing import cast import pytest from sqlalchemy import event, func, select @@ -17,6 +19,7 @@ from services.entities.app_entities import AppEvent from services.errors.workflow_service import WorkflowConversionError from services.workflow.conversion_service import WorkflowConversionService +from services.workflow.workflow_converter import WorkflowConverter from tests.unit_tests.model_factories import make_account, make_app, make_tenant CONTEXT = RequestContext("conversion", None, "account-1", "tenant-1") @@ -82,7 +85,7 @@ class ConversionSession(Session): ] ) config.id = "config-1" - return factory + return cast(sessionmaker[Session], factory) @pytest.mark.parametrize("mode", [AppMode.CHAT, AppMode.COMPLETION]) @@ -126,7 +129,10 @@ def created(*, event: AppEvent, account_id: str, backing_agent_id: str | None) - notifications.append(event) service = WorkflowConversionService( - ConsoleAppRepository(session_factory=sessions), decrypt_token=decrypt, notify_created=created + ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), + decrypt_token=decrypt, + notify_created=created, ) new_id = service.convert(CONTEXT, "app-1", {"name": "", "icon_type": "", "icon": "", "icon_background": ""}) assert len(notifications) == 1 @@ -189,6 +195,7 @@ def fail_write(session: Session, *_: object) -> None: notifications = [] service = WorkflowConversionService( ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), decrypt_token=lambda *_: "key", notify_created=lambda **kwargs: notifications.append(kwargs), ) @@ -224,10 +231,11 @@ def test_conversion_rejects_invalid_owned_data_before_creating_target( extension.tenant_id = "another-tenant" else: session.delete(extension) - notifications = [] - decryptions = [] + notifications: list[Mapping[str, object]] = [] + decryptions: list[tuple[object, ...]] = [] service = WorkflowConversionService( ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), decrypt_token=lambda *args: decryptions.append(args) or "key", notify_created=lambda **kwargs: notifications.append(kwargs), ) @@ -242,7 +250,10 @@ def test_conversion_rejects_invalid_owned_data_before_creating_target( def test_conversion_rejects_other_tenant(sessions: sessionmaker[Session]) -> None: service = WorkflowConversionService( - ConsoleAppRepository(session_factory=sessions), decrypt_token=lambda *_: "key", notify_created=lambda **_: None + ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), + decrypt_token=lambda *_: "key", + notify_created=lambda **_: None, ) with pytest.raises(ConsoleAppNotFoundError): service.convert(CONTEXT._replace(active_workspace_id="other-tenant"), "app-1", {}) @@ -253,7 +264,10 @@ def fail(*_: str) -> str: raise RuntimeError("key service unavailable") service = WorkflowConversionService( - ConsoleAppRepository(session_factory=sessions), decrypt_token=fail, notify_created=lambda **_: None + ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), + decrypt_token=fail, + notify_created=lambda **_: None, ) with pytest.raises(RuntimeError, match="key service unavailable"): service.convert(CONTEXT, "app-1", {}) diff --git a/api/tests/unit_tests/services/workflow/test_generation_read_boundary.py b/api/tests/unit_tests/services/workflow/test_generation_read_boundary.py index 1da40fb58a5eab..1cfef2cf479279 100644 --- a/api/tests/unit_tests/services/workflow/test_generation_read_boundary.py +++ b/api/tests/unit_tests/services/workflow/test_generation_read_boundary.py @@ -1,7 +1,7 @@ """Generation must release request reads before waiting for a worker's response.""" import json -from unittest.mock import Mock +from unittest.mock import Mock, create_autospec import pytest from flask import Flask @@ -14,10 +14,10 @@ from models.enums import EndUserType from models.model import App, AppMode, EndUser from models.workflow import Workflow, WorkflowType +from services.app.generation.runtime import AppGenerationRuntime from services.app_generate_service import AppGenerateService from services.workflow.execution.adapters.chatflow import app_generator as chatflow_module from services.workflow.execution.adapters.workflow import app_generator as workflow_module -from services.workflow.execution.ports import WorkflowRuntime from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.unit_tests.services.test_app_generate_service import _DummyRateLimit @@ -27,7 +27,7 @@ def test_blocking_generation_detaches_request_inputs_before_waiting( app: Flask, sqlite_session: Session, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, workflow_variables: WorkflowExecutionVariables, monkeypatch: pytest.MonkeyPatch, mode: AppMode, @@ -70,21 +70,22 @@ def test_blocking_generation_detaches_request_inputs_before_waiting( sqlite_session.refresh(application) assert sqlite_session.in_transaction() - module = chatflow_module if mode == AppMode.ADVANCED_CHAT else workflow_module - generator = module.AdvancedChatAppGenerator if mode == AppMode.ADVANCED_CHAT else module.WorkflowAppGenerator - converter = ( - module.AdvancedChatAppGenerateResponseConverter - if mode == AppMode.ADVANCED_CHAT - else module.WorkflowAppGenerateResponseConverter - ) + trace_queue_manager = create_autospec(TraceQueueManager, instance=True, spec_set=True) + if mode == AppMode.ADVANCED_CHAT: + generator = chatflow_module.AdvancedChatAppGenerator + converter = chatflow_module.AdvancedChatAppGenerateResponseConverter + monkeypatch.setattr(chatflow_module, "TraceQueueManager", Mock(return_value=trace_queue_manager)) + else: + generator = workflow_module.WorkflowAppGenerator + converter = workflow_module.WorkflowAppGenerateResponseConverter + monkeypatch.setattr(workflow_module, "TraceQueueManager", Mock(return_value=trace_queue_manager)) monkeypatch.setattr("services.app_generate_service.RateLimit", _DummyRateLimit) monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", lambda *_args, **_kwargs: None) - monkeypatch.setattr(module, "TraceQueueManager", Mock(return_value=Mock(spec=TraceQueueManager))) monkeypatch.setattr(generator, "_generate_worker", staticmethod(lambda **_kwargs: None)) monkeypatch.setattr(converter, "convert", lambda *, response, **_kwargs: response) - waits = [] + waits: list[str] = [] - def wait_for_response(_self, **_kwargs): + def wait_for_response(_self: object, **_kwargs: object) -> dict[str, str]: assert not sqlite_session.in_transaction() for row in (application, user, workflow): assert inspect(row).detached diff --git a/api/tests/unit_tests/services/workflow/test_human_input_persistence.py b/api/tests/unit_tests/services/workflow/test_human_input_persistence.py index 2f2995cf6f16d9..49d63b30f2f3a7 100644 --- a/api/tests/unit_tests/services/workflow/test_human_input_persistence.py +++ b/api/tests/unit_tests/services/workflow/test_human_input_persistence.py @@ -1,11 +1,13 @@ """Human Input nodes persist through the execution's injected database.""" import json +from collections.abc import Iterator from datetime import timedelta +from typing import cast import pytest -from sqlalchemy import create_engine, event, select -from sqlalchemy.orm import sessionmaker +from sqlalchemy import Table, create_engine, event, select +from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import QueuePool from core.app.entities.app_invoke_entities import InvokeFrom @@ -14,25 +16,33 @@ from enums.human_input import HumanInputFormStatus, RecipientType from extensions.application_services.workflow import build_workflow_execution_dependencies from extensions.application_services.workflow_variables import build_workflow_variable_service -from graphon.node_events import PauseRequestedEvent +from graphon.graph_engine import GraphEngine +from graphon.graph_events import NodeRunSucceededEvent +from graphon.node_events import PauseRequestedEvent, StreamCompletedEvent from graphon.runtime import GraphRuntimeState, VariablePool from libs.datetime_utils import naive_utc_now from models.base import TypeBase from models.human_input import HumanInputDelivery, HumanInputForm, HumanInputFormRecipient from models.human_input_entities import HumanInputNodeData, UserActionConfig from services.workflow.execution.adapters.node_factory import DifyNodeFactory +from services.workflow.execution.ports import WorkflowRuntime from tasks.app_generate.workflow_execute_task import AppExecutionParams, _Account, _AppRunner from tests.workflow_test_utils import build_test_graph_init_params +type FormDatabases = tuple[sessionmaker[Session], sessionmaker[Session]] + @pytest.fixture(params=[False, True], ids=["retain-after-commit", "expire-after-commit"]) -def form_databases(monkeypatch, request): +def form_databases(monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest) -> Iterator[FormDatabases]: engines = [create_engine("sqlite://", poolclass=QueuePool) for _ in range(2)] for engine in engines: - TypeBase.metadata.create_all( - engine, tables=[model.__table__ for model in (HumanInputForm, HumanInputDelivery, HumanInputFormRecipient)] - ) - injected, global_sessions = [sessionmaker(engine, expire_on_commit=request.param) for engine in engines] + tables: list[Table] = [] + for model in (HumanInputForm, HumanInputDelivery, HumanInputFormRecipient): + table = model.__table__ + assert isinstance(table, Table) + tables.append(table) + TypeBase.metadata.create_all(engine, tables=tables) + injected, global_sessions = [sessionmaker(engine, expire_on_commit=bool(request.param)) for engine in engines] monkeypatch.setattr(session_factory, "create_session", global_sessions) try: yield injected, global_sessions @@ -41,7 +51,7 @@ def form_databases(monkeypatch, request): engine.dispose() -def factory(runtime, invoke_from=InvokeFrom.DEBUGGER): +def factory(runtime: WorkflowRuntime | None, invoke_from: InvokeFrom = InvokeFrom.DEBUGGER) -> DifyNodeFactory: state = GraphRuntimeState( variable_pool=VariablePool.from_bootstrap( system_variables=build_system_variables(app_id="app", workflow_execution_id="run") @@ -55,7 +65,7 @@ def factory(runtime, invoke_from=InvokeFrom.DEBUGGER): ) -def node_config(): +def node_config() -> dict[str, object]: return { "id": "human-node", "data": HumanInputNodeData( @@ -69,7 +79,9 @@ def node_config(): @pytest.mark.parametrize("invoke_from", [InvokeFrom.DEBUGGER, InvokeFrom.SERVICE_API]) @pytest.mark.parametrize("rebind", [False, True]) @pytest.mark.parametrize("entry", ["factory", "engine-worker"]) -def test_node_creates_and_resumes_form_in_injected_database(form_databases, invoke_from, rebind, entry): +def test_node_creates_and_resumes_form_in_injected_database( + form_databases: FormDatabases, invoke_from: InvokeFrom, rebind: bool, entry: str +) -> None: sessions, global_sessions = form_databases if entry == "engine-worker": runner = _AppRunner( @@ -119,6 +131,7 @@ def test_node_creates_and_resumes_form_in_injected_database(form_databases, invo completed = list(node._run()) assert not any(isinstance(event, PauseRequestedEvent) for event in completed) + assert isinstance(completed[-1], StreamCompletedEvent) assert completed[-1].node_run_result.outputs["__action_id"].value == "approve" with sessions() as session: assert session.scalars(select(HumanInputForm)).one().id == form_id @@ -127,17 +140,19 @@ def test_node_creates_and_resumes_form_in_injected_database(form_databases, invo assert sessions.kw["bind"].pool.checkedout() == 0 -def test_human_input_node_requires_persistence_dependency(): +def test_human_input_node_requires_persistence_dependency() -> None: with pytest.raises(ValueError, match="execution dependencies are required for Human Input"): factory(None).create_node(node_config()) -def test_create_form_commit_failure_does_not_return_a_pause_or_persist_form(form_databases): +def test_create_form_commit_failure_does_not_return_a_pause_or_persist_form( + form_databases: FormDatabases, +) -> None: sessions, _ = form_databases node = factory(build_workflow_execution_dependencies(sessions)).create_node(node_config()) node.bind_execution_id("human-execution") - def fail_commit(_session): + def fail_commit(_session: Session) -> None: raise RuntimeError("commit failed") event.listen(sessions, "before_commit", fail_commit) @@ -152,11 +167,11 @@ def fail_commit(_session): assert session.scalar(select(HumanInputDelivery)) is None -def test_created_form_timeout_reaches_event_stream_in_same_database(form_databases): +def test_created_form_timeout_reaches_event_stream_in_same_database(form_databases: FormDatabases) -> None: from dataclasses import dataclass from graphon.graph import Graph - from graphon.graph_events import NodeRunHumanInputFormTimeoutEvent, NodeRunSucceededEvent + from graphon.graph_events import NodeRunHumanInputFormTimeoutEvent from services.workflow.execution.adapters.workflow_entry import iter_dify_graph_engine_events sessions, global_sessions = form_databases @@ -175,12 +190,13 @@ class NodeEvents: graph: Graph graph_runtime_state: GraphRuntimeState - def run(self): + def run(self) -> Iterator[object]: yield from node.run() events = list( iter_dify_graph_engine_events( - NodeEvents(Graph(root_node=node), node.graph_runtime_state), human_form_reader=runtime.human_form_reader + cast(GraphEngine, NodeEvents(Graph(root_node=node), node.graph_runtime_state)), + human_form_reader=runtime.human_form_reader, ) ) timeout = next(event for event in events if isinstance(event, NodeRunHumanInputFormTimeoutEvent)) diff --git a/api/tests/unit_tests/services/workflow/test_runtime_gateway.py b/api/tests/unit_tests/services/workflow/test_runtime_gateway.py index e54e1ef2982105..f1424693e58e28 100644 --- a/api/tests/unit_tests/services/workflow/test_runtime_gateway.py +++ b/api/tests/unit_tests/services/workflow/test_runtime_gateway.py @@ -2,8 +2,8 @@ import json from collections.abc import Callable, Generator, Iterable -from typing import Self, cast, override -from unittest.mock import Mock +from typing import NoReturn, Self, cast, override +from unittest.mock import Mock, create_autospec import pytest from flask import Flask @@ -42,6 +42,7 @@ from services.agent.workflow_publish_service import WorkflowAgentPublishService from services.app_generate_service import AppGenerateService from services.file_service import FileService +from services.tools.provider_queries import ToolProviderIcons from services.workflow import runtime_gateway as module from services.workflow.console_service import ( ConsoleWorkflowService, @@ -57,6 +58,7 @@ from services.workflow.variable_contracts import WorkflowExecutionVariables from services.workflow.variable_service import WorkflowVariableService from services.workflow_service import WorkflowService +from tests.unit_tests.config_override import apply_config_overrides from tests.unit_tests.model_factories import make_account, make_app, make_tenant, make_workflow from tests.unit_tests.workflow_execution import debug_lease, set_debug_deadline @@ -109,7 +111,7 @@ def events() -> Generator[str, None, None]: return events() - generator = Mock(spec=AppGenerateService) + generator = create_autospec(AppGenerateService, spec_set=True) generator.generate_workflow_stream.side_effect = generate gateway = WorkflowRuntimeGateway( runtime=build_workflow_execution_dependencies(factory), @@ -119,7 +121,7 @@ def events() -> Generator[str, None, None]: reservations=WorkflowDebugReservationRepository(factory), executions=WorkflowNodeExecutionRepository(factory), generator=generator, - graph_engine=Mock(spec=GraphEngineManager), + graph_engine=create_autospec(GraphEngineManager, instance=True, spec_set=True), file_access=DatabaseFileAccessController(), variables=workflow_variables, ) @@ -622,8 +624,11 @@ def test_real_stream_preparation_releases_reads_before_external_io( assert app is not None app.mode = mode app.max_active_requests = 2 - monkeypatch.setattr(dify_config, "DEPLOYMENT_EDITION", DeploymentEdition.CLOUD) - monkeypatch.setattr(dify_config, "PUBSUB_REDIS_CHANNEL_TYPE", "streams") + apply_config_overrides( + monkeypatch, + DEPLOYMENT_EDITION=DeploymentEdition.CLOUD, + PUBSUB_REDIS_CHANNEL_TYPE="streams", + ) monkeypatch.setattr(limiter_module.RateLimit, "_instance_dict", {}) observed: list[str] = [] @@ -741,7 +746,7 @@ def test_node_response_materializes_actor_extras_and_offload_flags( ) monkeypatch.setattr(gateway._workflows, "run_draft_workflow_node", lambda **_kwargs: execution) - def extras(_execution: WorkflowNodeExecutionModel, *, tool_providers) -> dict[str, object]: + def extras(_execution: WorkflowNodeExecutionModel, *, tool_providers: ToolProviderIcons) -> dict[str, object]: assert tool_providers is gateway._runtime.tool_providers assert all(session.closed for session in sessions) return {} @@ -888,15 +893,15 @@ def poller(*, draft_workflow: Workflow, **_kwargs: object) -> Poller: monkeypatch.setattr(module, "create_event_poller", poller) service = ConsoleWorkflowService( - conversion=Mock(spec=WorkflowConversion), + conversion=create_autospec(WorkflowConversion, instance=True, spec_set=True), agent_services=WorkflowAgentPublishService, - drafts=Mock(spec=WorkflowDraftService), - definitions=Mock(spec=WorkflowDefinitions), - lifecycle=Mock(spec=WorkflowDefinitionLifecycle), + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + definitions=create_autospec(WorkflowDefinitions, instance=True, spec_set=True), + lifecycle=create_autospec(WorkflowDefinitionLifecycle, instance=True, spec_set=True), runtime=gateway, - apps=Mock(spec=WorkflowAppLookup), - presence=Mock(spec=WorkflowPresence), - access=Mock(spec=WorkflowAccess), + apps=create_autospec(WorkflowAppLookup, instance=True, spec_set=True), + presence=create_autospec(WorkflowPresence, instance=True, spec_set=True), + access=create_autospec(WorkflowAccess, instance=True, spec_set=True), ) result = service.trigger(CONTEXT, "app-1", ["trigger"], single_node=True, select_all=False) assert isinstance(result, dict) @@ -924,8 +929,11 @@ def test_single_node_reads_offloaded_outputs_after_repository_session_closes( content = "complete output " * 1000 objects: dict[str, bytes] = {} reads: list[str] = [] - monkeypatch.setattr(dify_config, "WORKFLOW_VARIABLE_TRUNCATION_MAX_SIZE", 256) - monkeypatch.setattr(dify_config, "WORKFLOW_VARIABLE_TRUNCATION_STRING_LENGTH", 128) + apply_config_overrides( + monkeypatch, + WORKFLOW_VARIABLE_TRUNCATION_MAX_SIZE=256, + WORKFLOW_VARIABLE_TRUNCATION_STRING_LENGTH=128, + ) monkeypatch.setattr(storage, "save", lambda key, data: objects.__setitem__(key, data)) def load(key: str) -> bytes: @@ -1093,7 +1101,6 @@ def test_full_trigger_execution_preserves_revision_through_task_and_worker( *, workflow_variables: WorkflowExecutionVariables, ) -> None: - from configs import dify_config from core.app.apps import base_app_queue_manager from core.app.features.rate_limiting.rate_limit import RateLimit from core.ops.ops_trace_manager import TraceQueueManager @@ -1145,8 +1152,8 @@ def select_events(*, draft_workflow: Workflow, **kwargs: object) -> TriggerDebug monkeypatch.setattr(module, "create_event_poller", poller) monkeypatch.setattr(module, "select_trigger_debug_events", select_events) - monkeypatch.setattr(dify_config, "PUBSUB_REDIS_CHANNEL_TYPE", "streams") - limiter = Mock(spec=RateLimit) + apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE="streams") + limiter = create_autospec(RateLimit, instance=True, spec_set=True) limiter.generate.side_effect = lambda response, *_args: response def guardrails(*, action: Callable[[RateLimit, str], object], **_kwargs: object) -> object: @@ -1163,15 +1170,15 @@ def retrieve_events(*_args: object, on_subscribe: Callable[[], None]) -> Generat monkeypatch.setattr(WorkflowEventStream, "retrieve_events", retrieve_events) service = ConsoleWorkflowService( - conversion=Mock(spec=WorkflowConversion), + conversion=create_autospec(WorkflowConversion, instance=True, spec_set=True), agent_services=WorkflowAgentPublishService, - drafts=Mock(spec=WorkflowDraftService), - definitions=Mock(spec=WorkflowDefinitions), - lifecycle=Mock(spec=WorkflowDefinitionLifecycle), + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + definitions=create_autospec(WorkflowDefinitions, instance=True, spec_set=True), + lifecycle=create_autospec(WorkflowDefinitionLifecycle, instance=True, spec_set=True), runtime=gateway, - apps=Mock(spec=WorkflowAppLookup), - presence=Mock(spec=WorkflowPresence), - access=Mock(spec=WorkflowAccess), + apps=create_autospec(WorkflowAppLookup, instance=True, spec_set=True), + presence=create_autospec(WorkflowPresence, instance=True, spec_set=True), + access=create_autospec(WorkflowAccess, instance=True, spec_set=True), ) with Flask(__name__).app_context(): response = service.trigger(CONTEXT, "app-1", ["trigger"], single_node=False, select_all=select_all) @@ -1204,7 +1211,8 @@ def empty_response(*_args: object, **_kwargs: object) -> Generator[str, None, No yield from () monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", lambda *_args, **_kwargs: None) - monkeypatch.setattr(generator_module, "TraceQueueManager", Mock(return_value=Mock(spec=TraceQueueManager))) + trace_manager = create_autospec(TraceQueueManager, instance=True, spec_set=True) + monkeypatch.setattr(generator_module, "TraceQueueManager", lambda **_kwargs: trace_manager) monkeypatch.setattr(generator_module, "WorkflowAppRunner", GraphRunner) monkeypatch.setattr(WorkflowAppGenerator, "_handle_response", empty_response) monkeypatch.setattr(task_module, "_publish_streaming_response", lambda response, *_args, **_kwargs: list(response)) @@ -1293,7 +1301,7 @@ def setex(key: str, *_args: object) -> None: def response(*_args: object, **_kwargs: object) -> Generator[str, None, None]: yield from () - def unexpected_gateway_session(): + def unexpected_gateway_session() -> NoReturn: raise AssertionError("Single-node generation must use the repositories' sessions") monkeypatch.setattr(gateway, "_sessions", unexpected_gateway_session) @@ -1416,8 +1424,8 @@ def poll(self) -> TriggerDebugEvent: monkeypatch.setattr(module, "create_event_poller", lambda **_kwargs: Poller()) monkeypatch.setattr(module, "select_trigger_debug_events", lambda **_kwargs: Poller().poll()) - monkeypatch.setattr(dify_config, "PUBSUB_REDIS_CHANNEL_TYPE", "streams") - limiter = Mock(spec=RateLimit) + apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE="streams") + limiter = create_autospec(RateLimit, instance=True, spec_set=True) limiter.generate.side_effect = lambda response, *_args: response monkeypatch.setattr( AppGenerateService, "_run_with_guardrails", lambda *, action, **_kwargs: action(limiter, "request") @@ -1620,7 +1628,7 @@ def test_reserved_trigger_stream_owns_cleanup_until_enqueue( == [] ) monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", lambda **_kwargs: None) - monkeypatch.setattr(dify_config, "PUBSUB_REDIS_CHANNEL_TYPE", channel_type) + apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE=channel_type) def unexpected_timer(*_args: object, **_kwargs: object) -> None: pytest.fail("a reservation must not enqueue after the stream releases its references") @@ -1771,7 +1779,7 @@ def test_global_timeout_retires_paused_trigger_agent_after_last_execution( ) assert session.scalar(select(WorkflowAgentNodeBinding.agent_id)) == "agent-1" monkeypatch.setattr(timeout_module, "db", SimpleNamespace(engine=sqlite_session_factory.kw["bind"])) - monkeypatch.setattr(dify_config, "HUMAN_INPUT_GLOBAL_TIMEOUT_SECONDS", 3600) + apply_config_overrides(monkeypatch, HUMAN_INPUT_GLOBAL_TIMEOUT_SECONDS=3600) collector = Mock() monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", collector) @@ -1903,6 +1911,7 @@ def test_paused_debug_lease_is_protected_and_refreshed_atomically_on_resume( assert run is not None assert lease is not None assert run.status == WorkflowExecutionStatus.RUNNING + assert lease.expires_at is not None assert lease.expires_at > now assert reservations.expire(snapshot.execution_id, now) is None From f68f74655353fdab8f5d75bde5947e2a65d46b58 Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Fri, 9 Oct 2026 13:54:01 +0800 Subject: [PATCH 04/20] fix: keep intermediate activation callers and test fixtures valid --- api/.importlinter | 1 - .../annotation_reply/annotation_reply.py | 4 +- .../based_generate_task_pipeline.py | 18 +- api/core/workflow/node_factory.py | 6 +- .../sync_plugin_trigger_when_app_created.py | 34 ---- ...nc_workflow_schedule_when_app_published.py | 83 --------- ...oin_when_app_published_workflow_updated.py | 69 -------- ...ers_when_app_published_workflow_updated.py | 112 ------------ api/services/agent/composer_service.py | 17 ++ .../agent/legacy_retirement_service.py | 5 +- .../agent/legacy_workflow_publish_service.py | 49 +++--- api/services/agent/workspace_service.py | 16 ++ api/services/annotation_service.py | 53 +++++- api/services/knowledge/dataset_service.py | 5 + api/services/snippet_generate_service.py | 5 + .../workflow_draft_variable_service.py | 25 ++- .../controllers/console/app/test_app_apis.py | 4 + .../controllers/console/app/test_workflow.py | 123 ++++++++++--- .../test_based_generate_task_pipeline.py | 28 +++ .../unit_tests/core/tools/test_base_tool.py | 21 ++- .../core/tools/test_builtin_tools_extra.py | 161 +++++++++++++----- .../unit_tests/core/tools/test_custom_tool.py | 8 +- .../core/tools/test_dataset_retriever_tool.py | 10 +- .../unit_tests/core/tools/test_mcp_tool.py | 9 +- .../unit_tests/core/tools/test_plugin_tool.py | 5 +- .../unit_tests/core/tools/test_tool_engine.py | 19 ++- .../test_ext_application_services.py | 5 +- .../services/workflow/test_runtime_gateway.py | 10 +- .../tasks/test_resume_agent_app_task.py | 24 ++- api/tests/unit_tests/tools/test_api_tool.py | 33 ++-- api/tests/unit_tests/tools/test_mcp_tool.py | 39 ++--- 31 files changed, 521 insertions(+), 480 deletions(-) delete mode 100644 api/events/event_handlers/sync_plugin_trigger_when_app_created.py delete mode 100644 api/events/event_handlers/sync_workflow_schedule_when_app_published.py delete mode 100644 api/events/event_handlers/update_app_dataset_join_when_app_published_workflow_updated.py delete mode 100644 api/events/event_handlers/update_app_triggers_when_app_published_workflow_updated.py diff --git a/api/.importlinter b/api/.importlinter index 0019e69c82f463..7b47cf71c40b64 100644 --- a/api/.importlinter +++ b/api/.importlinter @@ -63,7 +63,6 @@ ignore_imports = core.app.apps.workflow.app_generator -> services.workflow_draft_variable_service core.app.apps.workflow_app_runner -> tasks.mail_human_input_delivery_task core.app.features.annotation_reply.annotation_reply -> services.annotation_service - core.app.features.annotation_reply.annotation_reply -> services.knowledge.dataset_service core.app.llm.quota -> services.credit_pool_service core.app.task_pipeline.message_cycle_manager -> services.annotation_service core.app.workflow.layers.persistence -> services.workflow.inspector_events diff --git a/api/core/app/features/annotation_reply/annotation_reply.py b/api/core/app/features/annotation_reply/annotation_reply.py index 454a239494256c..4cf21bed4d6c81 100644 --- a/api/core/app/features/annotation_reply/annotation_reply.py +++ b/api/core/app/features/annotation_reply/annotation_reply.py @@ -9,8 +9,8 @@ from models.dataset import Dataset from models.enums import CollectionBindingType, ConversationFromSource from models.model import AnnotationReplyEnabledConfig, App, Message, MessageAnnotation, load_annotation_reply_config +from repositories.knowledge.collection_binding_repository import DatasetCollectionBindingRepository from services.annotation_service import AppAnnotationService -from services.knowledge.dataset_service import DatasetCollectionBindingService logger = logging.getLogger(__name__) @@ -46,7 +46,7 @@ def query( embedding_provider_name = enabled_config["embedding_model"]["embedding_provider_name"] embedding_model_name = enabled_config["embedding_model"]["embedding_model_name"] - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( + dataset_collection_binding = DatasetCollectionBindingRepository.get_dataset_collection_binding( embedding_provider_name, embedding_model_name, session, CollectionBindingType.ANNOTATION ) diff --git a/api/core/app/task_pipeline/based_generate_task_pipeline.py b/api/core/app/task_pipeline/based_generate_task_pipeline.py index 347bd46b03e829..1ff89b92db82c7 100644 --- a/api/core/app/task_pipeline/based_generate_task_pipeline.py +++ b/api/core/app/task_pipeline/based_generate_task_pipeline.py @@ -1,6 +1,9 @@ import logging import time +from sqlalchemy import select +from sqlalchemy.orm import Session + from clients.agent_backend.errors import AgentBackendError from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.entities.app_invoke_entities import ( @@ -16,6 +19,8 @@ from core.errors.error import QuotaExceededError from core.moderation.output_moderation import ModerationRule, OutputModeration from graphon.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError +from models.enums import MessageStatus +from models.model import Message logger = logging.getLogger(__name__) @@ -42,7 +47,9 @@ def __init__( self.output_moderation_handler = self._init_output_moderation() self.stream = stream - def handle_error(self, *, event: QueueErrorEvent): + def handle_error( + self, *, event: QueueErrorEvent, session: Session | None = None, message_id: str = "" + ) -> Exception: logger.debug("error: %s", event.error) e = event.error err: Exception @@ -56,6 +63,15 @@ def handle_error(self, *, event: QueueErrorEvent): description = getattr(e, "description", None) err = Exception(description if description is not None else str(e)) + if not message_id or session is None: + return err + + message = session.scalar(select(Message).where(Message.id == message_id)) + if message is None: + return err + + message.status = MessageStatus.ERROR + message.error = self._error_to_desc(err) return err def _error_to_desc(self, e: Exception) -> str: diff --git a/api/core/workflow/node_factory.py b/api/core/workflow/node_factory.py index b4a5f5eb395bc5..c471dc3168f777 100644 --- a/api/core/workflow/node_factory.py +++ b/api/core/workflow/node_factory.py @@ -46,7 +46,7 @@ PluginAgentStrategyResolver, ) from core.workflow.nodes.agent.runtime_support import AgentRuntimeSupport -from core.workflow.nodes.agent_v2 import DifyAgentNode +from core.workflow.nodes.agent_v2.agent_node import DifyAgentNode from core.workflow.nodes.agent_v2.binding_resolver import WorkflowAgentBindingResolver from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data from core.workflow.nodes.agent_v2.output_adapter import WorkflowAgentOutputAdapter @@ -383,7 +383,9 @@ def __init__( self._agent_strategy_resolver = PluginAgentStrategyResolver() self._agent_strategy_presentation_provider = PluginAgentStrategyPresentationProvider() self._agent_runtime_support = AgentRuntimeSupport() - self._agent_message_transformer = AgentMessageTransformer() + self._agent_message_transformer = AgentMessageTransformer( + lambda name, default: self._tool_runtime.resolve_provider_icons(provider_name=name, default_icon=default) + ) def with_runtime_state(self, graph_runtime_state: "GraphRuntimeState") -> "DifyNodeFactory": return DifyNodeFactory( diff --git a/api/events/event_handlers/sync_plugin_trigger_when_app_created.py b/api/events/event_handlers/sync_plugin_trigger_when_app_created.py deleted file mode 100644 index cc6e851ad870c2..00000000000000 --- a/api/events/event_handlers/sync_plugin_trigger_when_app_created.py +++ /dev/null @@ -1,34 +0,0 @@ -import logging - -from events.app_event import app_draft_workflow_was_synced, app_published_workflow_was_updated -from models.model import App, AppMode -from models.workflow import Workflow -from services.trigger.trigger_service import TriggerService - -logger = logging.getLogger(__name__) - - -@app_draft_workflow_was_synced.connect -@app_published_workflow_was_updated.connect -def handle( - sender, - synced_draft_workflow: Workflow | None = None, - published_workflow: Workflow | None = None, - **kwargs, -): - """ - Sync plugin trigger relationships when a draft changes or is published. - - The published workflow must be reconciled as well because production trigger - dispatch relies on these relationships, while debug dispatch does not. - """ - app: App = sender - if app.mode != AppMode.WORKFLOW.value: - # only handle workflow app, chatflow is not supported yet - return - - workflow = published_workflow if published_workflow is not None else synced_draft_workflow - if workflow is None: - return - - TriggerService.sync_plugin_trigger_relationships(app, workflow) diff --git a/api/events/event_handlers/sync_workflow_schedule_when_app_published.py b/api/events/event_handlers/sync_workflow_schedule_when_app_published.py deleted file mode 100644 index 5f8fcd86170b7a..00000000000000 --- a/api/events/event_handlers/sync_workflow_schedule_when_app_published.py +++ /dev/null @@ -1,83 +0,0 @@ -import logging -from typing import cast - -from sqlalchemy import select -from sqlalchemy.orm import sessionmaker - -from core.workflow.nodes.trigger_schedule.entities import SchedulePlanUpdate -from events.app_event import app_published_workflow_was_updated -from extensions.ext_database import db -from models import AppMode, Workflow, WorkflowSchedulePlan -from services.trigger.schedule_service import ScheduleService - -logger = logging.getLogger(__name__) - - -@app_published_workflow_was_updated.connect -def handle(sender, **kwargs): - """ - Handle app published workflow update event to sync workflow_schedule_plans table. - - When a workflow is published, this handler will: - 1. Extract schedule trigger nodes from the workflow graph - 2. Compare with existing workflow_schedule_plans records - 3. Create/update/delete schedule plans as needed - """ - app = sender - if app.mode != AppMode.WORKFLOW.value: - return - - published_workflow = kwargs.get("published_workflow") - published_workflow = cast(Workflow, published_workflow) - - sync_schedule_from_workflow(tenant_id=app.tenant_id, app_id=app.id, workflow=published_workflow) - - -def sync_schedule_from_workflow(tenant_id: str, app_id: str, workflow: Workflow) -> WorkflowSchedulePlan | None: - """ - Sync schedule plan from workflow graph configuration. - - Args: - tenant_id: Tenant ID - app_id: App ID - workflow: Published workflow instance - - Returns: - Updated or created WorkflowSchedulePlan, or None if no schedule node - """ - with sessionmaker(db.engine).begin() as session: - schedule_config = ScheduleService.extract_schedule_config(workflow) - - existing_plan = session.scalar( - select(WorkflowSchedulePlan).where( - WorkflowSchedulePlan.tenant_id == tenant_id, - WorkflowSchedulePlan.app_id == app_id, - ) - ) - - if not schedule_config: - if existing_plan: - logger.info("No schedule node in workflow for app %s, removing schedule plan", app_id) - ScheduleService.delete_schedule(session=session, schedule_id=existing_plan.id) - return None - - if existing_plan: - updates = SchedulePlanUpdate( - node_id=schedule_config.node_id, - cron_expression=schedule_config.cron_expression, - timezone=schedule_config.timezone, - ) - updated_plan = ScheduleService.update_schedule( - session=session, - schedule_id=existing_plan.id, - updates=updates, - ) - return updated_plan - else: - new_plan = ScheduleService.create_schedule( - session=session, - tenant_id=tenant_id, - app_id=app_id, - config=schedule_config, - ) - return new_plan diff --git a/api/events/event_handlers/update_app_dataset_join_when_app_published_workflow_updated.py b/api/events/event_handlers/update_app_dataset_join_when_app_published_workflow_updated.py deleted file mode 100644 index 6769b94cde0093..00000000000000 --- a/api/events/event_handlers/update_app_dataset_join_when_app_published_workflow_updated.py +++ /dev/null @@ -1,69 +0,0 @@ -from typing import cast - -from sqlalchemy import delete, select - -from core.workflow.nodes.knowledge_retrieval.entities import KnowledgeRetrievalNodeData -from events.app_event import app_published_workflow_was_updated -from extensions.ext_database import db -from graphon.nodes import BuiltinNodeTypes -from models.dataset import AppDatasetJoin -from models.workflow import Workflow - - -@app_published_workflow_was_updated.connect -def handle(sender, **kwargs): - app = sender - published_workflow = kwargs.get("published_workflow") - published_workflow = cast(Workflow, published_workflow) - - dataset_ids = get_dataset_ids_from_workflow(published_workflow) - app_dataset_joins = db.session.scalars(select(AppDatasetJoin).where(AppDatasetJoin.app_id == app.id)).all() - - removed_dataset_ids: set[str] = set() - if not app_dataset_joins: - added_dataset_ids = dataset_ids - else: - old_dataset_ids: set[str] = set() - old_dataset_ids.update(app_dataset_join.dataset_id for app_dataset_join in app_dataset_joins) - - added_dataset_ids = dataset_ids - old_dataset_ids - removed_dataset_ids = old_dataset_ids - dataset_ids - - if removed_dataset_ids: - for dataset_id in removed_dataset_ids: - db.session.execute( - delete(AppDatasetJoin).where(AppDatasetJoin.app_id == app.id, AppDatasetJoin.dataset_id == dataset_id) - ) - - if added_dataset_ids: - for dataset_id in added_dataset_ids: - app_dataset_join = AppDatasetJoin(app_id=app.id, dataset_id=dataset_id) - db.session.add(app_dataset_join) - - db.session.commit() - - -def get_dataset_ids_from_workflow(published_workflow: Workflow) -> set[str]: - dataset_ids: set[str] = set() - graph = published_workflow.graph_dict - if not graph: - return dataset_ids - - nodes = graph.get("nodes", []) - - # fetch all knowledge retrieval nodes - knowledge_retrieval_nodes = [ - node for node in nodes if node.get("data", {}).get("type") == BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL - ] - - if not knowledge_retrieval_nodes: - return dataset_ids - - for node in knowledge_retrieval_nodes: - try: - node_data = KnowledgeRetrievalNodeData.model_validate(node.get("data", {})) - dataset_ids.update(dataset_id for dataset_id in node_data.dataset_ids) - except Exception: - continue - - return dataset_ids diff --git a/api/events/event_handlers/update_app_triggers_when_app_published_workflow_updated.py b/api/events/event_handlers/update_app_triggers_when_app_published_workflow_updated.py deleted file mode 100644 index d55fe262fb25a3..00000000000000 --- a/api/events/event_handlers/update_app_triggers_when_app_published_workflow_updated.py +++ /dev/null @@ -1,112 +0,0 @@ -from typing import cast - -from sqlalchemy import select -from sqlalchemy.orm import sessionmaker - -from core.trigger.constants import TRIGGER_NODE_TYPES -from events.app_event import app_published_workflow_was_updated -from extensions.ext_database import db -from models import AppMode -from models.enums import AppTriggerStatus -from models.trigger import AppTrigger -from models.workflow import Workflow - - -@app_published_workflow_was_updated.connect -def handle(sender, **kwargs): - """ - Handle app published workflow update event to sync app_triggers table. - - When a workflow is published, this handler will: - 1. Extract trigger nodes from the workflow graph - 2. Compare with existing app_triggers records - 3. Add new triggers and remove obsolete ones - """ - app = sender - if app.mode != AppMode.WORKFLOW.value: - return - - published_workflow = kwargs.get("published_workflow") - published_workflow = cast(Workflow, published_workflow) - # Extract trigger info from workflow - trigger_infos = get_trigger_infos_from_workflow(published_workflow) - - with sessionmaker(db.engine).begin() as session: - # Get existing app triggers - existing_triggers = ( - session.execute( - select(AppTrigger).where(AppTrigger.tenant_id == app.tenant_id, AppTrigger.app_id == app.id) - ) - .scalars() - .all() - ) - - # Convert existing triggers to dict for easy lookup - existing_triggers_map = {trigger.node_id: trigger for trigger in existing_triggers} - - # Get current and new node IDs - existing_node_ids = set(existing_triggers_map.keys()) - new_node_ids = {info["node_id"] for info in trigger_infos} - - # Calculate changes - added_node_ids = new_node_ids - existing_node_ids - removed_node_ids = existing_node_ids - new_node_ids - - # Remove obsolete triggers - for node_id in removed_node_ids: - session.delete(existing_triggers_map[node_id]) - - for trigger_info in trigger_infos: - node_id = trigger_info["node_id"] - - if node_id in added_node_ids: - # Create new trigger - app_trigger = AppTrigger( - tenant_id=app.tenant_id, - app_id=app.id, - trigger_type=trigger_info["node_type"], - title=trigger_info["node_title"], - node_id=node_id, - provider_name=trigger_info.get("node_provider_name", ""), - status=AppTriggerStatus.ENABLED, - ) - session.add(app_trigger) - elif node_id in existing_node_ids: - # Update existing trigger if needed - existing_trigger = existing_triggers_map[node_id] - new_title = trigger_info["node_title"] - if new_title and existing_trigger.title != new_title: - existing_trigger.title = new_title - session.add(existing_trigger) - - -def get_trigger_infos_from_workflow(published_workflow: Workflow) -> list[dict]: - """ - Extract trigger node information from the workflow graph. - - Returns: - List of trigger info dictionaries containing: - - node_type: The type of the trigger node ('trigger-webhook', 'trigger-schedule', 'trigger-plugin') - - node_id: The node ID in the workflow - - node_title: The title of the node - - node_provider_name: The name of the node's provider, only for plugin - """ - graph = published_workflow.graph_dict - if not graph: - return [] - - nodes = graph.get("nodes", []) - trigger_types = TRIGGER_NODE_TYPES - - trigger_infos = [ - { - "node_type": node.get("data", {}).get("type"), - "node_id": node.get("id"), - "node_title": node.get("data", {}).get("title"), - "node_provider_name": node.get("data", {}).get("provider_name"), - } - for node in nodes - if node.get("data", {}).get("type") in trigger_types - ] - - return trigger_infos diff --git a/api/services/agent/composer_service.py b/api/services/agent/composer_service.py index 877300ff15de39..93f79ee50094ee 100644 --- a/api/services/agent/composer_service.py +++ b/api/services/agent/composer_service.py @@ -1985,6 +1985,23 @@ def _require_agent_app_agent(cls, *, session: Session, tenant_id: str, app_id: s raise AgentNotFoundError() return agent + @classmethod + def get_or_create_normal_agent_draft( + cls, + *, + session: Session, + tenant_id: str, + agent: Agent, + created_by: str | None, + ) -> AgentConfigDraft: + """Resolve the normal draft for legacy Agent App execution callers.""" + return AgentConfigRepository.normal_draft( + session=session, + tenant_id=tenant_id, + agent=agent, + created_by=created_by, + ) + @classmethod def _save_agent_draft( cls, diff --git a/api/services/agent/legacy_retirement_service.py b/api/services/agent/legacy_retirement_service.py index c800a8ea51bd58..ef3b67e5862a89 100644 --- a/api/services/agent/legacy_retirement_service.py +++ b/api/services/agent/legacy_retirement_service.py @@ -21,7 +21,7 @@ ) from models.model import App, AppMode from models.workflow import Workflow -from services.agent.home_snapshot_service import AgentHomeSnapshotService +from repositories.agent.home_snapshot_repository import AgentHomeSnapshotRepository from services.agent.workspace_service import AgentWorkspaceService from tasks.collect_agent_resources_task import enqueue_agent_resource_collection from tasks.remove_app_and_related_data_task import remove_app_and_related_data_task @@ -105,8 +105,7 @@ def retire_unowned( ) retired_bindings.append(binding.id) retired_snapshots.extend( - AgentHomeSnapshotService.retire_all_for_agent( - session=session, + AgentHomeSnapshotRepository(session).retire_all_for_agent( tenant_id=tenant_id, agent_id=agent_id, ) diff --git a/api/services/agent/legacy_workflow_publish_service.py b/api/services/agent/legacy_workflow_publish_service.py index 358b49e80a18dc..8060a2a8dfd2c7 100644 --- a/api/services/agent/legacy_workflow_publish_service.py +++ b/api/services/agent/legacy_workflow_publish_service.py @@ -25,6 +25,7 @@ WorkflowPreviousNodeOutputRef, ) from models.workflow import Workflow +from repositories.agent.workflow_binding_repository import WorkflowAgentBindingRepository, workflow_binding_scope from services.agent.composer_validator import ComposerConfigValidator from services.agent.prompt_mentions import ( extract_workflow_node_output_selectors, @@ -440,40 +441,40 @@ def _clone_inline_graph_binding_for_node( source_snapshot_id: str, account_id: str, ) -> tuple[Agent, str]: - source_agent = session.scalar( - select(Agent) - .where( - Agent.tenant_id == draft_workflow.tenant_id, - Agent.id == source_agent_id, - Agent.scope == AgentScope.WORKFLOW_ONLY, - Agent.status == AgentStatus.ACTIVE, - ) - .limit(1) - ) - if source_agent is None: + binding_repository = WorkflowAgentBindingRepository(session) + source_agent = binding_repository.get_agent(draft_workflow.tenant_id, source_agent_id) + if ( + source_agent is None + or source_agent.scope != AgentScope.WORKFLOW_ONLY + or source_agent.status != AgentStatus.ACTIVE + ): raise ValueError(f"Workflow Agent node {node_id} references an unavailable inline agent.") - source_snapshot = session.scalar( - select(AgentConfigSnapshot) - .where( - AgentConfigSnapshot.tenant_id == draft_workflow.tenant_id, - AgentConfigSnapshot.agent_id == source_agent.id, - AgentConfigSnapshot.id == source_snapshot_id, - ) - .limit(1) + source_snapshot = binding_repository.get_snapshot( + draft_workflow.tenant_id, + source_agent.id, + source_snapshot_id, ) if source_snapshot is None: raise ValueError(f"Workflow Agent node {node_id} references a missing inline agent config snapshot.") - from services.agent.dsl_service import AgentDslService - - agent, snapshot = AgentDslService(session).clone_inline_binding_for_node( - workflow=draft_workflow, + cloned_agent, snapshot_id = binding_repository.clone( + workflow=workflow_binding_scope(draft_workflow), node_id=node_id, source_agent=source_agent, source_snapshot=source_snapshot, account_id=account_id, ) - return agent, snapshot.id + agent = session.scalar( + select(Agent) + .where( + Agent.tenant_id == draft_workflow.tenant_id, + Agent.id == cloned_agent.id, + ) + .limit(1) + ) + if agent is None: + raise RuntimeError(f"Workflow Agent clone for node {node_id} was not persisted.") + return agent, snapshot_id @classmethod def _resolve_roster_agent_graph_binding( diff --git a/api/services/agent/workspace_service.py b/api/services/agent/workspace_service.py index e748b4f18ed505..322c46a0921090 100644 --- a/api/services/agent/workspace_service.py +++ b/api/services/agent/workspace_service.py @@ -364,6 +364,22 @@ def collect_retired_workspace(cls, *, tenant_id: str, workspace_id: str) -> None ) session.commit() + @staticmethod + def validate_binding_generation( + binding: AgentWorkspaceBinding, + *, + base_home_snapshot_id: str | None, + agent_config_version_id: str, + agent_config_version_kind: AgentConfigVersionKind, + ) -> None: + """Validate a legacy execution caller against the repository-owned generation policy.""" + AgentWorkspaceRepository.validate_binding_generation( + binding, + base_home_snapshot_id=base_home_snapshot_id, + agent_config_version_id=agent_config_version_id, + agent_config_version_kind=agent_config_version_kind, + ) + @staticmethod def _client() -> Client: base_url = dify_config.AGENT_BACKEND_BASE_URL diff --git a/api/services/annotation_service.py b/api/services/annotation_service.py index 3b66c85c345c22..451efd5191fb52 100644 --- a/api/services/annotation_service.py +++ b/api/services/annotation_service.py @@ -3,7 +3,7 @@ from typing import TypedDict import pandas as pd -from sqlalchemy import delete, or_, select +from sqlalchemy import delete, or_, select, update from sqlalchemy.orm import Session from werkzeug.datastructures import FileStorage @@ -13,6 +13,7 @@ from libs.datetime_utils import naive_utc_now from libs.login import current_account_with_tenant from libs.pagination import paginate_query +from models.account import Account from models.dataset import DatasetCollectionBinding from models.model import App, AppAnnotationHitHistory, AppAnnotationSetting, Message, MessageAnnotation from services.annotation.errors import AnnotationResourceNotFoundError @@ -589,6 +590,56 @@ def get_annotation_hit_histories(cls, annotation_ref: AnnotationRef, page, limit annotation_hit_histories = paginate_query(stmt, session=session, page=page, per_page=limit, max_per_page=100) return annotation_hit_histories.items, annotation_hit_histories.total or 0 + @classmethod + def get_annotation_by_id(cls, annotation_id: str, session: Session) -> MessageAnnotation | None: + return session.get(MessageAnnotation, annotation_id) + + @classmethod + def get_annotation_reply_by_id( + cls, annotation_id: str, session: Session + ) -> tuple[MessageAnnotation, str | None] | None: + """Read a reply and its author together, retaining annotations whose account was deleted.""" + row = session.execute( + select(MessageAnnotation, Account.name) + .outerjoin(Account, Account.id == MessageAnnotation.account_id) + .where(MessageAnnotation.id == annotation_id) + ).one_or_none() + return (row[0], row[1]) if row is not None else None + + @classmethod + def add_annotation_history( + cls, + annotation_id: str, + app_id: str, + annotation_question: str, + annotation_content: str, + query: str, + user_id: str, + message_id: str, + from_source: str, + score: float, + session: Session, + ) -> None: + session.execute( + update(MessageAnnotation) + .where(MessageAnnotation.id == annotation_id) + .values(hit_count=MessageAnnotation.hit_count + 1) + ) + session.add( + AppAnnotationHitHistory( + annotation_id=annotation_id, + app_id=app_id, + account_id=user_id, + question=query, + source=from_source, + score=score, + message_id=message_id, + annotation_question=annotation_question, + annotation_content=annotation_content, + ) + ) + session.flush() + @classmethod def get_app_annotation_setting_by_app_id( cls, app_id: str, session: Session diff --git a/api/services/knowledge/dataset_service.py b/api/services/knowledge/dataset_service.py index 0ad5652e0aed20..8c6934a346d42a 100644 --- a/api/services/knowledge/dataset_service.py +++ b/api/services/knowledge/dataset_service.py @@ -1279,6 +1279,11 @@ class DocumentService: } DOCUMENT_BATCH_DOWNLOAD_ZIP_FILENAME_EXTENSION = ".zip" + @staticmethod + def get_documents_position(dataset_id: str, session: Session) -> int: + """Return the next document position for legacy pipeline execution callers.""" + return next_document_position(dataset_id, session) + @classmethod def normalize_display_status(cls, status: str | None) -> str | None: if not status: diff --git a/api/services/snippet_generate_service.py b/api/services/snippet_generate_service.py index c445cf575e97f8..84a278a1e47f5c 100644 --- a/api/services/snippet_generate_service.py +++ b/api/services/snippet_generate_service.py @@ -227,6 +227,11 @@ def _ensure_start_node(cls, workflow: Workflow, snippet: CustomizedSnippet) -> W workflow.graph = json.dumps(modified_graph) return workflow + @classmethod + def ensure_start_node_for_worker(cls, workflow: Workflow, snippet: CustomizedSnippet) -> Workflow: + """Inject the virtual Start node after a worker reloads a snippet workflow.""" + return cls._ensure_start_node(workflow, snippet) + @classmethod def _inject_virtual_start_node( cls, diff --git a/api/services/workflow_draft_variable_service.py b/api/services/workflow_draft_variable_service.py index e0a0c18d5c6fbe..c9c26e55aa5458 100644 --- a/api/services/workflow_draft_variable_service.py +++ b/api/services/workflow_draft_variable_service.py @@ -43,7 +43,7 @@ from libs.datetime_utils import naive_utc_now from libs.uuid_utils import uuidv7 from models import Account, App, Conversation -from models.enums import ConversationFromSource, DraftVariableType +from models.enums import ConversationFromSource, DraftVariableType, ExecutionOffLoadType from models.utils.file_input_compat import build_file_from_stored_mapping from models.workflow import Workflow, WorkflowDraftVariable, WorkflowDraftVariableFile, is_system_variable_editable from repositories.factory import DifyAPIRepositoryFactory @@ -400,8 +400,11 @@ def _reset_node_var_or_sys_var( logger.warning("draft variable has no node_execution_id, id=%s, name=%s", variable.id, variable.name) return None - node_exec = self._api_node_execution_repo.get_execution_by_id(variable.node_execution_id) - if node_exec is None: + node_exec = self._api_node_execution_repo.get_execution_by_id( + variable.node_execution_id, + tenant_id=workflow.tenant_id, + ) + if node_exec is None or node_exec.app_id != workflow.app_id: logger.warning( "Node exectution not found for draft variable, id=%s, name=%s, node_execution_id=%s", variable.id, @@ -412,7 +415,21 @@ def _reset_node_var_or_sys_var( self._session.flush() return None - outputs_dict = node_exec.load_full_outputs(self._session, storage) or {} + output_offload = next( + (item for item in node_exec.offload_data if item.type_ == ExecutionOffLoadType.OUTPUTS), + None, + ) + if output_offload is None: + outputs_dict = node_exec.outputs_dict or {} + else: + if ( + output_offload.tenant_id != node_exec.tenant_id + or output_offload.app_id != node_exec.app_id + or output_offload.file is None + or output_offload.file.tenant_id != node_exec.tenant_id + ): + raise VariableResetError(f"Execution output file not found, execution_id={node_exec.id}") + outputs_dict = json.loads(storage.load(output_offload.file.key)) or {} # a sentinel value used to check the absent of the output variable key. absent = object() diff --git a/api/tests/unit_tests/controllers/console/app/test_app_apis.py b/api/tests/unit_tests/controllers/console/app/test_app_apis.py index 9e02cebcfee5ee..fb684bff7a058d 100644 --- a/api/tests/unit_tests/controllers/console/app/test_app_apis.py +++ b/api/tests/unit_tests/controllers/console/app/test_app_apis.py @@ -41,6 +41,9 @@ from controllers.console.app import ( workflow_app_log as workflow_app_log_module, ) +from controllers.console.app import ( + workflow_draft_variable as workflow_draft_variable_module, +) from controllers.console.app import ( workflow_statistic as workflow_statistic_module, ) @@ -63,6 +66,7 @@ from controllers.console.app.workflow_app_log import WorkflowAppLogQuery from controllers.console.app.workflow_draft_variable import ( EnvironmentVariableUpdatePayload, + WorkflowDraftVariableListQuery, WorkflowDraftVariableUpdatePayload, ) from controllers.console.app.workflow_statistic import WorkflowStatisticQuery diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow.py b/api/tests/unit_tests/controllers/console/app/test_workflow.py index 97f4a83ba99a74..f3b9c06aa201cb 100644 --- a/api/tests/unit_tests/controllers/console/app/test_workflow.py +++ b/api/tests/unit_tests/controllers/console/app/test_workflow.py @@ -1,6 +1,7 @@ """Transport contracts; workflow behavior is tested at the use-case and adapter boundaries.""" import inspect +from collections.abc import Callable from dataclasses import dataclass from datetime import UTC, datetime from unittest.mock import MagicMock, Mock, create_autospec @@ -39,9 +40,14 @@ def workflow_use_cases(monkeypatch: pytest.MonkeyPatch) -> MagicMock: return use_cases -def invoke(resource: type[Resource], method: str = "post", *args: object, **kwargs: object) -> object: +def invoke( + resource: type[Resource], + handler: Callable[..., object], + *args: object, + **kwargs: object, +) -> object: api = resource() - return inspect.unwrap(getattr(api, method))(api, *args, request_context=CONTEXT, app_id=APP_ID, **kwargs) + return inspect.unwrap(handler)(api, *args, request_context=CONTEXT, app_id=APP_ID, **kwargs) @pytest.mark.parametrize("content_type", ["application/json", "text/plain"]) @@ -60,7 +66,7 @@ def test_sync_parses_transport_and_preserves_patch(app: Flask, workflows: Mock, import json with app.test_request_context(method="POST", data=json.dumps(body), content_type=content_type): - response = invoke(controller.DraftWorkflowApi) + response = invoke(controller.DraftWorkflowApi, controller.DraftWorkflowApi.post) assert response == {"result": "success", "hash": "new-hash", "updated_at": 1704067200} context, app_id, command = workflows.sync.call_args.args assert (context, app_id) == (CONTEXT, str(APP_ID)) @@ -82,10 +88,13 @@ def test_sync_invalid_input_does_not_dispatch( with app.test_request_context(method="POST", data=data, content_type=content_type): if status == 415: with pytest.raises(HTTPException) as error: - invoke(controller.DraftWorkflowApi) + invoke(controller.DraftWorkflowApi, controller.DraftWorkflowApi.post) assert error.value.code == status else: - assert invoke(controller.DraftWorkflowApi) == ({"message": "Invalid JSON data"}, status) + assert invoke(controller.DraftWorkflowApi, controller.DraftWorkflowApi.post) == ( + {"message": "Invalid JSON data"}, + status, + ) workflows.sync.assert_not_called() @@ -96,7 +105,7 @@ def test_sync_invalid_input_does_not_dispatch( def test_sync_domain_errors(app: Flask, workflows: Mock, failure: Exception, error_type: type[Exception]) -> None: workflows.sync.side_effect = failure with app.test_request_context(method="POST", json={"graph": {}, "features": {}}), pytest.raises(error_type): - invoke(controller.DraftWorkflowApi) + invoke(controller.DraftWorkflowApi, controller.DraftWorkflowApi.post) @pytest.mark.parametrize( @@ -123,7 +132,11 @@ def test_sync_rejects_removed_full_environment_payload() -> None: @pytest.mark.parametrize("warning", [None, "branch may be skipped"]) def test_publish_serializes_advisory(workflows: Mock, warning: str | None) -> None: workflows.publish.return_value = (WorkflowPublication(datetime(2024, 1, 1, tzinfo=UTC), "{}"), warning) - response = invoke(controller.PublishedWorkflowApi, "post", controller.PublishWorkflowPayload()) + response = invoke( + controller.PublishedWorkflowApi, + controller.PublishedWorkflowApi.post, + controller.PublishWorkflowPayload(), + ) assert isinstance(response, dict) assert response["created_at"] == 1704067200 assert response.get("warning") == warning @@ -142,7 +155,11 @@ def test_publish_serializes_advisory(workflows: Mock, warning: str | None) -> No def test_restore_error_contract(workflows: Mock, failure: Exception, error_type: type[Exception]) -> None: workflows.restore.side_effect = failure with pytest.raises(error_type): - invoke(controller.DraftWorkflowRestoreApi, workflow_id="version") + invoke( + controller.DraftWorkflowRestoreApi, + controller.DraftWorkflowRestoreApi.post, + workflow_id="version", + ) @pytest.mark.parametrize( @@ -160,7 +177,11 @@ def test_restore_domain_error_http_response(workflows: Mock, failure: Exception, class Restore(Resource): def post(self) -> object: - return invoke(controller.DraftWorkflowRestoreApi, workflow_id="version") + return invoke( + controller.DraftWorkflowRestoreApi, + controller.DraftWorkflowRestoreApi.post, + workflow_id="version", + ) api.add_resource(Restore, "/restore") response = app.test_client().post("/restore") @@ -170,7 +191,11 @@ def post(self) -> object: def test_restore_serializes_change(workflows: Mock) -> None: workflows.restore.return_value = WorkflowChange("hash", datetime(2024, 1, 1, tzinfo=UTC)) - assert invoke(controller.DraftWorkflowRestoreApi, workflow_id="version") == { + assert invoke( + controller.DraftWorkflowRestoreApi, + controller.DraftWorkflowRestoreApi.post, + workflow_id="version", + ) == { "result": "success", "hash": "hash", "updated_at": 1704067200, @@ -178,37 +203,69 @@ def test_restore_serializes_change(workflows: Mock) -> None: @pytest.mark.parametrize( - ("resource", "payload", "kwargs"), + ("resource", "handler", "payload", "kwargs"), [ - (controller.DraftWorkflowTriggerRunApi, controller.DraftWorkflowTriggerRunPayload(node_id="node"), {}), - (controller.DraftWorkflowTriggerRunAllApi, controller.DraftWorkflowTriggerRunAllPayload(node_ids=["node"]), {}), - (controller.DraftWorkflowTriggerNodeApi, None, {"node_id": "node"}), + ( + controller.DraftWorkflowTriggerRunApi, + controller.DraftWorkflowTriggerRunApi.post, + controller.DraftWorkflowTriggerRunPayload(node_id="node"), + {}, + ), + ( + controller.DraftWorkflowTriggerRunAllApi, + controller.DraftWorkflowTriggerRunAllApi.post, + controller.DraftWorkflowTriggerRunAllPayload(node_ids=["node"]), + {}, + ), + ( + controller.DraftWorkflowTriggerNodeApi, + controller.DraftWorkflowTriggerNodeApi.post, + None, + {"node_id": "node"}, + ), ], ) def test_trigger_waiting_and_error_contract( - workflows: Mock, resource: type[Resource], payload: BaseModel | None, kwargs: dict[str, str] + workflows: Mock, + resource: type[Resource], + handler: Callable[..., object], + payload: BaseModel | None, + kwargs: dict[str, str], ) -> None: args = (payload,) if payload is not None else () workflows.trigger.return_value = None - assert invoke(resource, "post", *args, **kwargs) == {"status": "waiting", "retry_in": 2000} + assert invoke(resource, handler, *args, **kwargs) == {"status": "waiting", "retry_in": 2000} workflows.trigger.side_effect = WorkflowTriggerError("plugin unavailable") - assert invoke(resource, "post", *args, **kwargs) == ({"status": "error", "error": "plugin unavailable"}, 400) + assert invoke(resource, handler, *args, **kwargs) == ({"status": "error", "error": "plugin unavailable"}, 400) @pytest.mark.parametrize( - ("resource", "payload"), + ("resource", "handler", "payload"), [ - (controller.DraftWorkflowRunApi, controller.DraftWorkflowRunPayload(inputs={})), - (controller.AdvancedChatDraftWorkflowRunApi, controller.AdvancedChatWorkflowRunPayload(inputs={})), + ( + controller.DraftWorkflowRunApi, + controller.DraftWorkflowRunApi.post, + controller.DraftWorkflowRunPayload(inputs={}), + ), + ( + controller.AdvancedChatDraftWorkflowRunApi, + controller.AdvancedChatDraftWorkflowRunApi.post, + controller.AdvancedChatWorkflowRunPayload(inputs={}), + ), ], ) def test_debug_run_passes_external_trace_and_stream( - app: Flask, workflows: Mock, monkeypatch: pytest.MonkeyPatch, resource: type[Resource], payload: BaseModel + app: Flask, + workflows: Mock, + monkeypatch: pytest.MonkeyPatch, + resource: type[Resource], + handler: Callable[..., object], + payload: BaseModel, ) -> None: workflows.generate.return_value = {"task_id": "task"} monkeypatch.setattr(controller, "get_external_trace_id", lambda request: "external-trace") with app.test_request_context(method="POST", json={}): - response = invoke(resource, "post", payload) + response = invoke(resource, handler, payload) assert isinstance(response, Response) assert response.get_json() == {"task_id": "task"} assert workflows.generate.call_args.args[2]["external_trace_id"] == "external-trace" @@ -217,12 +274,24 @@ def test_debug_run_passes_external_trace_and_stream( def test_convert_delegates_permissions_result(workflows: Mock) -> None: result = {"new_app_id": "new-app", "permission_keys": ["app.acl.edit"]} workflows.convert.return_value = result - assert invoke(controller.ConvertToWorkflowApi, "post", controller.ConvertToWorkflowPayload(name="Copy")) == result + assert ( + invoke( + controller.ConvertToWorkflowApi, + controller.ConvertToWorkflowApi.post, + controller.ConvertToWorkflowPayload(name="Copy"), + ) + == result + ) workflows.convert.assert_called_once_with(CONTEXT, str(APP_ID), {"name": "Copy"}) def test_update_empty_payload_does_not_dispatch(workflows: Mock) -> None: - assert invoke(controller.WorkflowByIdApi, "patch", controller.WorkflowUpdatePayload(), workflow_id="version") == ( + assert invoke( + controller.WorkflowByIdApi, + controller.WorkflowByIdApi.patch, + controller.WorkflowUpdatePayload(), + workflow_id="version", + ) == ( {"message": "No valid fields to update"}, 400, ) @@ -230,7 +299,11 @@ def test_update_empty_payload_does_not_dispatch(workflows: Mock) -> None: def test_delete_has_no_response_body(workflows: Mock) -> None: - assert invoke(controller.WorkflowByIdApi, "delete", workflow_id="version") == (None, 204) + assert invoke( + controller.WorkflowByIdApi, + controller.WorkflowByIdApi.delete, + workflow_id="version", + ) == (None, 204) workflows.delete.assert_called_once_with(CONTEXT, WorkflowOwner(str(APP_ID)), "version") diff --git a/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py b/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py index 30af20fa6318e9..100149afe6607a 100644 --- a/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py +++ b/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py @@ -97,6 +97,34 @@ def test_handle_error_preserves_agent_backend_run_failed_error(self, pipeline): assert "Knowledge retrieval failed" in str(err) assert "agent_run_id=run-1" in str(err) + @pytest.mark.parametrize("sqlite_session", [(Message,)], indirect=True) + def test_handle_error_updates_message_when_found(self, pipeline, sqlite_session: Session): + event = QueueErrorEvent(error=ValueError("oops")) + _persist_message(sqlite_session, message_id="msg-1") + + err = pipeline.handle_error(event=event, session=sqlite_session, message_id="msg-1") + + assert err is event.error + sqlite_session.flush() + sqlite_session.expire_all() + updated_message = sqlite_session.get(Message, "msg-1") + assert updated_message is not None + assert updated_message.status == MessageStatus.ERROR + assert updated_message.error == "oops" + + @pytest.mark.parametrize("sqlite_session", [(Message,)], indirect=True) + def test_handle_error_leaves_other_messages_unchanged(self, pipeline, sqlite_session: Session): + event = QueueErrorEvent(error=ValueError("oops")) + _persist_message(sqlite_session, message_id="other-message") + + err = pipeline.handle_error(event=event, session=sqlite_session, message_id="missing-message") + + assert err is event.error + untouched_message = sqlite_session.get(Message, "other-message") + assert untouched_message is not None + assert untouched_message.status == MessageStatus.NORMAL + assert untouched_message.error is None + def test_error_to_stream_response_and_ping(self, pipeline): error_response = pipeline.error_to_stream_response(ValueError("boom")) ping_response = pipeline.ping_stream_response() diff --git a/api/tests/unit_tests/core/tools/test_base_tool.py b/api/tests/unit_tests/core/tools/test_base_tool.py index b46b763ee6ca44..1445d4fb2b23f1 100644 --- a/api/tests/unit_tests/core/tools/test_base_tool.py +++ b/api/tests/unit_tests/core/tools/test_base_tool.py @@ -5,6 +5,7 @@ from typing import Any, cast import pytest +from sqlalchemy.orm import Session from core.app.entities.app_invoke_entities import InvokeFrom from core.tools.__base.tool import Tool @@ -51,6 +52,7 @@ def tool_provider_type(self) -> ToolProviderType: def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, @@ -92,7 +94,7 @@ def _build_tool(runtime: ToolRuntime | None = None) -> DummyTool: return DummyTool(entity=entity, runtime=runtime) -def test_invoke_supports_single_message_and_parameter_casting(): +def test_invoke_supports_single_message_and_parameter_casting(unbound_session: Session): runtime = ToolRuntime( tenant_id="tenant-1", invoke_from=InvokeFrom.DEBUGGER, @@ -110,6 +112,7 @@ def test_invoke_supports_single_message_and_parameter_casting(): messages = list( tool.invoke( + session=unbound_session, user_id="user-1", tool_parameters={"age": "18", "raw": "keep"}, conversation_id="conv-1", @@ -129,7 +132,7 @@ def test_invoke_supports_single_message_and_parameter_casting(): } -def test_invoke_preserves_multiple_select_values(): +def test_invoke_preserves_multiple_select_values(unbound_session: Session): tool = _build_tool() parameter = ToolParameter.get_simple_instance( name="choice", @@ -141,18 +144,18 @@ def test_invoke_preserves_multiple_select_values(): parameter.multiple = True tool.entity.parameters = [parameter] - list(tool.invoke(user_id="user-1", tool_parameters={"choice": ["a", "b"]})) + list(tool.invoke(session=unbound_session, user_id="user-1", tool_parameters={"choice": ["a", "b"]})) assert tool.last_invocation is not None assert tool.last_invocation["tool_parameters"] == {"choice": ["a", "b"]} with pytest.raises(ValueError, match="must be a list"): - tool.invoke(user_id="user-1", tool_parameters={"choice": "a"}) + tool.invoke(session=unbound_session, user_id="user-1", tool_parameters={"choice": "a"}) -def test_invoke_supports_list_and_generator_results(): +def test_invoke_supports_list_and_generator_results(unbound_session: Session): tool = _build_tool() tool.result = [tool.create_text_message("a"), tool.create_text_message("b")] - list_messages = list(tool.invoke(user_id="user-1", tool_parameters={})) + list_messages = list(tool.invoke(session=unbound_session, user_id="user-1", tool_parameters={})) assert [msg.message.text for msg in list_messages] == ["a", "b"] def _message_generator() -> Generator[ToolInvokeMessage, None, None]: @@ -160,7 +163,7 @@ def _message_generator() -> Generator[ToolInvokeMessage, None, None]: yield tool.create_text_message("g2") tool.result = _message_generator() - generated_messages = list(tool.invoke(user_id="user-2", tool_parameters={})) + generated_messages = list(tool.invoke(session=unbound_session, user_id="user-2", tool_parameters={})) assert [msg.message.text for msg in generated_messages] == ["g1", "g2"] @@ -369,6 +372,6 @@ def test_message_factory_helpers(): assert variable_message.message.stream is False -def test_base_abstract_invoke_placeholder_returns_none(): +def test_base_abstract_invoke_placeholder_returns_none(unbound_session: Session): tool = _build_tool() - assert Tool._invoke(tool, user_id="u", tool_parameters={}) is None + assert Tool._invoke(tool, session=unbound_session, user_id="u", tool_parameters={}) is None diff --git a/api/tests/unit_tests/core/tools/test_builtin_tools_extra.py b/api/tests/unit_tests/core/tools/test_builtin_tools_extra.py index f1c5b5f1f0f127..2a477ae75482db 100644 --- a/api/tests/unit_tests/core/tools/test_builtin_tools_extra.py +++ b/api/tests/unit_tests/core/tools/test_builtin_tools_extra.py @@ -7,6 +7,7 @@ from zoneinfo import ZoneInfo import pytest +from sqlalchemy.orm import Session from core.app.entities.app_invoke_entities import InvokeFrom from core.tools.__base.tool_runtime import ToolRuntime @@ -50,19 +51,24 @@ def _raise_runtime_error(*_args: object, **_kwargs: object) -> None: raise RuntimeError("boom") -def test_current_time_tool(): +def test_current_time_tool(unbound_session: Session): current_tool = _build_builtin_tool(CurrentTimeTool) - utc_text = list(current_tool.invoke(user_id="u", tool_parameters={"timezone": "UTC"}))[0].message.text + utc_text = list(current_tool.invoke(session=unbound_session, user_id="u", tool_parameters={"timezone": "UTC"}))[ + 0 + ].message.text assert utc_text - invalid_tz = list(current_tool.invoke(user_id="u", tool_parameters={"timezone": "Invalid/TZ"}))[0].message.text + invalid_tz = list( + current_tool.invoke(session=unbound_session, user_id="u", tool_parameters={"timezone": "Invalid/TZ"}) + )[0].message.text assert "Invalid timezone" in invalid_tz -def test_localtime_to_timestamp_tool(): +def test_localtime_to_timestamp_tool(unbound_session: Session): localtime_tool = _build_builtin_tool(LocaltimeToTimestampTool) ts_message = list( localtime_tool.invoke( + session=unbound_session, user_id="u", tool_parameters={"localtime": "2024-01-01 10:00:00", "timezone": "UTC"}, ) @@ -88,20 +94,25 @@ def test_localtime_to_timestamp_tool(): LocaltimeToTimestampTool.localtime_to_timestamp("bad", "%Y-%m-%d %H:%M:%S", "UTC") -def test_timestamp_to_localtime_tool(): +def test_timestamp_to_localtime_tool(unbound_session: Session): to_local_tool = _build_builtin_tool(TimestampToLocaltimeTool) - local_text = list(to_local_tool.invoke(user_id="u", tool_parameters={"timestamp": 1704067200, "timezone": "UTC"}))[ - 0 - ].message.text + local_text = list( + to_local_tool.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"timestamp": 1704067200, "timezone": "UTC"}, + ) + )[0].message.text assert "2024" in local_text with pytest.raises(ToolInvokeError): TimestampToLocaltimeTool.timestamp_to_localtime("bad", "UTC") # type: ignore[arg-type] -def test_timezone_conversion_tool(): +def test_timezone_conversion_tool(unbound_session: Session): timezone_tool = _build_builtin_tool(TimezoneConversionTool) converted = list( timezone_tool.invoke( + session=unbound_session, user_id="u", tool_parameters={ "current_time": "2024-01-01 08:00:00", @@ -115,9 +126,15 @@ def test_timezone_conversion_tool(): TimezoneConversionTool.timezone_convert("bad", "UTC", "Asia/Tokyo") -def test_weekday_tool(): +def test_weekday_tool(unbound_session: Session): weekday_tool = _build_builtin_tool(WeekdayTool) - valid = list(weekday_tool.invoke(user_id="u", tool_parameters={"year": 2024, "month": 1, "day": 1}))[0].message.text + valid = list( + weekday_tool.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"year": 2024, "month": 1, "day": 1}, + ) + )[0].message.text expected_date = date(2024, 1, 1) expected_message = ( f"{calendar.month_name[expected_date.month]} " @@ -125,17 +142,31 @@ def test_weekday_tool(): f"is {calendar.day_name[expected_date.weekday()]}." ) assert valid == expected_message - invalid = list(weekday_tool.invoke(user_id="u", tool_parameters={"year": 2024, "month": 2, "day": 31}))[ - 0 - ].message.text + invalid = list( + weekday_tool.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"year": 2024, "month": 2, "day": 31}, + ) + )[0].message.text assert "Invalid date" in invalid with pytest.raises(ValueError, match="Month is required"): - list(weekday_tool.invoke(user_id="u", tool_parameters={"year": 2024, "day": 1})) + list( + weekday_tool.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"year": 2024, "day": 1}, + ) + ) # LLMs often send numeric parameters as strings; these must be accepted. - string_params = list(weekday_tool.invoke(user_id="u", tool_parameters={"year": "2024", "month": "3", "day": "5"}))[ - 0 - ].message.text + string_params = list( + weekday_tool.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"year": "2024", "month": "3", "day": "5"}, + ) + )[0].message.text expected_date = date(2024, 3, 5) assert string_params == ( f"{calendar.month_name[expected_date.month]} " @@ -144,11 +175,11 @@ def test_weekday_tool(): ) # Missing or non-numeric values must yield an "Invalid date" message, not crash. for params in ({"year": 2024, "month": 3}, {"year": "abc", "month": 3, "day": 5}): - result = list(weekday_tool.invoke(user_id="u", tool_parameters=params))[0].message.text + result = list(weekday_tool.invoke(session=unbound_session, user_id="u", tool_parameters=params))[0].message.text assert "Invalid date" in result -def test_simple_code_valid_execution(monkeypatch: pytest.MonkeyPatch): +def test_simple_code_valid_execution(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): simple_code = _build_builtin_tool(SimpleCode) monkeypatch.setattr( @@ -157,6 +188,7 @@ def test_simple_code_valid_execution(monkeypatch: pytest.MonkeyPatch): ) result = list( simple_code.invoke( + session=unbound_session, user_id="u", tool_parameters={"language": "python3", "code": "print(1)"}, ) @@ -164,14 +196,20 @@ def test_simple_code_valid_execution(monkeypatch: pytest.MonkeyPatch): assert result == "ok" -def test_simple_code_invalid_language(): +def test_simple_code_invalid_language(unbound_session: Session): simple_code = _build_builtin_tool(SimpleCode) with pytest.raises(ValueError, match="Only python3 and javascript"): - list(simple_code.invoke(user_id="u", tool_parameters={"language": "go", "code": "fmt.Println(1)"})) + list( + simple_code.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"language": "go", "code": "fmt.Println(1)"}, + ) + ) -def test_simple_code_execution_error(monkeypatch: pytest.MonkeyPatch): +def test_simple_code_execution_error(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): simple_code = _build_builtin_tool(SimpleCode) monkeypatch.setattr( @@ -181,31 +219,39 @@ def test_simple_code_execution_error(monkeypatch: pytest.MonkeyPatch): with pytest.raises(ToolInvokeError, match="boom"): list( simple_code.invoke( + session=unbound_session, user_id="u", tool_parameters={"language": "python3", "code": "print(1)"}, ) ) -def test_webscraper_empty_url(): +def test_webscraper_empty_url(unbound_session: Session): webscraper = _build_builtin_tool(WebscraperTool) - empty = list(webscraper.invoke(user_id="u", tool_parameters={"url": ""}))[0].message.text + empty = list(webscraper.invoke(session=unbound_session, user_id="u", tool_parameters={"url": ""}))[0].message.text assert empty == "Please input url" -def test_webscraper_fetch(monkeypatch: pytest.MonkeyPatch): +def test_webscraper_fetch(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): webscraper = _build_builtin_tool(WebscraperTool) monkeypatch.setattr("core.tools.builtin_tool.providers.webscraper.tools.webscraper.get_url", lambda *a, **k: "page") - full = list(webscraper.invoke(user_id="u", tool_parameters={"url": "https://example.com"}))[0].message.text + full = list( + webscraper.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"url": "https://example.com"}, + ) + )[0].message.text assert full == "page" -def test_webscraper_summary(monkeypatch: pytest.MonkeyPatch): +def test_webscraper_summary(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): webscraper = _build_builtin_tool(WebscraperTool) monkeypatch.setattr("core.tools.builtin_tool.providers.webscraper.tools.webscraper.get_url", lambda *a, **k: "page") monkeypatch.setattr(webscraper, "summary", lambda user_id, content: "summary") summarized = list( webscraper.invoke( + session=unbound_session, user_id="u", tool_parameters={"url": "https://example.com", "generate_summary": True}, ) @@ -213,24 +259,32 @@ def test_webscraper_summary(monkeypatch: pytest.MonkeyPatch): assert summarized == "summary" -def test_webscraper_fetch_error(monkeypatch: pytest.MonkeyPatch): +def test_webscraper_fetch_error(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): webscraper = _build_builtin_tool(WebscraperTool) monkeypatch.setattr( "core.tools.builtin_tool.providers.webscraper.tools.webscraper.get_url", _raise_runtime_error, ) with pytest.raises(ToolInvokeError, match="boom"): - list(webscraper.invoke(user_id="u", tool_parameters={"url": "https://example.com"})) + list( + webscraper.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"url": "https://example.com"}, + ) + ) -def test_asr_invalid_file(): +def test_asr_invalid_file(unbound_session: Session): asr = _build_builtin_tool(ASRTool) file_obj = SimpleNamespace(type=FileType.DOCUMENT) - invalid_file = list(asr.invoke(user_id="u", tool_parameters={"audio_file": file_obj}))[0].message.text + invalid_file = list(asr.invoke(session=unbound_session, user_id="u", tool_parameters={"audio_file": file_obj}))[ + 0 + ].message.text assert "not a valid audio file" in invalid_file -def test_asr_valid_file_invocation(monkeypatch: pytest.MonkeyPatch): +def test_asr_valid_file_invocation(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): asr = _build_builtin_tool(ASRTool) model_instance = type("M", (), {"invoke_speech2text": lambda self, file: "transcript"})() model_manager = type("Mgr", (), {"get_model_instance": lambda *a, **k: model_instance})() @@ -242,7 +296,13 @@ def test_asr_valid_file_invocation(monkeypatch: pytest.MonkeyPatch): lambda **kwargs: captured_manager_kwargs.update(kwargs) or model_manager, ) audio_file = SimpleNamespace(type=FileType.AUDIO) - ok = list(asr.invoke(user_id="u", tool_parameters={"audio_file": audio_file, "model": "p#m"}))[0].message.text + ok = list( + asr.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"audio_file": audio_file, "model": "p#m"}, + ) + )[0].message.text assert ok == "transcript" assert captured_manager_kwargs == {"tenant_id": "tenant-1", "user_id": "u"} @@ -258,7 +318,7 @@ def test_asr_available_models_and_runtime_parameters(monkeypatch: pytest.MonkeyP assert asr.get_runtime_parameters()[0].name == "model" -def test_tts_invoke_returns_messages(monkeypatch: pytest.MonkeyPatch): +def test_tts_invoke_returns_messages(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): tts = _build_builtin_tool(TTSTool) captured_manager_kwargs = {} voices_model_instance = type( @@ -276,7 +336,13 @@ def test_tts_invoke_returns_messages(monkeypatch: pytest.MonkeyPatch): or type("M", (), {"get_model_instance": lambda *a, **k: voices_model_instance})() ), ) - messages = list(tts.invoke(user_id="u", tool_parameters={"model": "p#m", "text": "hello"})) + messages = list( + tts.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"model": "p#m", "text": "hello"}, + ) + ) assert [m.type for m in messages] == [ToolInvokeMessage.MessageType.TEXT, ToolInvokeMessage.MessageType.BLOB] assert messages[1].meta == {"mime_type": "audio/mpeg"} assert captured_manager_kwargs == {"tenant_id": "tenant-1", "user_id": "u"} @@ -289,18 +355,24 @@ def test_tts_get_available_models_requires_runtime(): tts.get_available_models() -def test_tts_tool_raises_when_runtime_missing(): +def test_tts_tool_raises_when_runtime_missing(unbound_session: Session): tts = _build_builtin_tool(TTSTool) tts.runtime = None with pytest.raises(ValueError, match="Runtime is required"): - list(tts.invoke(user_id="u", tool_parameters={"model": "p#m", "text": "hello"})) + list( + tts.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"model": "p#m", "text": "hello"}, + ) + ) @pytest.mark.parametrize( "voices", [[{"value": None}], []], ) -def test_tts_tool_raises_when_voice_unavailable(monkeypatch, voices): +def test_tts_tool_raises_when_voice_unavailable(monkeypatch, voices, unbound_session: Session): tts = _build_builtin_tool(TTSTool) tts.runtime = ToolRuntime(tenant_id="tenant-1", invoke_from=InvokeFrom.DEBUGGER) model_without_voice = type( @@ -316,7 +388,13 @@ def test_tts_tool_raises_when_voice_unavailable(monkeypatch, voices): lambda **_: type("Manager", (), {"get_model_instance": lambda *args, **kwargs: model_without_voice})(), ) with pytest.raises(ValueError, match="no voice available"): - list(tts.invoke(user_id="u", tool_parameters={"model": "p#m", "text": "hello"})) + list( + tts.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"model": "p#m", "text": "hello"}, + ) + ) def test_tts_tool_get_available_models_and_runtime_parameters(monkeypatch: pytest.MonkeyPatch): @@ -368,13 +446,14 @@ def test_provider_classes_and_builtin_sort(monkeypatch: pytest.MonkeyPatch): assert [p.name for p in sorted_providers] == ["a", "b"] -def test_localtime_to_timestamp_tool_epoch_zero(): +def test_localtime_to_timestamp_tool_epoch_zero(unbound_session: Session): localtime_tool = _build_builtin_tool(LocaltimeToTimestampTool) # 1970-01-01 08:00:00 in Asia/Shanghai is exactly Unix epoch 0. A valid # conversion to timestamp 0 must not be reported as "Invalid localtime" # just because 0 is falsy. epoch_message = list( localtime_tool.invoke( + session=unbound_session, user_id="u", tool_parameters={"localtime": "1970-01-01 08:00:00", "timezone": "Asia/Shanghai"}, ) diff --git a/api/tests/unit_tests/core/tools/test_custom_tool.py b/api/tests/unit_tests/core/tools/test_custom_tool.py index 4ea71deb4bbafe..0d3297ce91c2fc 100644 --- a/api/tests/unit_tests/core/tools/test_custom_tool.py +++ b/api/tests/unit_tests/core/tools/test_custom_tool.py @@ -241,7 +241,9 @@ def _fake_get(url, **kwargs): invalid_method_tool.do_http_request("https://api.example.com", "TRACE", headers={}, parameters={}) -def test_do_http_request_handles_file_upload_and_invoke_paths(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_do_http_request_handles_file_upload_and_invoke_paths( + monkeypatch: pytest.MonkeyPatch, unbound_session: Session +): openapi = { "parameters": [], "requestBody": { @@ -281,11 +283,11 @@ def _fake_post(url, **kwargs): monkeypatch.setattr(tool, "assembling_request", lambda parameters: {}) monkeypatch.setattr(tool, "do_http_request", lambda *args, **kwargs: httpx.Response(200, text='{"a":1}')) monkeypatch.setattr(tool, "validate_and_parse_response", lambda _: ParsedResponse({"a": 1}, True)) - messages = list(tool.invoke(user_id="u1", tool_parameters={})) + messages = list(tool.invoke(session=unbound_session, user_id="u1", tool_parameters={})) assert [m.type for m in messages] == [ToolInvokeMessage.MessageType.JSON, ToolInvokeMessage.MessageType.TEXT] # _invoke text path monkeypatch.setattr(tool, "validate_and_parse_response", lambda _: ParsedResponse("plain", False)) - messages = list(tool.invoke(user_id="u1", tool_parameters={})) + messages = list(tool.invoke(session=unbound_session, user_id="u1", tool_parameters={})) assert len(messages) == 1 assert messages[0].message.text == "plain" diff --git a/api/tests/unit_tests/core/tools/test_dataset_retriever_tool.py b/api/tests/unit_tests/core/tools/test_dataset_retriever_tool.py index d682f4f4d4030c..e64b545534b5e6 100644 --- a/api/tests/unit_tests/core/tools/test_dataset_retriever_tool.py +++ b/api/tests/unit_tests/core/tools/test_dataset_retriever_tool.py @@ -54,7 +54,7 @@ def dataset_tools(sqlite_session: Session, sqlite_session_factory): return tools, retrieval, config -def test_tool_metadata_and_fork_preserve_injected_retrieval(dataset_tools): +def test_tool_metadata_and_fork_preserve_injected_retrieval(dataset_tools, unbound_session: Session): tools, retrieval, config = dataset_tools assert len(tools) == 1 tool = tools[0] @@ -62,15 +62,17 @@ def test_tool_metadata_and_fork_preserve_injected_retrieval(dataset_tools): assert [parameter.name for parameter in tool.get_runtime_parameters()] == ["query"] assert config.retrieve_strategy == "multiple" forked = tool.fork_tool_runtime(ToolRuntime(tenant_id="tenant")) - result = list(forked.invoke(user_id="user", tool_parameters={"query": "hello"})) + result = list(forked.invoke(session=unbound_session, user_id="user", tool_parameters={"query": "hello"})) assert result[0].message.text == "result:hello" assert retrieval.calls[0]["inputs"] == {"x": 1} assert retrieval.calls[0]["app_id"] == "app" -def test_empty_query_does_not_start_retrieval(dataset_tools): +def test_empty_query_does_not_start_retrieval(dataset_tools, unbound_session: Session): tools, retrieval, _ = dataset_tools - assert list(tools[0].invoke(user_id="user", tool_parameters={}))[0].message.text == "please input query" + assert list(tools[0].invoke(session=unbound_session, user_id="user", tool_parameters={}))[0].message.text == ( + "please input query" + ) assert not retrieval.calls diff --git a/api/tests/unit_tests/core/tools/test_mcp_tool.py b/api/tests/unit_tests/core/tools/test_mcp_tool.py index 206e673f62370f..ab39c947ceaaa7 100644 --- a/api/tests/unit_tests/core/tools/test_mcp_tool.py +++ b/api/tests/unit_tests/core/tools/test_mcp_tool.py @@ -6,6 +6,7 @@ from unittest.mock import patch import pytest +from sqlalchemy.orm import Session from core.app.entities.app_invoke_entities import InvokeFrom from core.mcp.types import ( @@ -100,7 +101,7 @@ def test_mcp_tool_usage_extraction_helpers(): assert derived.total_tokens == 0 -def test_mcp_tool_invoke_handles_content_types_and_structured_output(): +def test_mcp_tool_invoke_handles_content_types_and_structured_output(unbound_session: Session): tool = _build_mcp_tool() img_data = base64.b64encode(b"img").decode() blob_data = base64.b64encode(b"blob").decode() @@ -126,7 +127,7 @@ def test_mcp_tool_invoke_handles_content_types_and_structured_output(): ) with patch.object(MCPTool, "invoke_remote_mcp_tool", return_value=result): - messages = list(tool.invoke(user_id="user-1", tool_parameters={"a": 1})) + messages = list(tool.invoke(session=unbound_session, user_id="user-1", tool_parameters={"a": 1})) types = [m.type for m in messages] assert ToolInvokeMessage.MessageType.JSON in types @@ -136,7 +137,7 @@ def test_mcp_tool_invoke_handles_content_types_and_structured_output(): assert tool.latest_usage.total_tokens == 5 -def test_mcp_tool_invoke_raises_for_unsupported_embedded_resource(): +def test_mcp_tool_invoke_raises_for_unsupported_embedded_resource(unbound_session: Session): tool = _build_mcp_tool() # Use model_construct to bypass pydantic validation and force unsupported resource path. bad_resource = EmbeddedResource.model_construct(type="resource", resource=object()) @@ -144,7 +145,7 @@ def test_mcp_tool_invoke_raises_for_unsupported_embedded_resource(): with patch.object(MCPTool, "invoke_remote_mcp_tool", return_value=result): with pytest.raises(ToolInvokeError, match="Unsupported embedded resource type"): - list(tool.invoke(user_id="user-1", tool_parameters={})) + list(tool.invoke(session=unbound_session, user_id="user-1", tool_parameters={})) def test_mcp_tool_handle_none_parameter_filters_empty_values(): diff --git a/api/tests/unit_tests/core/tools/test_plugin_tool.py b/api/tests/unit_tests/core/tools/test_plugin_tool.py index b3427878ad11df..7b6e1ce5c72f27 100644 --- a/api/tests/unit_tests/core/tools/test_plugin_tool.py +++ b/api/tests/unit_tests/core/tools/test_plugin_tool.py @@ -4,6 +4,7 @@ from unittest.mock import patch import httpx +from sqlalchemy.orm import Session import core.plugin.impl.base as plugin_client_module from core.app.entities.app_invoke_entities import InvokeFrom @@ -41,7 +42,7 @@ def _build_plugin_tool(*, has_runtime_parameters: bool) -> PluginTool: ) -def test_plugin_tool_invoke_and_fork_runtime(): +def test_plugin_tool_invoke_and_fork_runtime(unbound_session: Session): tool = _build_plugin_tool(has_runtime_parameters=False) requests: list[httpx.Request] = [] @@ -60,7 +61,7 @@ def handle_request(request: httpx.Request) -> httpx.Response: return_value={"converted": 1}, ), ): - messages = list(tool.invoke(user_id="user-1", tool_parameters={"raw": 1})) + messages = list(tool.invoke(session=unbound_session, user_id="user-1", tool_parameters={"raw": 1})) assert [m.message.text for m in messages] == ["ok"] assert len(requests) == 1 diff --git a/api/tests/unit_tests/core/tools/test_tool_engine.py b/api/tests/unit_tests/core/tools/test_tool_engine.py index df348a97fc934a..275a00b9bc0c86 100644 --- a/api/tests/unit_tests/core/tools/test_tool_engine.py +++ b/api/tests/unit_tests/core/tools/test_tool_engine.py @@ -79,6 +79,7 @@ def tool_provider_type(self) -> ToolProviderType: def _invoke( self, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, @@ -274,17 +275,18 @@ def test_create_message_files_and_invoke_generator(sqlite_session: Session, *, a assert {message_file.message_id for message_file in message_files} == {agent_message.id} tool = _build_tool() - invoked = list(ToolEngine._invoke(tool, {"a": 1}, user_id="u")) + invoked = list(ToolEngine._invoke(sqlite_session, tool, {"a": 1}, user_id="u")) assert invoked[0].type == ToolInvokeMessage.MessageType.TEXT assert isinstance(invoked[-1], ToolInvokeMeta) assert invoked[-1].error is None -def test_generic_invoke_success_and_error_paths(): +def test_generic_invoke_success_and_error_paths(unbound_session: Session): tool = _build_tool() callback = DifyWorkflowCallbackHandler() response = list( ToolEngine.generic_invoke( + session=unbound_session, tool=tool, tool_parameters={"x": 1}, user_id="u1", @@ -302,6 +304,7 @@ def test_generic_invoke_success_and_error_paths(): with pytest.raises(RuntimeError, match="boom"): list( ToolEngine.generic_invoke( + session=unbound_session, tool=tool, tool_parameters={"x": 1}, user_id="u1", @@ -311,7 +314,7 @@ def test_generic_invoke_success_and_error_paths(): ) -def test_agent_invoke_success(app_records): +def test_agent_invoke_success(unbound_session: Session, app_records): tool = _build_tool(with_llm_parameter=True) callback = DifyAgentCallbackHandler() message = _message() @@ -325,6 +328,7 @@ def test_agent_invoke_success(app_records): with patch.object(ToolEngine, "_extract_tool_response_binary_and_text", return_value=iter([])): with patch.object(ToolEngine, "_create_message_files", return_value=[]): result_text, message_files, result_meta = ToolEngine.agent_invoke( + session=unbound_session, records=app_records, tool=tool, tool_parameters="hello", @@ -340,13 +344,14 @@ def test_agent_invoke_success(app_records): assert result_meta.error is None -def test_agent_invoke_param_validation_error(app_records): +def test_agent_invoke_param_validation_error(unbound_session: Session, app_records): tool = _build_tool(with_llm_parameter=True) callback = DifyAgentCallbackHandler() message = _message() with patch.object(ToolEngine, "_invoke", side_effect=ToolParameterValidationError("bad-param")): error_text, files, error_meta = ToolEngine.agent_invoke( + session=unbound_session, records=app_records, tool=tool, tool_parameters={"a": 1}, @@ -362,7 +367,7 @@ def test_agent_invoke_param_validation_error(app_records): assert error_meta.error -def test_agent_invoke_engine_meta_error(app_records): +def test_agent_invoke_engine_meta_error(unbound_session: Session, app_records): tool = _build_tool(with_llm_parameter=True) callback = DifyAgentCallbackHandler() message = _message() @@ -370,6 +375,7 @@ def test_agent_invoke_engine_meta_error(app_records): with patch.object(ToolEngine, "_invoke", side_effect=engine_error): error_text, files, error_meta = ToolEngine.agent_invoke( + session=unbound_session, records=app_records, tool=tool, tool_parameters={"a": 1}, @@ -407,13 +413,14 @@ def test_convert_tool_response_excludes_variable_messages(): assert "variable_name" not in result -def test_agent_invoke_tool_invoke_error(app_records): +def test_agent_invoke_tool_invoke_error(unbound_session: Session, app_records): tool = _build_tool(with_llm_parameter=True) callback = DifyAgentCallbackHandler() message = _message() with patch.object(ToolEngine, "_invoke", side_effect=ToolInvokeError("invoke boom")): error_text, files, _ = ToolEngine.agent_invoke( + session=unbound_session, records=app_records, tool=tool, tool_parameters={"a": 1}, diff --git a/api/tests/unit_tests/extensions/test_ext_application_services.py b/api/tests/unit_tests/extensions/test_ext_application_services.py index 118ace4d08793a..4b8b58b7574527 100644 --- a/api/tests/unit_tests/extensions/test_ext_application_services.py +++ b/api/tests/unit_tests/extensions/test_ext_application_services.py @@ -56,6 +56,7 @@ from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository from repositories.upload_file_delivery_repository import UploadFileDeliveryQueryRepository from repositories.workflow.app_log_repository import WorkflowAppLogRepository +from repositories.workflow.runtime_context_repository import WorkflowRuntimeContextRepository from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepository from repositories.workspace.workspace_repository import WorkspaceRepository from services import audio_provider_gateway, recommended_app_catalog_gateway @@ -269,7 +270,9 @@ def test_build_application_services_preserves_composed_boundaries( assert isinstance(services.dataset_api_keys, DatasetApiKeyService) assert services.dataset_api_keys._access is services.knowledge.datasets._dataset_access assert services.snippet_generation._variables is services.workflow_variables - assert services.snippet_generation._runtime.contexts._sessions is sqlite_session_factory + runtime_contexts = services.snippet_generation._runtime.contexts + assert isinstance(runtime_contexts, WorkflowRuntimeContextRepository) + assert runtime_contexts._sessions is sqlite_session_factory assert services.knowledge.pipeline_generator._draft_variable_loader == services.workflow_variables.workflow_loader assert services.knowledge.pipeline_generator._draft_variable_saver == services.workflow_variables.saver_factory assert isinstance(services.oauth_device, OAuthDeviceApplicationService) diff --git a/api/tests/unit_tests/services/workflow/test_runtime_gateway.py b/api/tests/unit_tests/services/workflow/test_runtime_gateway.py index f1424693e58e28..a97f3d31dfad40 100644 --- a/api/tests/unit_tests/services/workflow/test_runtime_gateway.py +++ b/api/tests/unit_tests/services/workflow/test_runtime_gateway.py @@ -2,7 +2,7 @@ import json from collections.abc import Callable, Generator, Iterable -from typing import NoReturn, Self, cast, override +from typing import Literal, NoReturn, Self, cast, override from unittest.mock import Mock, create_autospec import pytest @@ -1272,7 +1272,7 @@ def test_container_debug_prepares_database_records_before_redis( sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, mode: AppMode, - entry: str, + entry: Literal["iteration", "loop"], ) -> None: import threading @@ -1312,8 +1312,12 @@ def unexpected_gateway_session() -> NoReturn: monkeypatch.setattr(threading.Thread, "join", lambda _thread, **_kwargs: None) monkeypatch.setattr(WorkflowAppGenerator, "_handle_response", response) monkeypatch.setattr(AdvancedChatAppGenerator, "_handle_advanced_chat_response", response) + debug_entries = { + "iteration": gateway.iteration, + "loop": gateway.loop, + } with Flask(__name__).app_context(): - result = getattr(gateway, entry)(CONTEXT, "app-1", "container", {"item": "value"}) + result = debug_entries[entry](CONTEXT, "app-1", "container", {"item": "value"}) list(result) assert writes assert all(session.closed and not session.in_transaction() for session in sessions) diff --git a/api/tests/unit_tests/tasks/test_resume_agent_app_task.py b/api/tests/unit_tests/tasks/test_resume_agent_app_task.py index 76fd4079259174..16fb018e23991a 100644 --- a/api/tests/unit_tests/tasks/test_resume_agent_app_task.py +++ b/api/tests/unit_tests/tasks/test_resume_agent_app_task.py @@ -12,6 +12,7 @@ from core.app.entities.app_invoke_entities import InvokeFrom from enums.human_input import HumanInputFormKind, HumanInputFormStatus +from extensions.ext_application_services import ApplicationServices from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole from models.enums import ConversationFromSource, EndUserType from models.enums import InvokeFrom as StoredInvokeFrom @@ -25,10 +26,13 @@ @pytest.fixture def task_session( - mocker: MockerFixture, sqlite_session_factory: sessionmaker[Session], workflow_application + mocker: MockerFixture, + sqlite_session_factory: sessionmaker[Session], + workflow_application: ApplicationServices, ) -> Iterator[Session]: - """Bind the task's Flask-SQLAlchemy session proxy to the shared SQLite database.""" - assert mod.application_services() is workflow_application + """Bind the task to the shared SQLite database and application composition.""" + application_services = mocker.patch.object(mod, "application_services", return_value=workflow_application) + assert application_services() is workflow_application registry = scoped_session(sqlite_session_factory) mocker.patch.object(mod.db, "session", registry) session = registry() @@ -100,7 +104,11 @@ def _seed_account(session: Session, *, tenant_id: str, account_id: str) -> Accou return account -def test_resume_happy_path_account_user_sets_tenant_and_runs(mocker: MockerFixture, task_session: Session) -> None: +def test_resume_happy_path_account_user_sets_tenant_and_runs( + mocker: MockerFixture, + task_session: Session, + workflow_application: ApplicationServices, +) -> None: tenant_id, app_id, conversation_id, form_id, account_id = (str(uuid4()) for _ in range(5)) app = _app(app_id=app_id, tenant_id=tenant_id) account = _seed_account(task_session, tenant_id=tenant_id, account_id=account_id) @@ -111,6 +119,14 @@ def test_resume_happy_path_account_user_sets_tenant_and_runs(mocker: MockerFixtu mod.resume_agent_app_execution(conversation_id=conversation_id, form_id=form_id) + constructor = generator.call_args + assert constructor is not None + assert constructor.kwargs["forms"] is workflow_application.workflow_runtime.human_forms + assert constructor.kwargs["agent_configs"] is workflow_application.workflow_runtime.agent_configs + assert constructor.kwargs["tool_providers"] is workflow_application.tools.tool_providers + assert constructor.kwargs["workflow_queries"] is workflow_application.tools.workflow_queries + assert constructor.kwargs["records"] is workflow_application.workflow_runtime.chat_records + assert constructor.kwargs["annotations"] is workflow_application.workflow_runtime.annotation_replies call = generator.return_value.resume_after_form_submission.call_args assert call is not None assert call.kwargs["conversation_id"] == conversation_id diff --git a/api/tests/unit_tests/tools/test_api_tool.py b/api/tests/unit_tests/tools/test_api_tool.py index 10789277de3bcf..f1ac02fa1c35bd 100644 --- a/api/tests/unit_tests/tools/test_api_tool.py +++ b/api/tests/unit_tests/tools/test_api_tool.py @@ -1,11 +1,9 @@ import json import operator -from collections.abc import Iterator from unittest.mock import Mock, patch import httpx import pytest -from sqlalchemy import Engine from sqlalchemy.orm import Session from core.tools.__base.tool_runtime import ToolRuntime @@ -19,13 +17,6 @@ ) -@pytest.fixture -def database_session(sqlite_engine: Engine) -> Iterator[Session]: - """Yield the live ORM session required by the shared tool invocation contract.""" - with Session(sqlite_engine) as session: - yield session - - def _get_message_by_type[T](msgs: list[ToolInvokeMessage], msg_type: type[T]) -> ToolInvokeMessage | None: return next((i for i in msgs if isinstance(i.message, msg_type)), None) @@ -65,7 +56,9 @@ def setup_method(self): ) @patch("core.tools.custom_tool.tool.ssrf_proxy.get") - def test_invoke_with_json_response_creates_text_message_with_serialized_json(self, mock_get: Mock) -> None: + def test_invoke_with_json_response_creates_text_message_with_serialized_json( + self, mock_get: Mock, unbound_session: Session + ) -> None: """Test that when upstream returns JSON, the output Text message contains JSON-serialized string.""" # Use the response's real JSON parsing and content decoding. json_response_data = { @@ -76,7 +69,7 @@ def test_invoke_with_json_response_creates_text_message_with_serialized_json(sel mock_get.return_value = httpx.Response(200, json=json_response_data) # Invoke the tool - result_generator = self.api_tool._invoke(user_id="test_user", tool_parameters={}) + result_generator = self.api_tool._invoke(session=unbound_session, user_id="test_user", tool_parameters={}) # Get the result from the generator result = list(result_generator) @@ -137,7 +130,7 @@ def test_invoke_with_json_response_creates_text_message_with_serialized_json(sel ids=operator.itemgetter(0), ) def test_invoke_with_non_dict_json_response_creates_text_message_with_serialized_json( - self, mock_get: Mock, test_case + self, mock_get: Mock, test_case, unbound_session: Session ) -> None: """Test that when upstream returns a non-dict JSON, the output Text message contains JSON-serialized string.""" # Explicit content also covers JSON null, which is distinct from no body. @@ -147,7 +140,7 @@ def test_invoke_with_non_dict_json_response_creates_text_message_with_serialized ) # Invoke the tool - result_generator = self.api_tool._invoke(user_id="test_user", tool_parameters={}) + result_generator = self.api_tool._invoke(session=unbound_session, user_id="test_user", tool_parameters={}) # Get the result from the generator result = list(result_generator) @@ -171,14 +164,16 @@ def test_invoke_with_non_dict_json_response_creates_text_message_with_serialized assert json_message is None, "_invoke should not yield a JSON message for JSON array response" @patch("core.tools.custom_tool.tool.ssrf_proxy.get") - def test_invoke_with_text_response_creates_text_message_with_original_text(self, mock_get: Mock) -> None: + def test_invoke_with_text_response_creates_text_message_with_original_text( + self, mock_get: Mock, unbound_session: Session + ) -> None: """Test that when upstream returns plain text, the output Text message contains the original text.""" # Plain text exercises the response's real JSON decoding failure. text_response_data = "This is a plain text response" mock_get.return_value = httpx.Response(200, text=text_response_data) # Invoke the tool - result_generator = self.api_tool._invoke(user_id="test_user", tool_parameters={}) + result_generator = self.api_tool._invoke(session=unbound_session, user_id="test_user", tool_parameters={}) # Get the result from the generator result = list(result_generator) @@ -194,12 +189,12 @@ def test_invoke_with_text_response_creates_text_message_with_original_text(self, assert message.message.text == text_response_data @patch("core.tools.custom_tool.tool.ssrf_proxy.get") - def test_invoke_with_empty_response(self, mock_get: Mock) -> None: + def test_invoke_with_empty_response(self, mock_get: Mock, unbound_session: Session) -> None: """Test that empty responses are handled correctly.""" mock_get.return_value = httpx.Response(200, content=b"", headers={"content-type": "application/json"}) # Invoke the tool - result_generator = self.api_tool._invoke(user_id="test_user", tool_parameters={}) + result_generator = self.api_tool._invoke(session=unbound_session, user_id="test_user", tool_parameters={}) # Get the result from the generator result = list(result_generator) @@ -215,11 +210,11 @@ def test_invoke_with_empty_response(self, mock_get: Mock) -> None: assert "Empty response from the tool" in message.message.text @patch("core.tools.custom_tool.tool.ssrf_proxy.get") - def test_invoke_with_error_response(self, mock_get: Mock) -> None: + def test_invoke_with_error_response(self, mock_get: Mock, unbound_session: Session) -> None: """Test that error responses are handled correctly.""" mock_get.return_value = httpx.Response(404, text="Not Found") - result_generator = self.api_tool._invoke(user_id="test_user", tool_parameters={}) + result_generator = self.api_tool._invoke(session=unbound_session, user_id="test_user", tool_parameters={}) # Invoke the tool and expect an error with pytest.raises(Exception) as exc_info: diff --git a/api/tests/unit_tests/tools/test_mcp_tool.py b/api/tests/unit_tests/tools/test_mcp_tool.py index 16f3e8716958cc..3e489a0cf0b67a 100644 --- a/api/tests/unit_tests/tools/test_mcp_tool.py +++ b/api/tests/unit_tests/tools/test_mcp_tool.py @@ -1,11 +1,9 @@ import base64 -from collections.abc import Iterator from decimal import Decimal from typing import Any from unittest.mock import patch import pytest -from sqlalchemy.engine import Engine from sqlalchemy.orm import Session from core.mcp.types import ( @@ -24,13 +22,6 @@ from graphon.model_runtime.entities.llm_entities import LLMUsage -@pytest.fixture -def orm_session(sqlite_engine: Engine) -> Iterator[Session]: - """Use a real ORM session while MCP transport remains mocked.""" - with Session(sqlite_engine) as session: - yield session - - def _make_mcp_tool(output_schema: dict[str, Any] | None = None) -> MCPTool: identity = ToolIdentity( author="test", @@ -65,7 +56,7 @@ class TestMCPToolInvoke: ), ], ) - def test_invoke_image_or_audio_yields_blob(self, content_factory, mime_type) -> None: + def test_invoke_image_or_audio_yields_blob(self, content_factory, mime_type, unbound_session: Session) -> None: tool = _make_mcp_tool() raw = b"\x00\x01test-bytes\x02" b64 = base64.b64encode(raw).decode() @@ -73,7 +64,7 @@ def test_invoke_image_or_audio_yields_blob(self, content_factory, mime_type) -> result = CallToolResult(content=[content]) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - messages = list(tool._invoke(user_id="test_user", tool_parameters={})) + messages = list(tool._invoke(session=unbound_session, user_id="test_user", tool_parameters={})) assert len(messages) == 1 msg = messages[0] @@ -82,14 +73,14 @@ def test_invoke_image_or_audio_yields_blob(self, content_factory, mime_type) -> assert msg.message.blob == raw assert msg.meta == {"mime_type": mime_type} - def test_invoke_embedded_text_resource_yields_text(self) -> None: + def test_invoke_embedded_text_resource_yields_text(self, unbound_session: Session) -> None: tool = _make_mcp_tool() text_resource = TextResourceContents(uri="file://test.txt", mimeType="text/plain", text="hello world") content = EmbeddedResource(type="resource", resource=text_resource) result = CallToolResult(content=[content]) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - messages = list(tool._invoke(user_id="test_user", tool_parameters={})) + messages = list(tool._invoke(session=unbound_session, user_id="test_user", tool_parameters={})) assert len(messages) == 1 msg = messages[0] @@ -101,7 +92,9 @@ def test_invoke_embedded_text_resource_yields_text(self) -> None: ("mime_type", "expected_mime"), [("application/pdf", "application/pdf"), (None, "application/octet-stream")], ) - def test_invoke_embedded_blob_resource_yields_blob(self, mime_type, expected_mime) -> None: + def test_invoke_embedded_blob_resource_yields_blob( + self, mime_type, expected_mime, unbound_session: Session + ) -> None: tool = _make_mcp_tool() raw = b"binary-data" b64 = base64.b64encode(raw).decode() @@ -110,7 +103,7 @@ def test_invoke_embedded_blob_resource_yields_blob(self, mime_type, expected_mim result = CallToolResult(content=[content]) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - messages = list(tool._invoke(user_id="test_user", tool_parameters={})) + messages = list(tool._invoke(session=unbound_session, user_id="test_user", tool_parameters={})) assert len(messages) == 1 msg = messages[0] @@ -119,12 +112,12 @@ def test_invoke_embedded_blob_resource_yields_blob(self, mime_type, expected_mim assert msg.message.blob == raw assert msg.meta == {"mime_type": expected_mime} - def test_invoke_yields_variables_when_structured_content_and_schema(self) -> None: + def test_invoke_yields_variables_when_structured_content_and_schema(self, unbound_session: Session) -> None: tool = _make_mcp_tool(output_schema={"type": "object"}) result = CallToolResult(content=[], structuredContent={"a": 1, "b": "x"}) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - messages = list(tool._invoke(user_id="test_user", tool_parameters={})) + messages = list(tool._invoke(session=unbound_session, user_id="test_user", tool_parameters={})) # Expect two variable messages corresponding to keys a and b assert len(messages) == 2 @@ -134,12 +127,12 @@ def test_invoke_yields_variables_when_structured_content_and_schema(self) -> Non values = {m.message.variable_name: m.message.variable_value for m in var_msgs} assert values == {"a": 1, "b": "x"} - def test_invoke_yields_json_when_structured_content_has_no_output_schema(self) -> None: + def test_invoke_yields_json_when_structured_content_has_no_output_schema(self, unbound_session: Session) -> None: tool = _make_mcp_tool() result = CallToolResult(content=[], structuredContent={"a": 1, "b": "x"}) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - messages = list(tool._invoke(user_id="test_user", tool_parameters={})) + messages = list(tool._invoke(session=unbound_session, user_id="test_user", tool_parameters={})) assert len(messages) == 1 msg = messages[0] @@ -288,7 +281,7 @@ def test_derive_usage_from_result_calculates_total_tokens(self) -> None: assert usage.prompt_tokens == 100 assert usage.completion_tokens == 50 - def test_invoke_sets_latest_usage_from_meta(self) -> None: + def test_invoke_sets_latest_usage_from_meta(self, unbound_session: Session) -> None: """Test that _invoke sets _latest_usage from result meta.""" tool = _make_mcp_tool() meta = { @@ -303,7 +296,7 @@ def test_invoke_sets_latest_usage_from_meta(self) -> None: result = CallToolResult(content=[TextContent(type="text", text="test")], _meta=meta) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - list(tool._invoke(user_id="test_user", tool_parameters={})) + list(tool._invoke(session=unbound_session, user_id="test_user", tool_parameters={})) # Verify latest_usage was set correctly assert tool.latest_usage.prompt_tokens == 200 @@ -311,13 +304,13 @@ def test_invoke_sets_latest_usage_from_meta(self) -> None: assert tool.latest_usage.total_tokens == 300 assert tool.latest_usage.total_price == Decimal("0.003") - def test_invoke_with_no_meta_returns_empty_usage(self) -> None: + def test_invoke_with_no_meta_returns_empty_usage(self, unbound_session: Session) -> None: """Test that _invoke returns empty usage when no meta is present.""" tool = _make_mcp_tool() result = CallToolResult(content=[TextContent(type="text", text="test")], _meta=None) with patch.object(tool, "invoke_remote_mcp_tool", return_value=result): - list(tool._invoke(user_id="test_user", tool_parameters={})) + list(tool._invoke(session=unbound_session, user_id="test_user", tool_parameters={})) # Verify latest_usage is empty assert tool.latest_usage.total_tokens == 0 From 6210169814c80a73733a9c6abf66facf2baa93b6 Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Fri, 9 Oct 2026 14:00:56 +0800 Subject: [PATCH 05/20] test: adapt human input controller calls to explicit handlers --- .../test_workflow_human_input_debug_api.py | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py b/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py index be760bd7c69863..d926c8d09d9a07 100644 --- a/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py +++ b/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py @@ -4,7 +4,6 @@ import pytest from flask import Flask -from flask_restx import Resource from pydantic import ValidationError from controllers.console.app import workflow as controller @@ -29,26 +28,36 @@ def debug_use_cases(monkeypatch: pytest.MonkeyPatch) -> Mock: @pytest.mark.parametrize( "resource", [controller.AdvancedChatDraftHumanInputFormPreviewApi, controller.WorkflowDraftHumanInputFormPreviewApi] ) -def test_preview_forwards_inputs(app: Flask, workflows: Mock, resource: type[Resource]) -> None: +def test_preview_forwards_inputs( + app: Flask, + workflows: Mock, + resource: type[ + controller.AdvancedChatDraftHumanInputFormPreviewApi | controller.WorkflowDraftHumanInputFormPreviewApi + ], +) -> None: payload = {"form_id": "human", "form_content": "Hello"} workflows.preview_form.return_value = payload args = controller.HumanInputFormPreviewPayload(inputs={"topic": "tech"}) with app.test_request_context(method="POST"): - assert invoke(resource, "post", args, node_id="human") == payload + assert invoke(resource, resource.post, args, node_id="human") == payload workflows.preview_form.assert_called_once_with(CONTEXT, str(APP_ID), "human", {"topic": "tech"}) @pytest.mark.parametrize( "resource", [controller.AdvancedChatDraftHumanInputFormRunApi, controller.WorkflowDraftHumanInputFormRunApi] ) -def test_submission_forwards_form_and_upstream_inputs(app: Flask, workflows: Mock, resource: type[Resource]) -> None: +def test_submission_forwards_form_and_upstream_inputs( + app: Flask, + workflows: Mock, + resource: type[controller.AdvancedChatDraftHumanInputFormRunApi | controller.WorkflowDraftHumanInputFormRunApi], +) -> None: outputs = {"answer": "42", "__action_id": "approve"} workflows.submit_form.return_value = outputs args = controller.HumanInputFormSubmitPayload( form_inputs={"answer": "42"}, inputs={"#upstream.output#": "text"}, action="approve" ) with app.test_request_context(method="POST"): - assert invoke(resource, "post", args, node_id="human") == outputs + assert invoke(resource, resource.post, args, node_id="human") == outputs workflows.submit_form.assert_called_once_with( CONTEXT, str(APP_ID), @@ -62,7 +71,15 @@ def test_submission_forwards_form_and_upstream_inputs(app: Flask, workflows: Moc def test_delivery_forwards_default_inputs(app: Flask, workflows: Mock) -> None: args = controller.HumanInputDeliveryTestPayload(delivery_method_id="email") with app.test_request_context(method="POST"): - assert invoke(controller.WorkflowDraftHumanInputDeliveryTestApi, "post", args, node_id="human") == {} + assert ( + invoke( + controller.WorkflowDraftHumanInputDeliveryTestApi, + controller.WorkflowDraftHumanInputDeliveryTestApi.post, + args, + node_id="human", + ) + == {} + ) workflows.test_delivery.assert_called_once_with( CONTEXT, str(APP_ID), "human", inputs={}, delivery_method_id="email" ) @@ -72,7 +89,12 @@ def test_delivery_preserves_domain_error(app: Flask, workflows: Mock) -> None: workflows.test_delivery.side_effect = ValueError("bad delivery method") args = controller.HumanInputDeliveryTestPayload(delivery_method_id="bad") with app.test_request_context(method="POST"), pytest.raises(ValueError, match="bad delivery method"): - invoke(controller.WorkflowDraftHumanInputDeliveryTestApi, "post", args, node_id="human") + invoke( + controller.WorkflowDraftHumanInputDeliveryTestApi, + controller.WorkflowDraftHumanInputDeliveryTestApi.post, + args, + node_id="human", + ) def test_preview_rejects_non_mapping_inputs() -> None: From 758966a412f942ab44a2fce3ec444158d89e5391 Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Fri, 9 Oct 2026 14:13:14 +0800 Subject: [PATCH 06/20] test: align integration fixtures with activated service contracts --- .../layers/test_pause_state_persist_layer.py | 31 ++++++++++++------- .../services/test_app_dsl_service.py | 30 +++++++++--------- .../services/test_app_generate_service.py | 8 ++++- .../test_workflow_draft_variable_service.py | 6 +++- .../services/test_workflow_run_service.py | 2 ++ .../services/test_workflow_service.py | 8 ++--- .../tools/test_mcp_tools_manage_service.py | 31 ++++++++++++------- .../tools/test_tools_transform_service.py | 24 +++++++++++--- .../test_workflow_tools_manage_service.py | 6 ++-- 9 files changed, 94 insertions(+), 52 deletions(-) diff --git a/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py b/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py index 56da1c948d1e24..d398d731c2103b 100644 --- a/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py +++ b/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py @@ -18,6 +18,7 @@ """ import uuid +from collections.abc import Iterator from time import time from unittest.mock import Mock @@ -48,6 +49,7 @@ from models.workflow import Workflow, WorkflowRun from repositories.factory import DifyAPIRepositoryFactory from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository +from repositories.tools.provider_repository import ToolProviderRepository from services.file_service import FileService from services.workflow_run_service import WorkflowRunService @@ -71,7 +73,7 @@ def _create_initialized_response_stream_filter() -> ResponseStreamFilter: class _TestCommandChannelImpl: """Real implementation of CommandChannel for testing.""" - def __init__(self): + def __init__(self) -> None: self._commands: list[GraphEngineCommand] = [] def fetch_commands(self) -> list[GraphEngineCommand]: @@ -87,19 +89,19 @@ class TestPauseStatePersistenceLayerTestContainers: """Comprehensive TestContainers-based integration tests for PauseStatePersistenceLayer class.""" @pytest.fixture - def engine(self, db_session_with_containers: Session): + def engine(self, db_session_with_containers: Session) -> Engine: """Get database engine from TestContainers session.""" bind = db_session_with_containers.get_bind() assert isinstance(bind, Engine) return bind @pytest.fixture - def file_service(self, engine: Engine): + def file_service(self, engine: Engine) -> FileService: """Create FileService instance with TestContainers engine.""" return FileService(engine) @pytest.fixture - def workflow_run_service(self, engine: Engine, file_service: FileService): + def workflow_run_service(self, engine: Engine, file_service: FileService) -> WorkflowRunService: """Create WorkflowRunService instance with TestContainers engine and FileService.""" session_factory = sessionmaker(bind=engine, expire_on_commit=False) workflow_runs = DifyAPISQLAlchemyWorkflowRunRepository(session_maker=session_factory) @@ -108,10 +110,16 @@ def workflow_run_service(self, engine: Engine, file_service: FileService): node_executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( session_maker=session_factory ), + tool_providers=ToolProviderRepository(session_factory), ) @pytest.fixture(autouse=True) - def setup_test_data(self, db_session_with_containers: Session, file_service, workflow_run_service): + def setup_test_data( + self, + db_session_with_containers: Session, + file_service: FileService, + workflow_run_service: WorkflowRunService, + ) -> Iterator[None]: """Set up test data for each test method using TestContainers.""" # Create test tenant and account from models.account import AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatus @@ -179,8 +187,7 @@ def setup_test_data(self, db_session_with_containers: Session, file_service, wor # Store session and service instances self.session = db_session_with_containers - self.file_service = file_service - self.workflow_run_service = workflow_run_service + self._workflow_run_service = workflow_run_service # Save test data to database self.session.add(self.test_workflow) @@ -192,7 +199,7 @@ def setup_test_data(self, db_session_with_containers: Session, file_service, wor # Cleanup self._cleanup_test_data() - def _cleanup_test_data(self): + def _cleanup_test_data(self) -> None: """Clean up test data after each test method.""" try: # Clean up workflow pauses @@ -317,8 +324,10 @@ def _create_pause_state_persistence_layer( workflow_id=str(workflow_id), ) + bind = self.session.get_bind() + assert isinstance(bind, Engine) return PauseStatePersistenceLayer( - session_factory=self.session.get_bind(), + session_factory=sessionmaker(bind=bind, expire_on_commit=False), state_owner_user_id=owner_id, generate_entity=entity, response_stream_filter=_create_initialized_response_stream_filter(), @@ -412,7 +421,7 @@ def test_state_persistence_and_retrieval(self, db_session_with_containers: Sessi layer.on_event(event) # Assert - Retrieve and verify - pause_entity = self.workflow_run_service._workflow_runs.get_workflow_pause(self.test_workflow_run_id) + pause_entity = self._workflow_run_service._workflow_runs.get_workflow_pause(self.test_workflow_run_id) assert pause_entity is not None assert pause_entity.workflow_execution_id == self.test_workflow_run_id assert pause_entity.get_pause_reasons() == event.reasons @@ -551,7 +560,7 @@ def test_workflow_with_different_creators(self, db_session_with_containers: Sess assert pause_model is not None # Verify the state owner is the workflow creator - pause_entity = self.workflow_run_service._workflow_runs.get_workflow_pause(different_workflow_run.id) + pause_entity = self._workflow_run_service._workflow_runs.get_workflow_pause(different_workflow_run.id) assert pause_entity is not None resumption_context = WorkflowResumptionContext.loads(pause_entity.get_state().decode()) assert resumption_context.get_generate_entity().workflow_execution_id == different_workflow_run.id diff --git a/api/tests/test_containers_integration_tests/services/test_app_dsl_service.py b/api/tests/test_containers_integration_tests/services/test_app_dsl_service.py index 5bcb0028462282..8f0a70657ba2fd 100644 --- a/api/tests/test_containers_integration_tests/services/test_app_dsl_service.py +++ b/api/tests/test_containers_integration_tests/services/test_app_dsl_service.py @@ -4,7 +4,7 @@ import json from types import SimpleNamespace from typing import Any -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, create_autospec, patch from uuid import uuid4 import pytest @@ -57,6 +57,7 @@ from services.app_service import AppService, CreateAppParams from services.dsl_version import check_version_compatibility from services.errors.app import WorkflowNotFoundError +from services.workflow.draft_service import WorkflowDraftService from tests.test_containers_integration_tests.helpers import accounts as account_fixtures from tests.test_containers_integration_tests.helpers import generate_valid_password @@ -470,7 +471,7 @@ def test_import_app_overwrite_only_allows_workflow_and_advanced_chat( app_id=app.id, ) assert result.status == ImportStatus.FAILED - assert "Only workflow or advanced chat apps" in result.error + assert result.error == "Only workflow or advanced chat DSLs can overwrite workflow Apps" # ── Import: Flow ────────────────────────────────────────────────── @@ -709,19 +710,7 @@ def test_create_or_update_app_existing_app_updates_fields( fixed_now = object() monkeypatch.setattr(app_dsl_service, "naive_utc_now", lambda: fixed_now) - workflow_service = MagicMock() - workflow_service.get_draft_workflow.return_value = None - monkeypatch.setattr(app_dsl_service, "WorkflowService", lambda: workflow_service) - monkeypatch.setattr( - app_dsl_service.variable_factory, - "build_environment_variable_from_mapping", - lambda _m: SimpleNamespace(kind="env"), - ) - monkeypatch.setattr( - app_dsl_service.variable_factory, - "build_conversation_variable_from_mapping", - lambda _m: SimpleNamespace(kind="conv"), - ) + drafts = create_autospec(WorkflowDraftService, instance=True, spec_set=True) app = _app_record( mode=AppMode.WORKFLOW, @@ -733,7 +722,7 @@ def test_create_or_update_app_existing_app_updates_fields( updated_by=None, updated_at=None, ) - service = build_app_dsl_service(db_session_with_containers) + service = AppDslService(db_session_with_containers, drafts=drafts) updated = service._create_or_update_app( app=app, data={ @@ -757,6 +746,15 @@ def test_create_or_update_app_existing_app_updates_fields( assert app.icon_background == "#222222" assert app.updated_at is fixed_now + drafts.sync.assert_called_once() + context, owner, command = drafts.sync.call_args.args + assert context.account_id == _DEFAULT_ACCOUNT_ID + assert context.active_workspace_id == app.tenant_id + assert owner.id == app.id + assert command.graph == {"nodes": []} + assert command.check_hash is False + assert command.clear_debug_variables is True + def test_create_or_update_app_new_app_requires_tenant(self, db_session_with_containers: Session): account = _account() account._current_tenant = None diff --git a/api/tests/test_containers_integration_tests/services/test_app_generate_service.py b/api/tests/test_containers_integration_tests/services/test_app_generate_service.py index 88b8b7be42005d..e20004ac30be39 100644 --- a/api/tests/test_containers_integration_tests/services/test_app_generate_service.py +++ b/api/tests/test_containers_integration_tests/services/test_app_generate_service.py @@ -3,6 +3,7 @@ import json import uuid from collections.abc import Generator +from types import MethodType from typing import TYPE_CHECKING, Literal from unittest.mock import ANY, MagicMock, patch @@ -370,7 +371,12 @@ def test_generate_agent_chat_mode_success( generator_factory.assert_called_once() constructor_args = generator_factory.call_args.kwargs assert constructor_args["tool_invoker"] is workflow_runtime.agent_tool_invoker - assert constructor_args["draft_variable_saver"] is workflow_variables.saver_factory + draft_variable_saver = constructor_args["draft_variable_saver"] + expected_saver = workflow_variables.saver_factory + assert isinstance(draft_variable_saver, MethodType) + assert isinstance(expected_saver, MethodType) + assert draft_variable_saver.__self__ is expected_saver.__self__ + assert draft_variable_saver.__func__ is expected_saver.__func__ assert constructor_args["workflow_runtime"] is workflow_runtime generator = generator_factory.return_value generator.generate.assert_called_once() diff --git a/api/tests/test_containers_integration_tests/services/test_workflow_draft_variable_service.py b/api/tests/test_containers_integration_tests/services/test_workflow_draft_variable_service.py index dfe4a750d1f3d2..1b35c3219d3d66 100644 --- a/api/tests/test_containers_integration_tests/services/test_workflow_draft_variable_service.py +++ b/api/tests/test_containers_integration_tests/services/test_workflow_draft_variable_service.py @@ -231,7 +231,11 @@ def test_get_variable_not_found(self, db_session_with_containers: Session, mock_ service = build_workflow_variable_service( database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) ) - retrieved_variable = service.get_variable(non_existent_id, app_id="missing-app", user_id="missing-user") + retrieved_variable = service.get_variable( + non_existent_id, + app_id=fake.uuid4(), + user_id=fake.uuid4(), + ) assert retrieved_variable is None def test_get_draft_variables_by_selectors_success( diff --git a/api/tests/test_containers_integration_tests/services/test_workflow_run_service.py b/api/tests/test_containers_integration_tests/services/test_workflow_run_service.py index f24cc6f772a9a7..1037cc8fa70e28 100644 --- a/api/tests/test_containers_integration_tests/services/test_workflow_run_service.py +++ b/api/tests/test_containers_integration_tests/services/test_workflow_run_service.py @@ -16,6 +16,7 @@ from models.workflow import WorkflowRun from repositories.factory import DifyAPIRepositoryFactory from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository +from repositories.tools.provider_repository import ToolProviderRepository from services.app_service import AppService, CreateAppParams from services.workflow_run_service import WorkflowRunService from tests.test_containers_integration_tests.helpers import accounts as account_fixtures @@ -36,6 +37,7 @@ def workflow_run_service(self, db_session_with_containers: Session) -> WorkflowR node_executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( session_maker=session_factory ), + tool_providers=ToolProviderRepository(session_factory), ) @pytest.fixture diff --git a/api/tests/test_containers_integration_tests/services/test_workflow_service.py b/api/tests/test_containers_integration_tests/services/test_workflow_service.py index 7b3429a5588aac..bb6c7192c47fdb 100644 --- a/api/tests/test_containers_integration_tests/services/test_workflow_service.py +++ b/api/tests/test_containers_integration_tests/services/test_workflow_service.py @@ -20,7 +20,7 @@ from models.model import AppMode from models.workflow import WorkflowType from repositories.workflow.definition_repository import WorkflowDefinitionStore -from services.workflow.contracts import DraftSyncCommand +from services.workflow.contracts import DraftSyncCommand, WorkflowOwner from services.workflow_ref_service import WorkflowRef from services.workflow_service import WorkflowService from tests.unit_tests.core.model_fixtures import make_model_instance @@ -685,7 +685,7 @@ def test_sync_draft_workflow_create_new(self, db_session_with_containers: Sessio # Act result = drafts.sync( RequestContext("test", None, account.id, app.tenant_id), - app.id, + WorkflowOwner(app.id), DraftSyncCommand( graph=graph, features=features, @@ -754,7 +754,7 @@ def test_sync_draft_workflow_update_existing(self, db_session_with_containers: S # Act result = drafts.sync( RequestContext("test", None, account.id, app.tenant_id), - app.id, + WorkflowOwner(app.id), DraftSyncCommand( graph=new_graph, features=new_features, @@ -819,7 +819,7 @@ def test_sync_draft_workflow_hash_mismatch_error(self, db_session_with_container with pytest.raises(WorkflowHashNotEqualError): drafts.sync( RequestContext("test", None, account.id, app.tenant_id), - app.id, + WorkflowOwner(app.id), DraftSyncCommand( graph=new_graph, features=new_features, diff --git a/api/tests/test_containers_integration_tests/services/tools/test_mcp_tools_manage_service.py b/api/tests/test_containers_integration_tests/services/tools/test_mcp_tools_manage_service.py index ba2ae62a053743..ed070609dcf7b2 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_mcp_tools_manage_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_mcp_tools_manage_service.py @@ -2,11 +2,12 @@ import pytest from faker import Faker -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from core.tools.entities.tool_entities import ToolProviderType from models import Account, Tenant from models.tools import MCPToolProvider +from repositories.tools.provider_repository import ToolProviderRepository from services.tools.mcp_tools_manage_service import UNCHANGED_SERVER_URL_PLACEHOLDER, MCPToolManageService @@ -378,9 +379,7 @@ def test_create_mcp_provider_success(self, db_session_with_containers: Session, ) # Assert: Verify the expected outcomes - assert result is not None - assert result.name == "Test MCP Provider" - assert result.type == ToolProviderType.MCP + assert isinstance(result, str) # Verify database state @@ -391,6 +390,8 @@ def test_create_mcp_provider_success(self, db_session_with_containers: Session, ) assert created_provider is not None + assert result == created_provider.id + assert created_provider.name == "Test MCP Provider" assert created_provider.server_identifier == "test_identifier_123" assert created_provider.timeout == 30.0 assert created_provider.sse_read_timeout == 300.0 @@ -401,7 +402,7 @@ def test_create_mcp_provider_success(self, db_session_with_containers: Session, mock_external_service_dependencies["encrypter"].encrypt_token.assert_called_once_with( tenant.id, "https://example.com/mcp" ) - mock_external_service_dependencies["tool_transform_service"].mcp_provider_to_user_provider.assert_called_once() + mock_external_service_dependencies["tool_transform_service"].mcp_provider_to_user_provider.assert_not_called() def test_create_mcp_provider_duplicate_name( self, db_session_with_containers: Session, mock_external_service_dependencies @@ -638,8 +639,10 @@ def test_retrieve_mcp_tools_success(self, db_session_with_containers: Session, m # Act: Execute the method under test - service = MCPToolManageService(db_session_with_containers) - result = service.list_providers(tenant_id=tenant.id) + tool_providers = ToolProviderRepository( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + result = MCPToolManageService.list_providers(tenant_id=tenant.id, tool_providers=tool_providers) # Assert: Verify the expected outcomes assert result is not None @@ -676,8 +679,10 @@ def test_retrieve_mcp_tools_empty_list( # Act: Execute the method under test - service = MCPToolManageService(db_session_with_containers) - result = service.list_providers(tenant_id=tenant.id) + tool_providers = ToolProviderRepository( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + result = MCPToolManageService.list_providers(tenant_id=tenant.id, tool_providers=tool_providers) # Assert: Verify the expected outcomes assert result is not None @@ -749,9 +754,11 @@ def test_retrieve_mcp_tools_tenant_isolation( # Act: Execute the method under test for both tenants - service = MCPToolManageService(db_session_with_containers) - result1 = service.list_providers(tenant_id=tenant1.id) - result2 = service.list_providers(tenant_id=tenant2.id) + tool_providers = ToolProviderRepository( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + result1 = MCPToolManageService.list_providers(tenant_id=tenant1.id, tool_providers=tool_providers) + result2 = MCPToolManageService.list_providers(tenant_id=tenant2.id, tool_providers=tool_providers) # Assert: Verify tenant isolation assert len(result1) == 1 diff --git a/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py b/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py index 7541d158ca9e7e..896bb08c07f2a4 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py @@ -4,7 +4,7 @@ import pytest from faker import Faker -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from core.app.apps.draft_variable_saver import DraftVariableSaverFactory from core.plugin.plugin_service import PluginService @@ -18,6 +18,7 @@ ToolProviderType, ) from models.tools import ApiToolProvider, WorkflowToolProvider +from repositories.tools.provider_repository import ToolProviderRepository from services.tools.tools_transform_service import ToolTransformService from tests.tool_fixtures import make_runtime_tool @@ -608,7 +609,12 @@ def test_api_provider_to_controller_success( db_session_with_containers.commit() # Act: Execute the method under test - result = ToolTransformService.api_provider_to_controller(provider) + tool_providers = ToolProviderRepository( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + provider_record = tool_providers.get(tenant_id=provider.tenant_id, provider_id=provider.id) + assert provider_record is not None + result = ToolTransformService.api_provider_to_controller(provider_record) # Assert: Verify the expected outcomes assert result is not None @@ -644,7 +650,12 @@ def test_api_provider_to_controller_api_key_query( db_session_with_containers.commit() # Act: Execute the method under test - result = ToolTransformService.api_provider_to_controller(provider) + tool_providers = ToolProviderRepository( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + provider_record = tool_providers.get(tenant_id=provider.tenant_id, provider_id=provider.id) + assert provider_record is not None + result = ToolTransformService.api_provider_to_controller(provider_record) # Assert: Verify the expected outcomes assert result is not None @@ -680,7 +691,12 @@ def test_api_provider_to_controller_backward_compatibility( db_session_with_containers.commit() # Act: Execute the method under test - result = ToolTransformService.api_provider_to_controller(provider) + tool_providers = ToolProviderRepository( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + provider_record = tool_providers.get(tenant_id=provider.tenant_id, provider_id=provider.id) + assert provider_record is not None + result = ToolTransformService.api_provider_to_controller(provider_record) # Assert: Verify the expected outcomes assert result is not None diff --git a/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py b/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py index 2ca43aa5a11803..a141ac379b2a17 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py @@ -212,9 +212,9 @@ def test_create_workflow_tool_success( # Verify external service calls assert set(db_session_with_containers.scalars(select(ToolLabelBinding.label_name))) == set(tool_labels) - assert ( - mock_external_service_dependencies["tool_transform_service"].workflow_provider_to_controller.call_count == 2 - ) + mock_external_service_dependencies[ + "tool_transform_service" + ].workflow_provider_to_controller.assert_called_once() def test_create_workflow_tool_duplicate_name_error( self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools From 2c16f0fb4f7b9630e4567fe7ff2810c52cdf15f2 Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Fri, 9 Oct 2026 14:21:19 +0800 Subject: [PATCH 07/20] test: verify active DSL dependencies and API provider behavior --- .../services/test_app_dsl_service.py | 7 ++- .../tools/test_tools_transform_service.py | 48 ++++++++++++++++--- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/api/tests/test_containers_integration_tests/services/test_app_dsl_service.py b/api/tests/test_containers_integration_tests/services/test_app_dsl_service.py index 8f0a70657ba2fd..6b32b386d43912 100644 --- a/api/tests/test_containers_integration_tests/services/test_app_dsl_service.py +++ b/api/tests/test_containers_integration_tests/services/test_app_dsl_service.py @@ -56,6 +56,7 @@ ) from services.app_service import AppService, CreateAppParams from services.dsl_version import check_version_compatibility +from services.entities.dsl_entities import AppDslOverwriteStore from services.errors.app import WorkflowNotFoundError from services.workflow.draft_service import WorkflowDraftService from tests.test_containers_integration_tests.helpers import accounts as account_fixtures @@ -722,7 +723,11 @@ def test_create_or_update_app_existing_app_updates_fields( updated_by=None, updated_at=None, ) - service = AppDslService(db_session_with_containers, drafts=drafts) + service = AppDslService( + db_session_with_containers, + drafts=drafts, + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) updated = service._create_or_update_app( app=app, data={ diff --git a/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py b/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py index 896bb08c07f2a4..97c179480d4be2 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py @@ -617,9 +617,23 @@ def test_api_provider_to_controller_success( result = ToolTransformService.api_provider_to_controller(provider_record) # Assert: Verify the expected outcomes - assert result is not None - assert hasattr(result, "from_db") - # Additional assertions would depend on the actual controller implementation + assert result.provider_id == provider_record.id + assert result.tenant_id == provider_record.tenant_id + assert result.provider_type is ToolProviderType.API + assert result.entity.identity.name == provider_record.name + assert result.entity.identity.author == provider_record.author + assert result.entity.identity.icon == provider_record.icon + assert result.tools == [] + assert provider_record.credentials == {"auth_type": "api_key_header", "api_key": "test_key"} + credentials_schema = {config.name: config for config in result.entity.credentials_schema} + assert set(credentials_schema) == { + "auth_type", + "api_key_header", + "api_key_value", + "api_key_header_prefix", + } + assert credentials_schema["api_key_header"].default == "Authorization" + assert credentials_schema["api_key_header_prefix"].default == "basic" def test_api_provider_to_controller_api_key_query( self, db_session_with_containers: Session, mock_external_service_dependencies @@ -658,8 +672,16 @@ def test_api_provider_to_controller_api_key_query( result = ToolTransformService.api_provider_to_controller(provider_record) # Assert: Verify the expected outcomes - assert result is not None - assert hasattr(result, "from_db") + assert result.provider_id == provider_record.id + assert result.tenant_id == provider_record.tenant_id + assert result.provider_type is ToolProviderType.API + assert result.entity.identity.name == provider_record.name + assert result.entity.identity.author == provider_record.author + assert result.tools == [] + assert provider_record.credentials == {"auth_type": "api_key_query", "api_key": "test_key"} + credentials_schema = {config.name: config for config in result.entity.credentials_schema} + assert set(credentials_schema) == {"auth_type", "api_key_query_param", "api_key_value"} + assert credentials_schema["api_key_query_param"].default == "key" def test_api_provider_to_controller_backward_compatibility( self, db_session_with_containers: Session, mock_external_service_dependencies @@ -699,8 +721,20 @@ def test_api_provider_to_controller_backward_compatibility( result = ToolTransformService.api_provider_to_controller(provider_record) # Assert: Verify the expected outcomes - assert result is not None - assert hasattr(result, "from_db") + assert result.provider_id == provider_record.id + assert result.tenant_id == provider_record.tenant_id + assert result.provider_type is ToolProviderType.API + assert result.entity.identity.name == provider_record.name + assert result.entity.identity.author == provider_record.author + assert result.tools == [] + assert provider_record.credentials == {"auth_type": "api_key", "api_key": "test_key"} + credentials_schema = {config.name: config for config in result.entity.credentials_schema} + assert set(credentials_schema) == { + "auth_type", + "api_key_header", + "api_key_value", + "api_key_header_prefix", + } def test_workflow_provider_to_controller_success( self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_queries From 5c33efbb9ac22e9da31e7491cb081131c399f278 Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Thu, 15 Oct 2026 16:33:22 +0800 Subject: [PATCH 08/20] fix(workflow): address review findings in #43680 --- .../app/apps/advanced_chat/app_generator.py | 8 + api/core/app/apps/workflow/app_generator.py | 8 + .../app/layers/pause_state_persist_layer.py | 15 +- api/services/app_generate_service.py | 13 +- api/services/message_service.py | 6 +- api/services/plugin/app_invocation.py | 10 +- .../rag_pipeline/execution_service.py | 15 +- api/services/rag_pipeline/rag_pipeline.py | 4 +- api/services/snippet_generate_service.py | 36 ++-- .../adapters/chatflow/app_generator.py | 1 + .../adapters/workflow/app_generator.py | 1 + .../execution/adapters/workflow_entry.py | 23 +-- api/services/workflow/runtime_gateway.py | 18 +- api/services/workflow/variable_contracts.py | 6 +- api/services/workflow/variable_service.py | 9 +- api/services/workflow_service.py | 24 ++- api/tasks/async_workflow_tasks.py | 11 +- .../layers/test_pause_state_persist_layer.py | 5 +- .../layers/test_pause_state_persist_layer.py | 26 ++- .../core/tools/workflow_as_tool/test_tool.py | 8 +- .../test_pipeline_generate_service.py | 16 +- .../rag_pipeline/test_rag_pipeline_service.py | 44 +++-- .../services/test_app_generate_service.py | 32 ++-- .../services/test_snippet_generate_service.py | 36 ++-- .../services/workflow/test_runtime_gateway.py | 2 +- .../workflow/test_runtime_variable_files.py | 11 +- .../workflow/test_variable_service.py | 5 +- .../tasks/test_async_workflow_tasks.py | 164 ++++++++++++++++++ 28 files changed, 403 insertions(+), 154 deletions(-) diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.py index aa9636837342dd..1934cd21cef288 100644 --- a/api/core/app/apps/advanced_chat/app_generator.py +++ b/api/core/app/apps/advanced_chat/app_generator.py @@ -56,6 +56,7 @@ from libs.flask_utils import preserve_flask_contexts from models import Account, App, Conversation, EndUser, Message, Workflow, WorkflowNodeExecutionTriggeredFrom from models.enums import WorkflowRunTriggeredFrom +from repositories.human_input.form_repository import HumanInputFormSubmissionRepository from services.conversation_service import ConversationService from services.errors.conversation import ConversationNotExistsError from services.workflow_draft_variable_service import ( @@ -581,6 +582,13 @@ def _generate( PauseStatePersistenceLayer( session_factory=pause_state_config.session_factory, generate_entity=application_generate_entity, + human_form_reader=HumanInputFormSubmissionRepository( + sessions=( + pause_state_config.session_factory + if isinstance(pause_state_config.session_factory, sessionmaker) + else sessionmaker(pause_state_config.session_factory) + ) + ), state_owner_user_id=pause_state_config.state_owner_user_id, response_stream_filter=resolved_response_stream_filter, ) diff --git a/api/core/app/apps/workflow/app_generator.py b/api/core/app/apps/workflow/app_generator.py index 65d79dd0d35d36..e9e99267c86666 100644 --- a/api/core/app/apps/workflow/app_generator.py +++ b/api/core/app/apps/workflow/app_generator.py @@ -55,6 +55,7 @@ from models.enums import WorkflowRunTriggeredFrom from models.model import App, EndUser from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom +from repositories.human_input.form_repository import HumanInputFormSubmissionRepository from services.workflow_draft_variable_service import DraftVarLoader, WorkflowDraftVariableService if TYPE_CHECKING: @@ -367,6 +368,13 @@ def _generate( PauseStatePersistenceLayer( session_factory=pause_state_config.session_factory, generate_entity=application_generate_entity, + human_form_reader=HumanInputFormSubmissionRepository( + sessions=( + pause_state_config.session_factory + if isinstance(pause_state_config.session_factory, sessionmaker) + else sessionmaker(pause_state_config.session_factory) + ) + ), state_owner_user_id=pause_state_config.state_owner_user_id, response_stream_filter=resolved_response_stream_filter, ) diff --git a/api/core/app/layers/pause_state_persist_layer.py b/api/core/app/layers/pause_state_persist_layer.py index 02c665623fbfc4..89bf287aea736f 100644 --- a/api/core/app/layers/pause_state_persist_layer.py +++ b/api/core/app/layers/pause_state_persist_layer.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from dataclasses import dataclass -from typing import Annotated, Literal, Self, override +from typing import TYPE_CHECKING, Annotated, Literal, Self, override from pydantic import BaseModel, Field from sqlalchemy import Engine @@ -13,9 +15,11 @@ from models.model import AppMode from repositories.api_workflow_run_repository import APIWorkflowRunRepository from repositories.factory import DifyAPIRepositoryFactory -from repositories.human_input.form_repository import HumanInputFormSubmissionRepository from services.workflow.execution.adapters.human_input_events import enrich_graph_pause_reasons +if TYPE_CHECKING: + from services.human_input.ports import HumanInputFormReader + # Wrapper types for `WorkflowAppGenerateEntity` and # `AdvancedChatAppGenerateEntity`. These wrappers enable type discrimination @@ -79,6 +83,7 @@ def __init__( self, session_factory: Engine | sessionmaker[Session], generate_entity: WorkflowAppGenerateEntity | AdvancedChatAppGenerateEntity, + human_form_reader: HumanInputFormReader, state_owner_user_id: str, response_stream_filter: ResponseStreamFilter, ): @@ -87,6 +92,9 @@ def __init__( The `state_owner_user_id` is used when creating state file for pause. It generally should id of the creator of workflow. + `human_form_reader` must use the same persistence wiring as the workflow + runtime so pause-reason enrichment reads the forms created by this run. + `response_stream_filter` must be the exact same instance that `WorkflowEntry` is using to stream this run's events — this layer dumps its state on pause, and a different instance would silently @@ -98,6 +106,7 @@ def __init__( self._session_maker = session_factory self._state_owner_user_id = state_owner_user_id self._generate_entity = generate_entity + self._human_form_reader = human_form_reader self._response_stream_filter = response_stream_filter def _get_repo(self) -> APIWorkflowRunRepository: @@ -152,7 +161,7 @@ def on_event(self, event: GraphEngineEvent) -> None: # here so repository/model layers only handle Dify-owned pause reasons. pause_reasons = enrich_graph_pause_reasons( reasons=event.reasons, - form_repository=HumanInputFormSubmissionRepository(), + form_repository=self._human_form_reader, variable_pool=self.graph_runtime_state.variable_pool, ) repo = self._get_repo() diff --git a/api/services/app_generate_service.py b/api/services/app_generate_service.py index eb543c2bf85c81..0fa3db4f40dc53 100644 --- a/api/services/app_generate_service.py +++ b/api/services/app_generate_service.py @@ -19,7 +19,6 @@ from extensions.otel import AppGenerateHandler, trace_span from models.annotation_reply import AnnotationReplies from models.model import Account, App, AppMode, EndUser -from models.workflow import Workflow from repositories.workflow.definition_repository import WorkflowDefinitionStore from services.app.generation.adapters.agent_app import AgentAppGenerator from services.app.generation.adapters.agent_chat import AgentChatAppGenerator @@ -163,7 +162,7 @@ def generate_workflow_stream( *, app_model: App, user: Account | EndUser, - workflow: Workflow, + workflow: WorkflowSnapshot, args: Mapping[str, Any], invoke_from: InvokeFrom, variables: WorkflowExecutionVariables, @@ -205,7 +204,7 @@ def _stream_workflow( *, app_model: App, user: Account | EndUser, - workflow: Workflow, + workflow: WorkflowSnapshot, args: Mapping[str, Any], invoke_from: InvokeFrom, variables: WorkflowExecutionVariables, @@ -480,7 +479,7 @@ def _dispatch_generate( raise ValueError(f"Invalid app mode {app_model.mode}") @staticmethod - def _ensure_workflow_service_mode_available(*, workflow: Workflow, invoke_from: InvokeFrom) -> None: + def _ensure_workflow_service_mode_available(*, workflow: WorkflowSnapshot, invoke_from: InvokeFrom) -> None: if invoke_from not in _MANUAL_WORKFLOW_INVOKE_SOURCES: return @@ -518,7 +517,7 @@ def generate_single_iteration( node_id: str, args: Any, *, - workflow: Workflow, + workflow: WorkflowSnapshot, variables: WorkflowExecutionVariables, runtime: AppGenerationRuntime, streaming: bool = True, @@ -569,7 +568,7 @@ def generate_single_loop( node_id: str, args: Mapping[str, Any], *, - workflow: Workflow, + workflow: WorkflowSnapshot, variables: WorkflowExecutionVariables, runtime: AppGenerationRuntime, streaming: bool = True, @@ -655,7 +654,7 @@ def _get_workflow( *, session: Session, snapshot: WorkflowSnapshot | None = None, - ) -> Workflow: + ) -> WorkflowSnapshot: """ Get workflow :param app_model: app model diff --git a/api/services/message_service.py b/api/services/message_service.py index 2a89416c421d07..748844cbeb2b88 100644 --- a/api/services/message_service.py +++ b/api/services/message_service.py @@ -37,6 +37,7 @@ from repositories.sqlalchemy_execution_extra_content_repository import ( SQLAlchemyExecutionExtraContentRepository, ) +from repositories.workflow.definition_repository import workflow_snapshot from services.conversation_service import ConversationService from services.errors.message import ( FirstMessageNotExistsError, @@ -384,7 +385,8 @@ def prepare_suggested_questions_after_answer( if workflow is None: return None - app_config = AdvancedChatAppConfigManager.get_app_config(app_model=app_model, workflow=workflow) + workflow_definition = workflow_snapshot(workflow) + app_config = AdvancedChatAppConfigManager.get_app_config(app_model=app_model, workflow=workflow_definition) if not app_config.additional_features: raise ValueError("Additional features not found") @@ -392,7 +394,7 @@ def prepare_suggested_questions_after_answer( if not app_config.additional_features.suggested_questions_after_answer: raise SuggestedQuestionsAfterAnswerDisabledError() - suggested_questions_after_answer = workflow.features_dict.get("suggested_questions_after_answer") + suggested_questions_after_answer = workflow_definition.features_dict.get("suggested_questions_after_answer") if isinstance(suggested_questions_after_answer, dict): suggested_questions_after_answer_config = cast( SuggestedQuestionsAfterAnswerConfig, suggested_questions_after_answer diff --git a/api/services/plugin/app_invocation.py b/api/services/plugin/app_invocation.py index ee0ba817818d84..56299a96ed59d7 100644 --- a/api/services/plugin/app_invocation.py +++ b/api/services/plugin/app_invocation.py @@ -22,12 +22,14 @@ load_annotation_reply_config, ) from models.workflow import Workflow +from repositories.workflow.definition_repository import workflow_snapshot from services.app.generation.adapters.agent_chat import AgentChatAppGenerator from services.app.generation.adapters.chat import ChatAppGenerator from services.app.generation.adapters.completion import CompletionAppGenerator from services.app.generation.ports import ChatRecords from services.app.generation.runtime import AppGenerationRuntime from services.knowledge.retrieval.ports import DatasetRetrievalFactory +from services.workflow.contracts import WorkflowSnapshot from services.workflow.execution.adapters.chatflow.app_generator import AdvancedChatAppGenerator from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator @@ -215,7 +217,7 @@ def invoke_chat_app( def invoke_workflow_app( cls, app: App, - workflow: Workflow, + workflow: WorkflowSnapshot, user: EndUser | Account, stream: bool, inputs: Mapping, @@ -323,7 +325,7 @@ def _get_app(cls, app_id: str, tenant_id: str) -> App: return app @classmethod - def _get_workflow(cls, app: App) -> Workflow | None: + def _get_workflow(cls, app: App) -> WorkflowSnapshot | None: """ get workflow with an owned session, additionally scoped by tenant and app @@ -339,9 +341,7 @@ def _get_workflow(cls, app: App) -> Workflow | None: .where(Workflow.id == app.workflow_id, Workflow.tenant_id == app.tenant_id, Workflow.app_id == app.id) .limit(1) ) - if workflow: - session.expunge(workflow) - return workflow + return workflow_snapshot(workflow) if workflow is not None else None @classmethod def _get_app_model_config_dict(cls, app: App) -> AppModelConfigDict | None: diff --git a/api/services/rag_pipeline/execution_service.py b/api/services/rag_pipeline/execution_service.py index d057310179750a..f6326a9c6e2713 100644 --- a/api/services/rag_pipeline/execution_service.py +++ b/api/services/rag_pipeline/execution_service.py @@ -49,10 +49,11 @@ WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, ) +from repositories.workflow.definition_repository import workflow_snapshot from services.data_source.provider_service import DatasourceProviderService from services.workflow.execution.adapters.workflow_entry import WorkflowEntry from services.workflow.execution.ports import WorkflowRuntime -from services.workflow.variable_contracts import WorkflowExecutionVariables +from services.workflow.variable_contracts import WorkflowExecutionVariables, environment_variable_values logger = logging.getLogger(__name__) @@ -113,9 +114,10 @@ def run_draft_workflow_node( Run draft workflow node """ # fetch draft workflow by app_model - draft_workflow = self._workflows.get_draft_workflow(pipeline=pipeline) - if not draft_workflow: + draft_workflow_model = self._workflows.get_draft_workflow(pipeline=pipeline) + if not draft_workflow_model: raise ValueError("Workflow not initialized") + draft_workflow = workflow_snapshot(draft_workflow_model) # run draft workflow node start_at = time.perf_counter() @@ -139,7 +141,7 @@ def run_draft_workflow_node( variable_pool=_build_seeded_variable_pool( build_bootstrap_variables( system_variables=default_system_variables(), - environment_variables=draft_workflow.environment_variables, + environment_variables=environment_variable_values(draft_workflow), ) ), variable_loader=self._variables.workflow_loader(draft_workflow, account.id), @@ -574,9 +576,10 @@ def set_datasource_variables(self, pipeline: Pipeline, args: dict[str, Any], cur """ # fetch draft workflow by app_model - draft_workflow = self._workflows.get_draft_workflow(pipeline=pipeline) - if not draft_workflow: + draft_workflow_model = self._workflows.get_draft_workflow(pipeline=pipeline) + if not draft_workflow_model: raise ValueError("Workflow not initialized") + draft_workflow = workflow_snapshot(draft_workflow_model) # run draft workflow node start_at = time.perf_counter() diff --git a/api/services/rag_pipeline/rag_pipeline.py b/api/services/rag_pipeline/rag_pipeline.py index 29d6b9c9044910..c8e042ca38fbd2 100644 --- a/api/services/rag_pipeline/rag_pipeline.py +++ b/api/services/rag_pipeline/rag_pipeline.py @@ -33,7 +33,7 @@ WorkflowRun, ) from repositories.factory import DifyAPIRepositoryFactory -from repositories.workflow.definition_repository import WorkflowDefinitionStore, workflow_record +from repositories.workflow.definition_repository import WorkflowDefinitionStore, workflow_record, workflow_snapshot from services.credentials.query import CredentialQuery from services.data_source.provider_service import DatasourceProviderService from services.entities.knowledge_entities.rag_pipeline_entities import ( @@ -643,7 +643,7 @@ def retry_error_document( raise ValueError("Workflow not found") generator.generate( pipeline=pipeline, - workflow=workflow, + workflow=workflow_snapshot(workflow), user=user, args={ "inputs": document_pipeline_execution_log.input_data, diff --git a/api/services/snippet_generate_service.py b/api/services/snippet_generate_service.py index 84a278a1e47f5c..811acdba8a2990 100644 --- a/api/services/snippet_generate_service.py +++ b/api/services/snippet_generate_service.py @@ -21,10 +21,9 @@ import json import logging from collections.abc import Generator, Mapping, Sequence +from dataclasses import replace from typing import Any, Protocol, Union, cast -from sqlalchemy.orm import make_transient - from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.app.entities.app_invoke_entities import InvokeFrom from core.app.file_access import DatabaseFileAccessController @@ -35,7 +34,9 @@ from models.model import App, AppMode, EndUser from models.snippet import CustomizedSnippet from models.workflow import Workflow, WorkflowNodeExecutionModel +from repositories.workflow.definition_repository import workflow_snapshot from services.app.generation.response import convert_to_event_stream +from services.workflow.contracts import WorkflowSnapshot from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator from services.workflow.execution.ports import WorkflowRuntime from services.workflow.variable_contracts import WorkflowExecutionVariables @@ -173,9 +174,10 @@ def generate( :return: Blocking response mapping or SSE streaming generator :raises ValueError: If the snippet has no draft workflow """ - workflow = self._snippets.get_draft_workflow(snippet=snippet) - if not workflow: + workflow_model = self._snippets.get_draft_workflow(snippet=snippet) + if not workflow_model: raise ValueError("Workflow not initialized") + workflow = workflow_snapshot(workflow_model) # Inject a virtual Start node when the graph doesn't have one. workflow = self._ensure_start_node(workflow, snippet) @@ -200,14 +202,13 @@ def generate( return convert_to_event_stream(self._filter_virtual_start_events(response)) @classmethod - def _ensure_start_node(cls, workflow: Workflow, snippet: CustomizedSnippet) -> Workflow: + def _ensure_start_node(cls, workflow: WorkflowSnapshot, snippet: CustomizedSnippet) -> WorkflowSnapshot: """ Return *workflow* with a Start node. If the graph already contains a Start node, the original workflow is returned unchanged. Otherwise a virtual Start node is injected and the - workflow object is detached from the SQLAlchemy session so the in-memory - change is never flushed to the database. + returned snapshot contains the in-memory graph used for this execution. """ graph_dict = workflow.graph_dict nodes: list[dict[str, Any]] = graph_dict.get("nodes", []) @@ -221,11 +222,7 @@ def _ensure_start_node(cls, workflow: Workflow, snippet: CustomizedSnippet) -> W input_fields=snippet.input_fields_list, ) - # Detach from session to prevent accidental DB persistence of the - # modified graph. All attributes remain accessible for read. - make_transient(workflow) - workflow.graph = json.dumps(modified_graph) - return workflow + return replace(workflow, graph=json.dumps(modified_graph)) @classmethod def ensure_start_node_for_worker(cls, workflow: Workflow, snippet: CustomizedSnippet) -> Workflow: @@ -324,9 +321,10 @@ def run_draft_node( :return: WorkflowNodeExecutionModel with execution results :raises ValueError: If the snippet has no draft workflow """ - draft_workflow = self._snippets.get_draft_workflow(snippet=snippet) - if not draft_workflow: + draft_workflow_model = self._snippets.get_draft_workflow(snippet=snippet) + if not draft_workflow_model: raise ValueError("Workflow not initialized") + draft_workflow = workflow_snapshot(draft_workflow_model) app_proxy = cast(App, _SnippetAsApp(snippet)) @@ -364,9 +362,10 @@ def generate_single_iteration( :return: SSE streaming generator :raises ValueError: If the snippet has no draft workflow """ - workflow = self._snippets.get_draft_workflow(snippet=snippet) - if not workflow: + workflow_model = self._snippets.get_draft_workflow(snippet=snippet) + if not workflow_model: raise ValueError("Workflow not initialized") + workflow = workflow_snapshot(workflow_model) app_proxy = cast(App, _SnippetAsApp(snippet)) @@ -408,9 +407,10 @@ def generate_single_loop( :return: SSE streaming generator :raises ValueError: If the snippet has no draft workflow """ - workflow = self._snippets.get_draft_workflow(snippet=snippet) - if not workflow: + workflow_model = self._snippets.get_draft_workflow(snippet=snippet) + if not workflow_model: raise ValueError("Workflow not initialized") + workflow = workflow_snapshot(workflow_model) app_proxy = cast(App, _SnippetAsApp(snippet)) diff --git a/api/services/workflow/execution/adapters/chatflow/app_generator.py b/api/services/workflow/execution/adapters/chatflow/app_generator.py index df0028936c6704..5c4c4948f89bf9 100644 --- a/api/services/workflow/execution/adapters/chatflow/app_generator.py +++ b/api/services/workflow/execution/adapters/chatflow/app_generator.py @@ -543,6 +543,7 @@ def _generate( PauseStatePersistenceLayer( session_factory=pause_state_config.session_factory, generate_entity=application_generate_entity, + human_form_reader=self._runtime.human_form_reader, state_owner_user_id=pause_state_config.state_owner_user_id, response_stream_filter=resolved_response_stream_filter, ) diff --git a/api/services/workflow/execution/adapters/workflow/app_generator.py b/api/services/workflow/execution/adapters/workflow/app_generator.py index 1a5bec4a42f749..3f4e22259bfd19 100644 --- a/api/services/workflow/execution/adapters/workflow/app_generator.py +++ b/api/services/workflow/execution/adapters/workflow/app_generator.py @@ -352,6 +352,7 @@ def _generate( PauseStatePersistenceLayer( session_factory=pause_state_config.session_factory, generate_entity=application_generate_entity, + human_form_reader=self._runtime.human_form_reader, state_owner_user_id=pause_state_config.state_owner_user_id, response_stream_filter=resolved_response_stream_filter, ) diff --git a/api/services/workflow/execution/adapters/workflow_entry.py b/api/services/workflow/execution/adapters/workflow_entry.py index 6057594df0806e..eb371bdafc7def 100644 --- a/api/services/workflow/execution/adapters/workflow_entry.py +++ b/api/services/workflow/execution/adapters/workflow_entry.py @@ -1,7 +1,7 @@ import logging import time from collections.abc import Callable, Generator, Mapping, Sequence -from typing import Any, Protocol, TypedDict +from typing import Any, TypedDict from uuid import uuid4 from configs import dify_config @@ -43,6 +43,7 @@ from models import Account from models.agent_runtime_contracts import WorkflowAgentRuntimeBindings from services.human_input.ports import HumanInputFormReader +from services.workflow.contracts import WorkflowSnapshot from services.workflow.execution.adapters.human_input_events import HumanInputFormEventFilter from services.workflow.execution.adapters.node_factory import ( DifyGraphInitContext, @@ -104,24 +105,6 @@ class SingleNodeGraphDict(TypedDict): edges: list[_EdgeConfigDict] -class _WorkflowEntryDefinition(Protocol): - """Narrow workflow view shared by ORM rows and detached snapshots during migration.""" - - @property - def id(self) -> str: ... - - @property - def tenant_id(self) -> str: ... - - @property - def app_id(self) -> str: ... - - @property - def graph_dict(self) -> Mapping[str, Any]: ... - - def get_node_config_by_id(self, node_id: str) -> Mapping[str, Any]: ... - - class WorkflowEntry: def __init__( self, @@ -230,7 +213,7 @@ def run(self) -> Generator[GraphEngineEvent, None, None]: def single_step_run( cls, *, - workflow: _WorkflowEntryDefinition, + workflow: WorkflowSnapshot, node_id: str, user_id: str, user_inputs: Mapping[str, Any], diff --git a/api/services/workflow/runtime_gateway.py b/api/services/workflow/runtime_gateway.py index 11acbaa852a3ef..a545370f7dd57a 100644 --- a/api/services/workflow/runtime_gateway.py +++ b/api/services/workflow/runtime_gateway.py @@ -19,14 +19,10 @@ from graphon.graph_engine.manager import GraphEngineManager from machinery.context import RequestContext from models import Account, App -from models.workflow import Workflow, WorkflowNodeExecutionModel +from models.workflow import WorkflowNodeExecutionModel from repositories.agent.retirement_repository import WorkflowAgentRetirementRepository from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository -from repositories.workflow.definition_repository import ( - WorkflowDefinitionRepository, - workflow_from_snapshot, - workflow_snapshot, -) +from repositories.workflow.definition_repository import WorkflowDefinitionRepository from repositories.workflow.node_execution_repository import WorkflowNodeExecutionRepository from services.agent.retirement_service import WorkflowAgentRetirementService from services.app.generation.runtime import AppGenerationRuntime @@ -150,13 +146,15 @@ def loop( streaming=True, ) - def _draft(self, context: RequestContext, app_id: str, missing_message: str) -> tuple[App, Account, Workflow]: + def _draft( + self, context: RequestContext, app_id: str, missing_message: str + ) -> tuple[App, Account, WorkflowSnapshot]: try: return self._definitions.debug_context(context, app_id, workflow_id=None, snapshot=None) except ValueError as error: raise ValueError(missing_message) from error - def _files(self, workflow: Workflow, files: list[dict[str, Any]] | None) -> Sequence[File]: + def _files(self, workflow: WorkflowSnapshot, files: list[dict[str, Any]] | None) -> Sequence[File]: config = FileUploadConfigManager.convert(workflow.features_dict, is_vision=False) if config is None: return [] @@ -215,10 +213,10 @@ def poll_trigger( ) -> WorkflowTriggerEvent | None: if single_node: app, _, workflow = self._draft(context, app_id, "Workflow not found") - snapshot = workflow_snapshot(workflow) + snapshot = workflow else: app, snapshot = self._reservations.reserve_trigger_debug(context, app_id) - workflow = workflow_from_snapshot(snapshot) + workflow = snapshot if single_node: node_id = node_ids[0] config = workflow.get_node_config_by_id(node_id=node_id) diff --git a/api/services/workflow/variable_contracts.py b/api/services/workflow/variable_contracts.py index bf844aa81bfda8..3bcceb6eff8f19 100644 --- a/api/services/workflow/variable_contracts.py +++ b/api/services/workflow/variable_contracts.py @@ -13,7 +13,7 @@ from graphon.variables.variables import Variable from models import Account, App from models.workflow import Workflow, WorkflowDraftVariable, WorkflowNodeExecutionModel -from services.workflow.contracts import WorkflowSnapshot +from services.workflow.contracts import WorkflowBindingScope, WorkflowSnapshot def environment_variable_values(workflow: WorkflowSnapshot) -> Sequence[Variable]: @@ -32,9 +32,9 @@ def conversation_variable_values(workflow: WorkflowSnapshot) -> Sequence[Variabl class WorkflowExecutionVariables(Protocol): - def workflow_loader(self, workflow: Workflow, user_id: str) -> VariableLoader: ... + def workflow_loader(self, workflow: WorkflowSnapshot, user_id: str) -> VariableLoader: ... def saver_factory(self, tenant_id: str, account: Account) -> DraftVariableSaverFactory: ... - def get_or_create_conversation(self, account_id: str, app: App, workflow: Workflow) -> str: ... + def get_or_create_conversation(self, account_id: str, app: App, workflow: WorkflowBindingScope) -> str: ... def load_execution_outputs(self, execution: WorkflowNodeExecutionModel) -> Mapping[str, Any] | None: ... diff --git a/api/services/workflow/variable_service.py b/api/services/workflow/variable_service.py index 21f7728d6d0a83..058c1b36540d9c 100644 --- a/api/services/workflow/variable_service.py +++ b/api/services/workflow/variable_service.py @@ -22,6 +22,7 @@ WorkflowNodeExecutionModel, is_system_variable_editable, ) +from services.workflow.contracts import WorkflowBindingScope, WorkflowSnapshot from services.workflow.draft_variable_service import ( DraftFileRestorer, DraftFileUploader, @@ -33,7 +34,7 @@ VariableResetError, conversation_defaults, ) -from services.workflow.variable_contracts import WorkflowDraftVariableList +from services.workflow.variable_contracts import WorkflowDraftVariableList, conversation_variable_values logger = logging.getLogger(__name__) @@ -150,7 +151,7 @@ def get_or_create_conversation( self, account_id: str, app: App, - workflow: Workflow, + workflow: WorkflowBindingScope, ) -> str: """ get_or_create_conversation creates and returns the ID of a conversation for debugging. @@ -297,8 +298,8 @@ def _loader( file_inputs=self._file_inputs, ) - def workflow_loader(self, workflow: Workflow, user_id: str) -> VariableLoader: - return self._loader(workflow.app_id, workflow.tenant_id, user_id, workflow.conversation_variables) + def workflow_loader(self, workflow: WorkflowSnapshot, user_id: str) -> VariableLoader: + return self._loader(workflow.app_id, workflow.tenant_id, user_id, conversation_variable_values(workflow)) def saver_factory(self, tenant_id: str, account: Account) -> DraftVariableSaverFactory: return partial( diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index 51847dc348abb1..33206fc7ae1620 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -1,7 +1,7 @@ import time import uuid from collections.abc import Callable, Generator, Mapping, Sequence -from typing import Any, cast +from typing import Any, Protocol, cast from sqlalchemy.orm import Session, sessionmaker @@ -59,6 +59,7 @@ TriggerNodeLimitExceededError, ) from services.system_feature_service import SystemFeatureService +from services.workflow.contracts import WorkflowSnapshot from services.workflow.execution.adapters.chatflow.app_config_manager import AdvancedChatAppConfigManager from services.workflow.execution.adapters.node_factory import ( LATEST_VERSION, @@ -67,13 +68,18 @@ ) from services.workflow.execution.adapters.workflow.app_config_manager import WorkflowAppConfigManager from services.workflow.execution.adapters.workflow_entry import WorkflowEntry -from services.workflow.variable_contracts import WorkflowExecutionVariables +from services.workflow.variable_contracts import WorkflowExecutionVariables, environment_variable_values _file_access_controller = DatabaseFileAccessController() from services.workflow.execution.ports import WorkflowRuntime +class WorkflowIdentity(Protocol): + @property + def id(self) -> str: ... + + class WorkflowService: """ Workflow Service @@ -94,7 +100,9 @@ def __init__( session_maker ) - def get_node_last_run(self, app_model: App, workflow: Workflow, node_id: str) -> WorkflowNodeExecutionModel | None: + def get_node_last_run( + self, app_model: App, workflow: WorkflowIdentity, node_id: str + ) -> WorkflowNodeExecutionModel | None: """ Get the most recent execution for a specific node. @@ -480,7 +488,7 @@ def get_default_block_config( def run_draft_workflow_node( self, app_model: App, - draft_workflow: Workflow, + draft_workflow: WorkflowSnapshot, node_id: str, user_inputs: Mapping[str, Any], account: Account, @@ -495,7 +503,7 @@ def run_draft_workflow_node( files = files or [] node_config = draft_workflow.get_node_config_by_id(node_id) - node_type = Workflow.get_node_type_from_node_config(node_config) + node_type = draft_workflow.get_node_type_from_node_config(node_config) node_data = node_config["data"] if is_start_node_type(node_type): conversation_id = variables.get_or_create_conversation( @@ -528,7 +536,7 @@ def run_draft_workflow_node( variable_pool, build_bootstrap_variables( system_variables=default_system_variables(), - environment_variables=draft_workflow.environment_variables, + environment_variables=environment_variable_values(draft_workflow), ), ) @@ -823,7 +831,7 @@ def _setup_variable_pool( files: Sequence[File], user_id: str, user_inputs: Mapping[str, Any], - workflow: Workflow, + workflow: WorkflowSnapshot, node_id: str, node_type: NodeType, conversation_id: str, @@ -860,7 +868,7 @@ def _setup_variable_pool( variable_pool, build_bootstrap_variables( system_variables=system_variable, - environment_variables=workflow.environment_variables, + environment_variables=environment_variable_values(workflow), conversation_variables=cast(list[Variable], conversation_variables), ), ) diff --git a/api/tasks/async_workflow_tasks.py b/api/tasks/async_workflow_tasks.py index 2c2c1d09be0a6f..f2b00f5b01b750 100644 --- a/api/tasks/async_workflow_tasks.py +++ b/api/tasks/async_workflow_tasks.py @@ -31,6 +31,7 @@ from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom, WorkflowRun from repositories.factory import DifyAPIRepositoryFactory from repositories.sqlalchemy_workflow_trigger_log_repository import SQLAlchemyWorkflowTriggerLogRepository +from repositories.workflow.definition_repository import workflow_snapshot from services.errors.app import WorkflowNotFoundError from services.workflow.entities import ( TriggerData, @@ -139,9 +140,10 @@ def _execute_workflow_common( if not app_model: raise WorkflowNotFoundError(f"App not found: {trigger_log.app_id}") - workflow = session.scalar(select(Workflow).where(Workflow.id == trigger_log.workflow_id)) - if not workflow: + workflow_model = session.scalar(select(Workflow).where(Workflow.id == trigger_log.workflow_id)) + if not workflow_model: raise WorkflowNotFoundError(f"Workflow not found: {trigger_log.workflow_id}") + workflow = workflow_snapshot(workflow_model) user = _get_user(session, trigger_log) @@ -231,11 +233,12 @@ def resume_workflow_execution(task_data_dict: dict[str, Any]) -> None: response_stream_filter = resumption_context.get_response_stream_filter() with session_factory() as session: - workflow = session.scalar(select(Workflow).where(Workflow.id == workflow_run.workflow_id)) - if workflow is None: + workflow_model = session.scalar(select(Workflow).where(Workflow.id == workflow_run.workflow_id)) + if workflow_model is None: raise WorkflowNotFoundError( f"Workflow not found: workflow_run_id={workflow_run.id}, workflow_id={workflow_run.workflow_id}" ) + workflow = workflow_snapshot(workflow_model) user = _get_user(session, workflow_run) app_model = session.scalar(select(App).where(App.id == workflow_run.app_id)) if app_model is None: diff --git a/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py b/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py index d398d731c2103b..801c95d65d214b 100644 --- a/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py +++ b/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py @@ -48,6 +48,7 @@ from models.model import AppMode, UploadFile from models.workflow import Workflow, WorkflowRun from repositories.factory import DifyAPIRepositoryFactory +from repositories.human_input.form_repository import HumanInputFormSubmissionRepository from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository from repositories.tools.provider_repository import ToolProviderRepository from services.file_service import FileService @@ -326,10 +327,12 @@ def _create_pause_state_persistence_layer( bind = self.session.get_bind() assert isinstance(bind, Engine) + sessions = sessionmaker(bind=bind, expire_on_commit=False) return PauseStatePersistenceLayer( - session_factory=sessionmaker(bind=bind, expire_on_commit=False), + session_factory=sessions, state_owner_user_id=owner_id, generate_entity=entity, + human_form_reader=HumanInputFormSubmissionRepository(sessions=sessions), response_stream_filter=_create_initialized_response_stream_filter(), ) diff --git a/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py b/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py index 697cd531663bc8..8d5342b04564db 100644 --- a/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py +++ b/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py @@ -33,6 +33,7 @@ from graphon.variables.segments import Segment from models.model import AppMode from repositories.factory import DifyAPIRepositoryFactory +from services.human_input.ports import HumanInputFormReader @pytest.fixture @@ -196,6 +197,11 @@ def send_command(self, command: GraphEngineCommand) -> None: self._commands.append(command) +class _NullHumanInputFormReader: + def get_by_form_id(self, form_id: str) -> None: + _ = form_id + + class TestPauseStatePersistenceLayer: """Unit tests for PauseStatePersistenceLayer.""" @@ -220,16 +226,19 @@ def _create_generate_entity(workflow_execution_id: str = "run-123") -> WorkflowA def test_init_with_dependency_injection(self, sqlite_session_factory: sessionmaker[Session]): state_owner_user_id = "user-123" + human_form_reader: HumanInputFormReader = _NullHumanInputFormReader() layer = PauseStatePersistenceLayer( session_factory=sqlite_session_factory, state_owner_user_id=state_owner_user_id, generate_entity=self._create_generate_entity(), + human_form_reader=human_form_reader, response_stream_filter=ResponseStreamFilter(), ) assert layer._session_maker is sqlite_session_factory assert layer._state_owner_user_id == state_owner_user_id + assert layer._human_form_reader is human_form_reader with pytest.raises(GraphEngineLayerNotInitializedError): _ = layer.graph_runtime_state assert layer.command_channel is None @@ -239,6 +248,7 @@ def test_initialize_sets_dependencies(self, sqlite_session_factory: sessionmaker session_factory=sqlite_session_factory, state_owner_user_id="owner", generate_entity=self._create_generate_entity(), + human_form_reader=_NullHumanInputFormReader(), response_stream_filter=ResponseStreamFilter(), ) @@ -258,6 +268,7 @@ def test_on_event_with_graph_run_paused_event( session_factory=sqlite_session_factory, state_owner_user_id="owner-123", generate_entity=generate_entity, + human_form_reader=_NullHumanInputFormReader(), response_stream_filter=_create_initialized_response_stream_filter(), ) @@ -295,16 +306,17 @@ def test_on_event_enriches_hitl_pause_reasons_before_persisting( self, monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session] ): generate_entity = self._create_generate_entity(workflow_execution_id="run-123") + human_form_reader: HumanInputFormReader = _NullHumanInputFormReader() layer = PauseStatePersistenceLayer( session_factory=sqlite_session_factory, state_owner_user_id="owner-123", generate_entity=generate_entity, + human_form_reader=human_form_reader, response_stream_filter=_create_initialized_response_stream_filter(), ) mock_repo = Mock() mock_factory = Mock(return_value=mock_repo) - mock_form_repository = Mock(name="form_repository") enriched_reason = HumanInputRequired( form_id="form-123", form_content="Rendered content", @@ -315,12 +327,6 @@ def test_on_event_enriches_hitl_pause_reasons_before_persisting( ) enrich_mock = Mock(return_value=[enriched_reason]) monkeypatch.setattr(DifyAPIRepositoryFactory, "create_api_workflow_run_repository", mock_factory) - monkeypatch.setattr( - pause_layer_module, - "HumanInputFormSubmissionRepository", - Mock(return_value=mock_form_repository), - raising=False, - ) monkeypatch.setattr( pause_layer_module, "enrich_graph_pause_reasons", @@ -345,7 +351,7 @@ def test_on_event_enriches_hitl_pause_reasons_before_persisting( enrich_mock.assert_called_once_with( reasons=[raw_reason], - form_repository=mock_form_repository, + form_repository=human_form_reader, variable_pool=graph_runtime_state.variable_pool, ) assert mock_repo.create_workflow_pause.call_args.kwargs["pause_reasons"] == [enriched_reason] @@ -357,6 +363,7 @@ def test_on_event_ignores_non_paused_events( session_factory=sqlite_session_factory, state_owner_user_id="owner-123", generate_entity=self._create_generate_entity(), + human_form_reader=_NullHumanInputFormReader(), response_stream_filter=ResponseStreamFilter(), ) @@ -387,6 +394,7 @@ def test_on_event_raises_when_graph_runtime_state_is_uninitialized( session_factory=sqlite_session_factory, state_owner_user_id="owner-123", generate_entity=self._create_generate_entity(), + human_form_reader=_NullHumanInputFormReader(), response_stream_filter=ResponseStreamFilter(), ) @@ -402,6 +410,7 @@ def test_on_event_asserts_when_workflow_execution_id_missing( session_factory=sqlite_session_factory, state_owner_user_id="owner-123", generate_entity=self._create_generate_entity(), + human_form_reader=_NullHumanInputFormReader(), response_stream_filter=_create_initialized_response_stream_filter(), ) @@ -513,6 +522,7 @@ def test_on_event_persists_response_stream_filter_dump( session_factory=sqlite_session_factory, state_owner_user_id="owner-123", generate_entity=generate_entity, + human_form_reader=_NullHumanInputFormReader(), response_stream_filter=response_stream_filter, ) diff --git a/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py b/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py index aaeae6824be501..443ec62b9fce22 100644 --- a/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py +++ b/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py @@ -31,7 +31,9 @@ from models.model import App, AppMode, EndUser from models.workflow import Workflow, WorkflowType from repositories.tools.workflow_repository import WorkflowToolRepository +from repositories.workflow.definition_repository import workflow_snapshot from services.tools.workflow.tool import WorkflowTool +from services.workflow.contracts import WorkflowSnapshot from services.workflow.execution.ports import WorkflowRuntime TENANT_ID = "00000000-0000-0000-0000-000000000001" @@ -720,9 +722,11 @@ def test_get_workflow_and_get_app_db_branches( assert latest.id == latest_workflow.id assert specific.id == specific_workflow.id + assert isinstance(latest, WorkflowSnapshot) + assert isinstance(specific, WorkflowSnapshot) + assert latest == workflow_snapshot(latest_workflow) + assert specific == workflow_snapshot(specific_workflow) assert resolved_app.id == app.id - assert inspect(latest).detached is True - assert inspect(specific).detached is True assert inspect(resolved_app).detached is True with pytest.raises(ToolNotFoundError, match="workflow not found"): diff --git a/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py b/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py index f3c0f5ec06c450..5494805d5d43a8 100644 --- a/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py +++ b/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py @@ -7,6 +7,7 @@ from models.model import Account, EndUser from models.workflow import Workflow, WorkflowType from services.rag_pipeline.pipeline_generate_service import PipelineGenerateService +from services.workflow.contracts import WorkflowSnapshot from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator @@ -126,13 +127,11 @@ def test_generate_single_loop_delegates(mocker: MockerFixture, pipeline_generato @pytest.mark.parametrize("draft", [True, False]) -def test_pipeline_execution_definition_is_owned_and_detached_before_generation( +def test_pipeline_execution_definition_is_owned_snapshot_before_generation( sqlite_session_factory, workflow_application, draft: bool, ) -> None: - from sqlalchemy import inspect - from services.errors.app import WorkflowNotFoundError pipeline = _make_pipeline() @@ -145,8 +144,15 @@ def test_pipeline_execution_definition_is_owned_and_detached_before_generation( generator = workflow_application.knowledge.pipeline_generator invocation = InvokeFrom.DEBUGGER if draft else InvokeFrom.PUBLISHED_PIPELINE loaded = generator.load_workflow(pipeline, invocation) - assert loaded.id == workflow.id - assert inspect(loaded).detached + assert isinstance(loaded, WorkflowSnapshot) + assert (loaded.id, loaded.tenant_id, loaded.app_id, loaded.version) == ( + workflow.id, + workflow.tenant_id, + workflow.app_id, + workflow.version, + ) + assert loaded.graph_dict == {} + assert loaded.features_dict == {} assert sqlite_session_factory.kw["bind"].pool.checkedout() == 0 pipeline.tenant_id = "other-tenant" with pytest.raises(WorkflowNotFoundError): diff --git a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py index d9c688a985f283..6b30af65855fc9 100644 --- a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py +++ b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py @@ -39,7 +39,7 @@ from repositories.factory import DifyAPIRepositoryFactory from repositories.knowledge.dataset_read_repository import get_pipeline_dataset from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository -from repositories.workflow.definition_repository import WorkflowDefinitionRepository +from repositories.workflow.definition_repository import WorkflowDefinitionRepository, workflow_snapshot from services.credentials.query import CredentialQuery from services.data_source.credential_gateway import DatasourceProviderCredentialStore from services.entities.knowledge_entities.rag_pipeline_entities import IconInfo, PipelineTemplateInfoEntity @@ -48,6 +48,7 @@ from services.rag_pipeline import rag_pipeline as rag_pipeline_module from services.rag_pipeline.execution_service import RagPipelineExecutionService from services.rag_pipeline.rag_pipeline import RagPipelineService +from services.workflow.contracts import WorkflowSnapshot from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator from services.workflow.execution.ports import WorkflowRuntime from services.workflow.variable_contracts import WorkflowExecutionVariables @@ -898,6 +899,10 @@ def test_retry_error_document_success( # 3. Assertions mock_gen_instance.generate.assert_called_once() + execution_workflow = mock_gen_instance.generate.call_args.kwargs["workflow"] + assert isinstance(execution_workflow, WorkflowSnapshot) + assert execution_workflow == workflow_snapshot(workflow) + assert mock_gen_instance.generate.call_args.kwargs["is_retry"] is True # --- set_datasource_variables --- @@ -916,9 +921,10 @@ def test_set_datasource_variables_success( ) pipeline.id = "p-1" - draft_wf = mocker.Mock() - draft_wf.id = "wf-1" - draft_wf.get_enclosing_node_type_and_id.return_value = None # Avoid unpacking error + draft_wf = _make_workflow( + workflow_id="wf-1", + graph={"nodes": [{"id": "node-1", "data": {"type": "datasource"}}], "edges": []}, + ) mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=draft_wf) execution = WorkflowNodeExecution( @@ -1088,10 +1094,11 @@ def test_run_draft_workflow_node_seeds_llm_environment_variable( "description": "Shared summarization model", } ) - draft_workflow = _make_workflow(workflow_id="wf-1") + draft_workflow = _make_workflow( + workflow_id="wf-1", + graph={"nodes": [{"id": "node-1", "data": {"type": "llm"}}], "edges": []}, + ) draft_workflow.environment_variables = [llm_environment_variable] - mocker.patch.object(draft_workflow, "get_node_config_by_id", return_value={"id": "node-1"}) - mocker.patch.object(draft_workflow, "get_enclosing_node_type_and_id", return_value=None) mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=draft_workflow) execution = WorkflowNodeExecution( @@ -1146,9 +1153,20 @@ def test_run_draft_workflow_node_saves_execution_and_variables( ) -> None: pipeline = _make_pipeline() account = _make_account() - draft_workflow = _make_workflow(workflow_id="wf-1") - mocker.patch.object(draft_workflow, "get_node_config_by_id", return_value={"id": "node-1"}) - mocker.patch.object(draft_workflow, "get_enclosing_node_type_and_id", return_value=("loop", "enclosing-node")) + draft_workflow = _make_workflow( + workflow_id="wf-1", + graph={ + "nodes": [ + { + "id": "node-1", + "data": {"type": "llm"}, + "isInLoop": True, + "loop_id": "enclosing-node", + } + ], + "edges": [], + }, + ) mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=draft_workflow) execution = WorkflowNodeExecution( @@ -1247,7 +1265,7 @@ def result(getter: Callable[[], object], **_kwargs: object) -> WorkflowNodeExecu assert saved.tenant_id == pipeline.tenant_id assert saved.app_id == pipeline.id assert loaded == ["injected input"] - loader = workflow_variables.workflow_loader(workflow, account.id) + loader = workflow_variables.workflow_loader(workflow_snapshot(workflow), account.id) assert loader.load_variables([["node-1", "text"]])[0].value == "injected output" @@ -1315,7 +1333,9 @@ def test_debug_execution_commits_before_variable_save_with_celery_repository( assert saved is not None assert executions.get_execution_by_id(saved.id, tenant_id=pipeline.tenant_id) is not None queued.assert_called_once() # Debug writes did not depend on the queued task running. - loaded = workflow_variables.workflow_loader(workflow, account.id).load_variables([["node-1", "text"]]) + loaded = workflow_variables.workflow_loader(workflow_snapshot(workflow), account.id).load_variables( + [["node-1", "text"]] + ) assert loaded[0].value == "saved" diff --git a/api/tests/unit_tests/services/test_app_generate_service.py b/api/tests/unit_tests/services/test_app_generate_service.py index 9b807a30ca6cc1..2cbf97ea272d96 100644 --- a/api/tests/unit_tests/services/test_app_generate_service.py +++ b/api/tests/unit_tests/services/test_app_generate_service.py @@ -30,6 +30,7 @@ from models.enums import AppStatus from models.model import App, AppMode from models.workflow import Workflow, WorkflowType +from repositories.workflow.definition_repository import workflow_snapshot from services.app.generation.runtime import AppGenerationRuntime from services.app_generate_service import AppGenerateService from services.errors.app import ( @@ -37,6 +38,7 @@ WorkflowIdFormatError, WorkflowNotFoundError, ) +from services.workflow.contracts import WorkflowSnapshot from services.workflow.variable_contracts import WorkflowExecutionVariables @@ -334,7 +336,7 @@ def generate(generator: AppInputAdapter, **_kwargs: object) -> dict[str, str]: # Keep the real generator constructor and debugger saver selection. assert generator._draft_variable_loader == workflow_variables.workflow_loader assert generator._draft_variable_saver == workflow_variables.saver_factory - loader = workflow_variables.workflow_loader(workflow, user.id) + loader = workflow_variables.workflow_loader(workflow_snapshot(workflow), user.id) saver = generator._get_draft_var_saver_factory(InvokeFrom.DEBUGGER, user, tenant_id=workflow.tenant_id) saver(workflow.app_id, "node", BuiltinNodeTypes.CODE, "execution").save( process_data=None, outputs={"text": "injected value"} @@ -1060,10 +1062,17 @@ def test_debugger_fetches_draft(self): workflow = _make_workflow() self.session.add(workflow) self.session.commit() - assert ( - AppGenerateService._get_workflow(_make_app(AppMode.WORKFLOW), InvokeFrom.DEBUGGER, session=self.session) - is workflow + result = AppGenerateService._get_workflow( + _make_app(AppMode.WORKFLOW), InvokeFrom.DEBUGGER, session=self.session + ) + assert isinstance(result, WorkflowSnapshot) + assert (result.id, result.tenant_id, result.app_id, result.version) == ( + workflow.id, + workflow.tenant_id, + workflow.app_id, + "draft", ) + assert result.graph == workflow.graph def test_debugger_raises_when_no_draft(self): with pytest.raises(ValueError, match="Workflow not initialized"): @@ -1076,7 +1085,10 @@ def test_non_debugger_fetches_published(self): self.session.commit() app = _make_app(AppMode.WORKFLOW) app.workflow_id = workflow.id - assert AppGenerateService._get_workflow(app, InvokeFrom.SERVICE_API, session=self.session) is workflow + result = AppGenerateService._get_workflow(app, InvokeFrom.SERVICE_API, session=self.session) + assert isinstance(result, WorkflowSnapshot) + assert (result.id, result.version) == (workflow.id, "published") + assert result.graph == workflow.graph def test_non_debugger_raises_when_no_published(self): with pytest.raises(ValueError, match="Workflow not published"): @@ -1087,12 +1099,12 @@ def test_specific_workflow_id_valid_uuid(self): workflow.version = "published" self.session.add(workflow) self.session.commit() - assert ( - AppGenerateService._get_workflow( - _make_app(AppMode.WORKFLOW), InvokeFrom.SERVICE_API, workflow_id=workflow.id, session=self.session - ) - is workflow + result = AppGenerateService._get_workflow( + _make_app(AppMode.WORKFLOW), InvokeFrom.SERVICE_API, workflow_id=workflow.id, session=self.session ) + assert isinstance(result, WorkflowSnapshot) + assert (result.id, result.version) == (workflow.id, "published") + assert result.graph == workflow.graph def test_specific_workflow_id_invalid_uuid(self): with pytest.raises(WorkflowIdFormatError): diff --git a/api/tests/unit_tests/services/test_snippet_generate_service.py b/api/tests/unit_tests/services/test_snippet_generate_service.py index b977d4c9e389bf..a8fc3b2df72055 100644 --- a/api/tests/unit_tests/services/test_snippet_generate_service.py +++ b/api/tests/unit_tests/services/test_snippet_generate_service.py @@ -9,7 +9,9 @@ from models.account import Account from models.snippet import CustomizedSnippet, SnippetType from models.workflow import Workflow, WorkflowKind, WorkflowNodeExecutionModel +from repositories.workflow.definition_repository import workflow_snapshot from services.snippet_generate_service import SnippetGenerateService +from services.workflow.contracts import WorkflowSnapshot from services.workflow.execution.ports import WorkflowRuntime from services.workflow.variable_contracts import WorkflowExecutionVariables from services.workflow_service import WorkflowService @@ -89,7 +91,7 @@ def execute(generator: WorkflowAppGenerator, **_kwargs: object) -> dict[str, str saver(workflow.app_id, "node", BuiltinNodeTypes.CODE, "execution").save( process_data=None, outputs={"text": "injected"} ) - loader = workflow_variables.workflow_loader(workflow, user.id) + loader = workflow_variables.workflow_loader(workflow_snapshot(workflow), user.id) assert loader.load_variables([["node", "text"]])[0].value == "injected" calls.append(entry) return {"result": "ready"} @@ -140,7 +142,7 @@ def test_is_virtual_start_event(message, expected): def test_ensure_start_node_returns_workflow_when_start_already_exists(): - workflow = _workflow({"nodes": [{"id": "start", "data": {"type": "start"}}], "edges": []}) + workflow = workflow_snapshot(_workflow({"nodes": [{"id": "start", "data": {"type": "start"}}], "edges": []})) snippet = _snippet() result = SnippetGenerateService._ensure_start_node(workflow, snippet) @@ -148,7 +150,7 @@ def test_ensure_start_node_returns_workflow_when_start_already_exists(): assert result is workflow -def test_ensure_start_node_injects_virtual_start_for_root_candidates(monkeypatch: pytest.MonkeyPatch): +def test_ensure_start_node_injects_virtual_start_for_root_candidates(): graph = { "nodes": [ {"id": "llm-1", "data": {"type": "llm"}}, @@ -156,7 +158,7 @@ def test_ensure_start_node_injects_virtual_start_for_root_candidates(monkeypatch ], "edges": [{"source": "llm-1", "target": "answer-1"}], } - workflow = _workflow(graph) + workflow = workflow_snapshot(_workflow(graph)) snippet = _snippet( input_fields=[ { @@ -168,18 +170,15 @@ def test_ensure_start_node_injects_virtual_start_for_root_candidates(monkeypatch } ] ) - make_transient = Mock() - monkeypatch.setattr("services.snippet_generate_service.make_transient", make_transient) - result = SnippetGenerateService._ensure_start_node(workflow, snippet) - assert result is workflow - updated_graph = workflow.graph_dict + assert result is not workflow + assert workflow.graph_dict == graph + updated_graph = result.graph_dict assert updated_graph["nodes"][0]["id"] == SNIPPET_VIRTUAL_START_NODE_ID assert updated_graph["nodes"][0]["data"]["variables"][0]["max_length"] == 128 assert updated_graph["edges"][-1]["source"] == SNIPPET_VIRTUAL_START_NODE_ID assert updated_graph["edges"][-1]["target"] == "llm-1" - make_transient.assert_called_once_with(workflow) def test_parse_files_returns_empty_when_upload_config_disabled(monkeypatch: pytest.MonkeyPatch): @@ -239,7 +238,8 @@ def test_generate_delegates_to_workflow_generator_and_filters_stream( workflow_generator_class.convert_to_event_stream = Mock(side_effect=lambda response: response) snippet_reader.workflow = workflow - ensure_start_node = Mock(return_value=workflow) + expected_workflow = workflow_snapshot(workflow) + ensure_start_node = Mock(return_value=expected_workflow) monkeypatch.setattr(SnippetGenerateService, "_ensure_start_node", ensure_start_node) monkeypatch.setattr("services.snippet_generate_service.WorkflowAppGenerator", workflow_generator_class) @@ -253,11 +253,12 @@ def test_generate_delegates_to_workflow_generator_and_filters_stream( assert [json.loads(event.removeprefix("data: ")) for event in result] == [ {"event": "node_finished", "data": {"node_id": "llm-1"}} ] - ensure_start_node.assert_called_once_with(workflow, snippet) + ensure_start_node.assert_called_once_with(expected_workflow, snippet) generator.generate.assert_called_once() kwargs = generator.generate.call_args.kwargs assert kwargs["app_model"].id == "snippet-1" - assert kwargs["workflow"] is workflow + assert isinstance(kwargs["workflow"], WorkflowSnapshot) + assert kwargs["workflow"] == expected_workflow assert kwargs["user"] is user assert kwargs["streaming"] is True assert kwargs["call_depth"] == 0 @@ -292,7 +293,8 @@ def test_run_draft_node_delegates_to_workflow_service( workflow_service.run_draft_workflow_node.assert_called_once() kwargs = workflow_service.run_draft_workflow_node.call_args.kwargs assert kwargs["app_model"].id == "snippet-1" - assert kwargs["draft_workflow"] is workflow + assert isinstance(kwargs["draft_workflow"], WorkflowSnapshot) + assert kwargs["draft_workflow"] == workflow_snapshot(workflow) assert kwargs["node_id"] == "llm-1" assert kwargs["user_inputs"] == {"query": "hello"} assert kwargs["account"] is account @@ -344,7 +346,8 @@ def test_generate_single_iteration_delegates_to_workflow_generator( generator.single_iteration_generate.assert_called_once() kwargs = generator.single_iteration_generate.call_args.kwargs assert kwargs["app_model"].id == "snippet-1" - assert kwargs["workflow"] is workflow + assert isinstance(kwargs["workflow"], WorkflowSnapshot) + assert kwargs["workflow"] == workflow_snapshot(workflow) assert kwargs["node_id"] == "iteration-1" assert kwargs["user"] is user assert kwargs["streaming"] is True @@ -393,7 +396,8 @@ def test_generate_single_loop_delegates_to_workflow_generator( generator.single_loop_generate.assert_called_once() kwargs = generator.single_loop_generate.call_args.kwargs assert kwargs["app_model"].id == "snippet-1" - assert kwargs["workflow"] is workflow + assert isinstance(kwargs["workflow"], WorkflowSnapshot) + assert kwargs["workflow"] == workflow_snapshot(workflow) assert kwargs["node_id"] == "loop-1" assert kwargs["user"] is user assert kwargs["streaming"] is True diff --git a/api/tests/unit_tests/services/workflow/test_runtime_gateway.py b/api/tests/unit_tests/services/workflow/test_runtime_gateway.py index c5dc5dabaf2d10..a514044bfd9e66 100644 --- a/api/tests/unit_tests/services/workflow/test_runtime_gateway.py +++ b/api/tests/unit_tests/services/workflow/test_runtime_gateway.py @@ -1254,7 +1254,7 @@ def test_task_rejects_snapshot_outside_its_debugger_owner( user=actor, args={"inputs": {}}, invoke_from=InvokeFrom.DEBUGGER, - workflow_snapshot=module.workflow_snapshot(draft), + workflow_snapshot=draft, ) payload = json.loads(params.model_dump_json()) if field.startswith("workflow_snapshot."): diff --git a/api/tests/unit_tests/services/workflow/test_runtime_variable_files.py b/api/tests/unit_tests/services/workflow/test_runtime_variable_files.py index d65d2babcd8f11..46fd6e61be82b5 100644 --- a/api/tests/unit_tests/services/workflow/test_runtime_variable_files.py +++ b/api/tests/unit_tests/services/workflow/test_runtime_variable_files.py @@ -16,6 +16,7 @@ from models.dataset import Pipeline from models.snippet import CustomizedSnippet from models.workflow import Workflow, WorkflowDraftVariable, WorkflowKind, WorkflowType +from repositories.workflow.definition_repository import workflow_snapshot from services.workflow.variable_service import WorkflowVariableService from tests.unit_tests.model_factories import make_account, make_app, make_upload_file, make_workflow @@ -106,7 +107,7 @@ def test_load_file_variables_restores_canonical_metadata( _persist_file_variable( sqlite_session_factory, workflow, value_type, [mapping] if value_type == SegmentType.ARRAY_FILE else mapping ) - loaded = variables.workflow_loader(workflow, "account-1").load_variables([["node", "file"]]) + loaded = variables.workflow_loader(workflow_snapshot(workflow), "account-1").load_variables([["node", "file"]]) assert len(loaded) == 1 assert loaded[0].selector == ["node", "file"] files = loaded[0].value if value_type == SegmentType.ARRAY_FILE else [loaded[0].value] @@ -135,7 +136,7 @@ def test_load_file_variables_rejects_another_tenants_upload( sqlite_session_factory, workflow, value_type, [mapping] if value_type == SegmentType.ARRAY_FILE else mapping ) with pytest.raises(ValueError, match="Invalid upload file"): - variables.workflow_loader(workflow, "account-1").load_variables([["node", "file"]]) + variables.workflow_loader(workflow_snapshot(workflow), "account-1").load_variables([["node", "file"]]) @pytest.mark.parametrize("value_type", [SegmentType.FILE, SegmentType.ARRAY_FILE]) @@ -156,7 +157,7 @@ def test_load_external_file_does_not_fetch_it_again( _persist_file_variable( sqlite_session_factory, workflow, value_type, [mapping] if value_type == SegmentType.ARRAY_FILE else mapping ) - loaded = variables.workflow_loader(workflow, "account-1").load_variables([["node", "file"]]) + loaded = variables.workflow_loader(workflow_snapshot(workflow), "account-1").load_variables([["node", "file"]]) file = loaded[0].value[0] if value_type == SegmentType.ARRAY_FILE else loaded[0].value assert (file.remote_url, file.filename, file.size) == (mapping["url"], "document.txt", 12) @@ -166,7 +167,7 @@ def test_load_empty_file_array( ) -> None: variables, workflow = runtime_scope _persist_file_variable(sqlite_session_factory, workflow, SegmentType.ARRAY_FILE, []) - loaded = variables.workflow_loader(workflow, "account-1").load_variables([["node", "file"]]) + loaded = variables.workflow_loader(workflow_snapshot(workflow), "account-1").load_variables([["node", "file"]]) assert loaded[0].value == [] @@ -192,5 +193,5 @@ def test_start_node_files_round_trip_through_injected_gateway( ] }, ) - loaded = variables.workflow_loader(workflow, "account-1").load_variables([["sys", "files"]]) + loaded = variables.workflow_loader(workflow_snapshot(workflow), "account-1").load_variables([["sys", "files"]]) assert [file.filename for file in loaded[0].value] == ["start.txt"] diff --git a/api/tests/unit_tests/services/workflow/test_variable_service.py b/api/tests/unit_tests/services/workflow/test_variable_service.py index b2a790ee94ea10..9cc55dda0248e8 100644 --- a/api/tests/unit_tests/services/workflow/test_variable_service.py +++ b/api/tests/unit_tests/services/workflow/test_variable_service.py @@ -18,6 +18,7 @@ from models.account import TenantAccountJoin, TenantAccountRole from models.workflow import WorkflowDraftVariable, WorkflowDraftVariableFile from repositories.factory import DifyAPIRepositoryFactory +from repositories.workflow.definition_repository import workflow_snapshot from repositories.workflow.draft_variable_repository import WorkflowDraftVariableRepository from services.app.generation.input_adapter import AppInputAdapter from services.file_service import FileService @@ -74,7 +75,7 @@ def delete(key: str) -> None: account = make_account() workflow = make_workflow() workflow.conversation_variables = [StringVariable(name="greeting", value="hello")] - loader = variables.workflow_loader(workflow, account.id) + loader = variables.workflow_loader(workflow_snapshot(workflow), account.id) assert loader.load_variables([["conversation", "greeting"]])[0].value == "hello" saver_factory = generator._get_draft_var_saver_factory(InvokeFrom.DEBUGGER, account, tenant_id=workflow.tenant_id) @@ -334,7 +335,7 @@ def unexpected_global_read(_filename: str) -> bytes: assert len(contents) == 2 loader = ( - variables.workflow_loader(workflow, account.id) + variables.workflow_loader(workflow_snapshot(workflow), account.id) if loader_entry == "workflow" else variables.loader(RequestContext("request", None, account.id, workflow.tenant_id), workflow.app_id, []) ) diff --git a/api/tests/unit_tests/tasks/test_async_workflow_tasks.py b/api/tests/unit_tests/tasks/test_async_workflow_tasks.py index 9eabbb889f1569..bea2c402d52ba0 100644 --- a/api/tests/unit_tests/tasks/test_async_workflow_tasks.py +++ b/api/tests/unit_tests/tasks/test_async_workflow_tasks.py @@ -1,8 +1,172 @@ import ast import inspect +from types import SimpleNamespace +from unittest.mock import MagicMock +import pytest + +from core.app.app_config.entities import WorkflowUIBasedAppConfig +from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity +from models.enums import AppTriggerType, WorkflowRunTriggeredFrom +from models.model import AppMode +from repositories.workflow.definition_repository import workflow_snapshot from services.errors.app import WorkflowNotFoundError +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.entities import TriggerData, WorkflowTaskData from tasks import async_workflow_tasks +from tests.unit_tests.model_factories import make_account, make_app, make_workflow + + +def _session_context(session: MagicMock) -> MagicMock: + context = MagicMock() + context.__enter__.return_value = session + context.__exit__.return_value = False + return context + + +def test_execute_workflow_projects_the_persisted_workflow_before_generation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + workflow_model = make_workflow(workflow_id="workflow-1", tenant_id="tenant-1", app_id="app-1") + app = make_app(app_id="app-1", tenant_id="tenant-1", workflow_id=workflow_model.id) + user = make_account(account_id="account-1") + trigger_data = TriggerData( + app_id=app.id, + tenant_id=app.tenant_id, + workflow_id=workflow_model.id, + root_node_id="start", + inputs={"question": "hello"}, + trigger_type=AppTriggerType.TRIGGER_SCHEDULE, + trigger_from=WorkflowRunTriggeredFrom.SCHEDULE, + ) + trigger_log = SimpleNamespace( + id="trigger-log-1", + app_id=app.id, + workflow_id=workflow_model.id, + trigger_data=trigger_data.model_dump_json(), + status=None, + ) + + session = MagicMock() + session.scalar.side_effect = [app, workflow_model] + task_session_factory = MagicMock() + task_session_factory.create_session.return_value = _session_context(session) + task_session_factory.get_session_maker.return_value = MagicMock() + trigger_log_repository = MagicMock() + trigger_log_repository.get_by_id.return_value = trigger_log + generator = MagicMock() + + monkeypatch.setattr(async_workflow_tasks, "session_factory", task_session_factory) + monkeypatch.setattr( + async_workflow_tasks, + "SQLAlchemyWorkflowTriggerLogRepository", + MagicMock(return_value=trigger_log_repository), + ) + monkeypatch.setattr(async_workflow_tasks, "_get_user", MagicMock(return_value=user)) + monkeypatch.setattr(async_workflow_tasks, "build_workflow_execution_dependencies", MagicMock()) + monkeypatch.setattr(async_workflow_tasks, "WorkflowAppGenerator", MagicMock(return_value=generator)) + monkeypatch.setattr(async_workflow_tasks, "TriggerPostLayer", MagicMock(return_value=MagicMock())) + + async_workflow_tasks._execute_workflow_common( + WorkflowTaskData(workflow_trigger_log_id=trigger_log.id), + MagicMock(), + MagicMock(), + ) + + execution_workflow = generator.generate.call_args.kwargs["workflow"] + assert isinstance(execution_workflow, WorkflowSnapshot) + assert execution_workflow == workflow_snapshot(workflow_model) + assert execution_workflow is not workflow_model + + +def test_resume_workflow_projects_the_persisted_workflow_before_resuming( + monkeypatch: pytest.MonkeyPatch, +) -> None: + workflow_model = make_workflow(workflow_id="workflow-1", tenant_id="tenant-1", app_id="app-1") + app = make_app(app_id="app-1", tenant_id="tenant-1", workflow_id=workflow_model.id) + user = make_account(account_id="account-1") + workflow_run = SimpleNamespace( + id="workflow-run-1", + workflow_id=workflow_model.id, + app_id=app.id, + triggered_from=WorkflowRunTriggeredFrom.SCHEDULE, + ) + pause_entity = MagicMock(id="pause-1", workflow_execution_id=workflow_run.id) + pause_entity.get_state.return_value = b"serialized-resumption-context" + workflow_run_repository = MagicMock() + workflow_run_repository.get_workflow_pause.return_value = pause_entity + workflow_run_repository.get_workflow_run_by_id_without_tenant.return_value = workflow_run + + generate_entity = WorkflowAppGenerateEntity( + task_id="task-1", + app_config=WorkflowUIBasedAppConfig( + tenant_id=app.tenant_id, + app_id=app.id, + app_mode=AppMode.WORKFLOW, + workflow_id=workflow_model.id, + ), + inputs={}, + files=[], + user_id=user.id, + stream=False, + invoke_from=InvokeFrom.WEB_APP, + workflow_execution_id=workflow_run.id, + ) + resumption_context = MagicMock() + resumption_context.get_generate_entity.return_value = generate_entity + resumption_context.serialized_graph_runtime_state = "serialized-runtime-state" + response_stream_filter = MagicMock() + resumption_context.get_response_stream_filter.return_value = response_stream_filter + graph_runtime_state = MagicMock() + + session = MagicMock() + session.scalar.side_effect = [workflow_model, app] + resumed_session_factory = MagicMock() + resumed_session_factory.return_value = _session_context(session) + generator = MagicMock() + execution_repository = MagicMock() + node_execution_repository = MagicMock() + + monkeypatch.setattr(async_workflow_tasks, "db", SimpleNamespace(engine=object())) + monkeypatch.setattr(async_workflow_tasks, "sessionmaker", MagicMock(return_value=resumed_session_factory)) + monkeypatch.setattr( + async_workflow_tasks.DifyAPIRepositoryFactory, + "create_api_workflow_run_repository", + MagicMock(return_value=workflow_run_repository), + ) + monkeypatch.setattr( + async_workflow_tasks.WorkflowResumptionContext, + "loads", + MagicMock(return_value=resumption_context), + ) + monkeypatch.setattr( + async_workflow_tasks.GraphRuntimeState, + "from_snapshot", + MagicMock(return_value=graph_runtime_state), + ) + monkeypatch.setattr(async_workflow_tasks, "_get_user", MagicMock(return_value=user)) + monkeypatch.setattr( + async_workflow_tasks.DifyCoreRepositoryFactory, + "create_workflow_execution_repository", + MagicMock(return_value=execution_repository), + ) + monkeypatch.setattr( + async_workflow_tasks.DifyCoreRepositoryFactory, + "create_workflow_node_execution_repository", + MagicMock(return_value=node_execution_repository), + ) + monkeypatch.setattr(async_workflow_tasks, "build_workflow_execution_dependencies", MagicMock()) + monkeypatch.setattr(async_workflow_tasks, "WorkflowAppGenerator", MagicMock(return_value=generator)) + monkeypatch.setattr(async_workflow_tasks, "_query_trigger_log_info", MagicMock(return_value=None)) + + async_workflow_tasks.resume_workflow_execution.run({"workflow_run_id": workflow_run.id}) + + resumed_workflow = generator.resume.call_args.kwargs["workflow"] + assert isinstance(resumed_workflow, WorkflowSnapshot) + assert resumed_workflow == workflow_snapshot(workflow_model) + assert resumed_workflow is not workflow_model + workflow_run_repository.resume_workflow_pause.assert_called_once_with(workflow_run.id, pause_entity) + workflow_run_repository.delete_workflow_pause.assert_called_once_with(pause_entity) def test_workflow_not_found_error_message_is_a_single_formatted_string(): From 718f9d90b8293dec940ef2c8747e6536b83166ed Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Thu, 15 Oct 2026 16:42:40 +0800 Subject: [PATCH 09/20] fix(workflow): address review findings in #43680 --- api/services/snippet_generate_service.py | 11 +++++- .../app_generate/workflow_execute_task.py | 38 +++++++++++-------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/api/services/snippet_generate_service.py b/api/services/snippet_generate_service.py index 811acdba8a2990..00d060da5afca5 100644 --- a/api/services/snippet_generate_service.py +++ b/api/services/snippet_generate_service.py @@ -24,6 +24,8 @@ from dataclasses import replace from typing import Any, Protocol, Union, cast +from sqlalchemy.orm import make_transient + from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.app.entities.app_invoke_entities import InvokeFrom from core.app.file_access import DatabaseFileAccessController @@ -227,7 +229,14 @@ def _ensure_start_node(cls, workflow: WorkflowSnapshot, snippet: CustomizedSnipp @classmethod def ensure_start_node_for_worker(cls, workflow: Workflow, snippet: CustomizedSnippet) -> Workflow: """Inject the virtual Start node after a worker reloads a snippet workflow.""" - return cls._ensure_start_node(workflow, snippet) + snapshot = workflow_snapshot(workflow) + updated_snapshot = cls._ensure_start_node(snapshot, snippet) + if updated_snapshot.graph == snapshot.graph: + return workflow + + make_transient(workflow) + workflow.graph = updated_snapshot.graph + return workflow @classmethod def _inject_virtual_start_node( diff --git a/api/tasks/app_generate/workflow_execute_task.py b/api/tasks/app_generate/workflow_execute_task.py index ac8b16635e3900..37f96e9677cfa7 100644 --- a/api/tasks/app_generate/workflow_execute_task.py +++ b/api/tasks/app_generate/workflow_execute_task.py @@ -2,6 +2,7 @@ import logging import uuid from collections.abc import Callable, Generator, Mapping +from dataclasses import replace from enum import StrEnum from typing import Annotated, Any @@ -35,7 +36,7 @@ from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom, WorkflowRun from repositories.factory import DifyAPIRepositoryFactory from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository -from repositories.workflow.definition_repository import workflow_from_snapshot +from repositories.workflow.definition_repository import workflow_snapshot from repositories.workflow.execution_write_repository import WorkflowExecutionWriteRepository from services.agent.retirement_service import WorkflowAgentRetirementService from services.errors.workflow_service import WorkflowDebugReservationExpiredError @@ -112,7 +113,7 @@ class AppExecutionParams(BaseModel): def new( cls, app_model: App, - workflow: Workflow, + workflow: WorkflowSnapshot, user: Account | EndUser, args: Mapping[str, Any], invoke_from: InvokeFrom, @@ -231,13 +232,13 @@ def _run(self): snapshot.id, ) != (reserved_run.id, reserved_run.tenant_id, reserved_run.app_id, reserved_run.workflow_id): raise ValueError("Workflow snapshot does not belong to this debugger invocation") - workflow = session.get(Workflow, exec_params.workflow_id) - if workflow is None: + persisted_workflow = session.get(Workflow, exec_params.workflow_id) + if persisted_workflow is None: logger.warning("Workflow %s not found for execution", exec_params.workflow_id) return None - app = session.get(App, workflow.app_id) + app = session.get(App, persisted_workflow.app_id) if app is None: - logger.warning("App %s not found for workflow %s", workflow.app_id, exec_params.workflow_id) + logger.warning("App %s not found for workflow %s", persisted_workflow.app_id, exec_params.workflow_id) return None snapshot = exec_params.workflow_snapshot @@ -247,14 +248,15 @@ def _run(self): or exec_params.invoke_from != InvokeFrom.DEBUGGER or exec_params.app_mode != AppMode.WORKFLOW or app.mode != AppMode.WORKFLOW - or (workflow.id, workflow.tenant_id, workflow.app_id) + or (persisted_workflow.id, persisted_workflow.tenant_id, persisted_workflow.app_id) != (exec_params.workflow_id, exec_params.tenant_id, exec_params.app_id) - or (snapshot.id, snapshot.tenant_id, snapshot.app_id) != (workflow.id, app.tenant_id, app.id) + or (snapshot.id, snapshot.tenant_id, snapshot.app_id) + != (persisted_workflow.id, app.tenant_id, app.id) ): raise ValueError("Workflow snapshot does not belong to this debugger invocation") - workflow = workflow_from_snapshot(snapshot) + workflow = snapshot if reserved_run is not None: - workflow.graph = reserved_run.graph or snapshot.graph + workflow = replace(snapshot, graph=reserved_run.graph or snapshot.graph) if WorkflowDebugReservationRepository.claim(session, reserved_run, naive_utc_now()): self._validated_snapshot = True else: @@ -262,6 +264,8 @@ def _run(self): self._mark_execution_finalized() if exec_params.streaming: completed_creator = _resolve_user_for_run(session, reserved_run) + else: + workflow = workflow_snapshot(persisted_workflow) if completed_run is not None: # Redelivery never claims a lease, runs the engine, or repeats cleanup. @@ -325,7 +329,7 @@ def _run_app( self, *, app: App, - workflow: Workflow, + workflow: WorkflowSnapshot, user: Account | EndUser, pause_state_config: PauseStateLayerConfig, ): @@ -714,8 +718,8 @@ def _resume_app_execution(payload: dict[str, Any], *, variables: WorkflowExecuti logger.warning("Workflow run %s not found during resume", workflow_run_id) return - workflow = session.get(Workflow, workflow_run.workflow_id) - if workflow is None: + workflow_model = session.get(Workflow, workflow_run.workflow_id) + if workflow_model is None: logger.warning("Workflow %s not found during resume", workflow_run.workflow_id) return @@ -754,10 +758,12 @@ def _resume_app_execution(payload: dict[str, Any], *, variables: WorkflowExecuti logger.warning("Message not found for workflow run %s", workflow_run_id) return + workflow = workflow_snapshot(workflow_model) + binding_snapshot = workflow_run.graph_dict.get("_agent_bindings", {}) if binding_snapshot.get("execution_id") == workflow_run.id and workflow_run.graph is not None: # The resumed graph and its Agent generations belong to the original execution. - workflow.graph = workflow_run.graph + workflow = replace(workflow, graph=workflow_run.graph) if not isinstance(generate_entity, (AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity)): logger.error( @@ -821,7 +827,7 @@ def _resume_advanced_chat( *, variables: WorkflowExecutionVariables, app_model: App, - workflow: Workflow, + workflow: WorkflowSnapshot, user: Account | EndUser, conversation: Conversation, message: Message, @@ -898,7 +904,7 @@ def _resume_workflow( *, variables: WorkflowExecutionVariables, app_model: App, - workflow: Workflow, + workflow: WorkflowSnapshot, user: Account | EndUser, generate_entity: WorkflowAppGenerateEntity, graph_runtime_state: GraphRuntimeState, From 677ef7eb6a1a42bf6066774364249d579ffdafaa Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Tue, 20 Oct 2026 11:50:23 +0800 Subject: [PATCH 10/20] test(api): adapt tests to merged execution interfaces --- .../unit_tests/controllers/openapi/test_app_run_streaming.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/tests/unit_tests/controllers/openapi/test_app_run_streaming.py b/api/tests/unit_tests/controllers/openapi/test_app_run_streaming.py index d11d2d6a1e5fbb..0ce191fe8744e6 100644 --- a/api/tests/unit_tests/controllers/openapi/test_app_run_streaming.py +++ b/api/tests/unit_tests/controllers/openapi/test_app_run_streaming.py @@ -16,9 +16,9 @@ from sqlalchemy import event from sqlalchemy.orm import Session from werkzeug.datastructures import FileStorage -from werkzeug.exceptions import NotFound from constants.oauth_bearer import TokenType +from controllers.common.errors import NotFoundError from controllers.openapi import app_run from controllers.openapi._errors import AppModeMismatch from controllers.openapi._models import ( @@ -155,7 +155,7 @@ def test_stop_task_enforces_resolved_caller_ownership( if allowed: assert api.post.__handler__(api, ctx, app_id="app-1", task_id=task_id) == TaskStopResponse(result="success") else: - with pytest.raises(NotFound, match="Task not found"): + with pytest.raises(NotFoundError, match="Task not found"): api.post.__handler__(api, ctx, app_id="app-1", task_id=task_id) if allowed: assert redis.values[f"generate_task_stopped:{task_id}"] == b"1" From 0d5a3f3d3f5cfa257b0c5554f1402f273fac8285 Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Mon, 26 Oct 2026 22:13:43 +0800 Subject: [PATCH 11/20] Regenerate canonical API contracts and remove stale import exemption --- api/.importlinter | 1 - .../generated/api/console/apps/errors.gen.ts | 94 +----------- .../generated/api/console/apps/orpc.gen.ts | 138 +----------------- .../generated/api/console/errors.gen.ts | 46 ------ .../generated/api/console/rag/errors.gen.ts | 25 ---- 5 files changed, 11 insertions(+), 293 deletions(-) diff --git a/api/.importlinter b/api/.importlinter index da791019fa2e64..628be564513d31 100644 --- a/api/.importlinter +++ b/api/.importlinter @@ -101,7 +101,6 @@ ignore_imports = # The KnowledgeFS retrieval node is owned by services.knowledge_fs and only wired in here. core.workflow.node_factory -> services.knowledge_fs.knowledge_retrieval_v2.automatic_metadata_filter core.workflow.node_factory -> services.knowledge_fs.knowledge_retrieval_v2.entities - core.workflow.nodes.agent.message_transformer -> services.tools.legacy_builtin_tools_manage_service core.workflow.nodes.agent_v2.agent_node -> services.agent.knowledge_citations core.workflow.nodes.agent_v2.agent_node -> services.agent.prompt_mentions core.workflow.nodes.agent_v2.agent_node -> services.agent.workspace_service diff --git a/packages/contracts/generated/api/console/apps/errors.gen.ts b/packages/contracts/generated/api/console/apps/errors.gen.ts index ff8103364bbe0e..46872ca1e3d237 100644 --- a/packages/contracts/generated/api/console/apps/errors.gen.ts +++ b/packages/contracts/generated/api/console/apps/errors.gen.ts @@ -500,18 +500,7 @@ export const get4Errors = { app_not_found: { status: 404, data: zAppNotFoundErro export const get5Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const post4Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post5Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post6Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post7Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post8Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, -} +export const post8Errors = { rate_limit_error: { status: 429, data: zRateLimitError } } export const get6Errors = { agent_not_bound: { status: 400, data: zAgentNotBoundError }, @@ -597,8 +586,6 @@ export const get15Errors = { app_not_found: { status: 404, data: zAppNotFoundError }, } -export const get16Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const post13Errors = { no_file_uploaded: { status: 400, data: zNoFileUploadedError }, too_many_files: { status: 400, data: zTooManyFilesError }, @@ -664,8 +651,6 @@ export const post20Errors = { export const get30Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const post21Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get31Errors = { roster_agent_package_export_failed: { status: 500, data: zRosterAgentPackageExportFailedError }, } @@ -809,8 +794,6 @@ export const get50Errors = { app_not_found: { status: 404, data: zAppNotFoundErr export const get51Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const post39Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get52Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } export const get53Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } @@ -899,33 +882,10 @@ export const get63Errors = { app_not_found: { status: 404, data: zAppNotFoundErr export const get64Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const get65Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const get66Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const get67Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post44Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get68Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, } -export const post45Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post46Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post47Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post48Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post49Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post50Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post51Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get69Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } export const post52Errors = { @@ -971,20 +931,7 @@ export const put5Errors = { plaintext_secret_not_allowed_error: { status: 400, data: zPlaintextSecretNotAllowedError }, } -export const get71Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post56Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post57Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const delete11Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const get72Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post58Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, -} +export const post58Errors = { rate_limit_error: { status: 429, data: zRateLimitError } } export const get73Errors = { app_not_found: { status: 404, data: zAppNotFoundError }, @@ -1009,47 +956,22 @@ export const get76Errors = { workflow_run_not_found: { status: 404, data: zWorkflowRunNotFoundError }, } -export const get77Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post59Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, -} - -export const post60Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, -} - -export const put6Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const delete12Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } +export const post59Errors = { rate_limit_error: { status: 429, data: zRateLimitError } } -export const get78Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const patch2Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const delete13Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } +export const post60Errors = { rate_limit_error: { status: 429, data: zRateLimitError } } export const get79Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, } export const get80Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, } export const post61Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError }, } -export const get81Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post62Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get82Errors = { app_not_found: { status: 404, data: zAppNotFoundError }, workflow_run_not_found: { status: 404, data: zWorkflowRunNotFoundError }, @@ -1078,14 +1000,6 @@ export const get86Errors = { webhook_trigger_not_found: { status: 404, data: zWebhookTriggerNotFoundError }, } -export const post63Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const delete14Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const patch3Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const get87Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const post64Errors = { agent_not_published: { status: 409, data: zAgentNotPublishedError }, max_keys_exceeded: { status: 400, data: zMaxKeysExceededError }, diff --git a/packages/contracts/generated/api/console/apps/orpc.gen.ts b/packages/contracts/generated/api/console/apps/orpc.gen.ts index 74fb9e1bd54eb7..7dce5ce0abdfff 100644 --- a/packages/contracts/generated/api/console/apps/orpc.gen.ts +++ b/packages/contracts/generated/api/console/apps/orpc.gen.ts @@ -724,8 +724,6 @@ export const workflowRuns = { } /** - * Preview human input form content and placeholders - * * Get human input form preview for advanced chat workflow */ export const post4 = oc @@ -735,7 +733,6 @@ export const post4 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormPreview', path: '/apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/preview', - summary: 'Preview human input form content and placeholders', tags: ['console'], }) .input( @@ -745,15 +742,12 @@ export const post4 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormPreviewResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const preview = { post: post4, } /** - * Submit human input form preview - * * Submit human input form preview for advanced chat workflow */ export const post5 = oc @@ -763,7 +757,6 @@ export const post5 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/run', - summary: 'Submit human input form preview', tags: ['console'], }) .input( @@ -773,7 +766,6 @@ export const post5 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run = { post: post5, @@ -797,8 +789,6 @@ export const humanInput = { } /** - * Run draft workflow iteration node - * * Run draft workflow iteration node for advanced chat */ export const post6 = oc @@ -808,7 +798,6 @@ export const post6 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftIterationNodesByNodeIdRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/iteration/nodes/{node_id}/run', - summary: 'Run draft workflow iteration node', tags: ['console'], }) .input( @@ -818,7 +807,6 @@ export const post6 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftIterationNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run2 = { post: post6, @@ -837,8 +825,6 @@ export const iteration = { } /** - * Run draft workflow loop node - * * Run draft workflow loop node for advanced chat */ export const post7 = oc @@ -848,7 +834,6 @@ export const post7 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftLoopNodesByNodeIdRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/loop/nodes/{node_id}/run', - summary: 'Run draft workflow loop node', tags: ['console'], }) .input( @@ -858,7 +843,6 @@ export const post7 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftLoopNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run3 = { post: post7, @@ -877,8 +861,6 @@ export const loop = { } /** - * Run draft workflow - * * Run draft workflow for advanced chat application */ export const post8 = oc @@ -888,7 +870,6 @@ export const post8 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/run', - summary: 'Run draft workflow', tags: ['console'], }) .input( @@ -898,10 +879,7 @@ export const post8 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftRunResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, - }) + .errors({ rate_limit_error: { status: 429, data: zRateLimitError } }) export const run4 = { post: post8, @@ -1980,21 +1958,15 @@ export const conversationVariables = { } /** - * Convert basic mode of chatbot app to workflow mode - * * Convert application to workflow mode - * Convert expert mode of chatbot app to workflow mode - * Convert Completion App to Workflow App */ export const post21 = oc .route({ - description: - 'Convert application to workflow mode\nConvert expert mode of chatbot app to workflow mode\nConvert Completion App to Workflow App', + description: 'Convert application to workflow mode', inputStructure: 'detailed', method: 'POST', operationId: 'postAppsByAppIdConvertToWorkflow', path: '/apps/{app_id}/convert-to-workflow', - summary: 'Convert basic mode of chatbot app to workflow mode', tags: ['console'], }) .input( @@ -2004,7 +1976,6 @@ export const post21 = oc }), ) .output(zPostAppsByAppIdConvertToWorkflowResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const convertToWorkflow = { post: post21, @@ -3039,8 +3010,6 @@ export const count3 = { } /** - * Stop workflow task - * * Stop running workflow task */ export const post39 = oc @@ -3050,12 +3019,10 @@ export const post39 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowRunsTasksByTaskIdStop', path: '/apps/{app_id}/workflow-runs/tasks/{task_id}/stop', - summary: 'Stop workflow task', tags: ['console'], }) .input(z.object({ params: zPostAppsByAppIdWorkflowRunsTasksByTaskIdStopPath })) .output(zPostAppsByAppIdWorkflowRunsTasksByTaskIdStopResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const stop3 = { post: post39, @@ -3677,8 +3644,6 @@ export const workflow = { } /** - * Get default block config - * * Get default block configuration by type */ export const get65 = oc @@ -3688,7 +3653,6 @@ export const get65 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsByBlockType', path: '/apps/{app_id}/workflows/default-workflow-block-configs/{block_type}', - summary: 'Get default block config', tags: ['console'], }) .input( @@ -3698,15 +3662,12 @@ export const get65 = oc }), ) .output(zGetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsByBlockTypeResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const byBlockType = { get: get65, } /** - * Get default block config - * * Get default block configurations for workflow */ export const get66 = oc @@ -3716,12 +3677,10 @@ export const get66 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDefaultWorkflowBlockConfigs', path: '/apps/{app_id}/workflows/default-workflow-block-configs', - summary: 'Get default block config', tags: ['console'], }) .input(z.object({ params: zGetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsPath })) .output(zGetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const defaultWorkflowBlockConfigs = { get: get66, @@ -3831,15 +3790,12 @@ export const post46 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftFeaturesResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const features = { post: post46, } /** - * Test human input delivery - * * Test human input delivery for workflow */ export const post47 = oc @@ -3849,7 +3805,6 @@ export const post47 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdDeliveryTest', path: '/apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/delivery-test', - summary: 'Test human input delivery', tags: ['console'], }) .input( @@ -3859,15 +3814,12 @@ export const post47 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdDeliveryTestResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const deliveryTest = { post: post47, } /** - * Preview human input form content and placeholders - * * Get human input form preview for workflow */ export const post48 = oc @@ -3877,7 +3829,6 @@ export const post48 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormPreview', path: '/apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/preview', - summary: 'Preview human input form content and placeholders', tags: ['console'], }) .input( @@ -3887,15 +3838,12 @@ export const post48 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormPreviewResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const preview4 = { post: post48, } /** - * Submit human input form preview - * * Submit human input form preview for workflow */ export const post49 = oc @@ -3905,7 +3853,6 @@ export const post49 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormRun', path: '/apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/run', - summary: 'Submit human input form preview', tags: ['console'], }) .input( @@ -3915,7 +3862,6 @@ export const post49 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run5 = { post: post49, @@ -3940,8 +3886,6 @@ export const humanInput2 = { } /** - * Run draft workflow iteration node - * * Run draft workflow iteration node */ export const post50 = oc @@ -3951,7 +3895,6 @@ export const post50 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftIterationNodesByNodeIdRun', path: '/apps/{app_id}/workflows/draft/iteration/nodes/{node_id}/run', - summary: 'Run draft workflow iteration node', tags: ['console'], }) .input( @@ -3961,7 +3904,6 @@ export const post50 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftIterationNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run6 = { post: post50, @@ -3980,8 +3922,6 @@ export const iteration2 = { } /** - * Run draft workflow loop node - * * Run draft workflow loop node */ export const post51 = oc @@ -3991,7 +3931,6 @@ export const post51 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftLoopNodesByNodeIdRun', path: '/apps/{app_id}/workflows/draft/loop/nodes/{node_id}/run', - summary: 'Run draft workflow loop node', tags: ['console'], }) .input( @@ -4001,7 +3940,6 @@ export const post51 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftLoopNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run7 = { post: post51, @@ -4210,15 +4148,12 @@ export const get71 = oc }) .input(z.object({ params: zGetAppsByAppIdWorkflowsDraftNodesByNodeIdLastRunPath })) .output(zGetAppsByAppIdWorkflowsDraftNodesByNodeIdLastRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const lastRun = { get: get71, } /** - * Run draft workflow node - * * Run draft workflow node */ export const post56 = oc @@ -4228,7 +4163,6 @@ export const post56 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftNodesByNodeIdRun', path: '/apps/{app_id}/workflows/draft/nodes/{node_id}/run', - summary: 'Run draft workflow node', tags: ['console'], }) .input( @@ -4238,15 +4172,12 @@ export const post56 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run8 = { post: post56, } /** - * Poll for trigger events and execute single node when event arrives - * * Poll for trigger events and execute single node when event arrives */ export const post57 = oc @@ -4256,12 +4187,10 @@ export const post57 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRun', path: '/apps/{app_id}/workflows/draft/nodes/{node_id}/trigger/run', - summary: 'Poll for trigger events and execute single node when event arrives', tags: ['console'], }) .input(z.object({ params: zPostAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRunPath })) .output(zPostAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run9 = { post: post57, @@ -4320,8 +4249,6 @@ export const nodes7 = { } /** - * Run draft workflow - * * Run draft workflow */ export const post58 = oc @@ -4331,7 +4258,6 @@ export const post58 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftRun', path: '/apps/{app_id}/workflows/draft/run', - summary: 'Run draft workflow', tags: ['console'], }) .input( @@ -4341,10 +4267,7 @@ export const post58 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftRunResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, - }) + .errors({ rate_limit_error: { status: 429, data: zRateLimitError } }) export const run10 = { post: post58, @@ -4484,8 +4407,6 @@ export const systemVariables = { } /** - * Poll for trigger events and execute full workflow when event arrives - * * Poll for trigger events and execute full workflow when event arrives */ export const post59 = oc @@ -4495,7 +4416,6 @@ export const post59 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftTriggerRun', path: '/apps/{app_id}/workflows/draft/trigger/run', - summary: 'Poll for trigger events and execute full workflow when event arrives', tags: ['console'], }) .input( @@ -4505,18 +4425,13 @@ export const post59 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftTriggerRunResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, - }) + .errors({ rate_limit_error: { status: 429, data: zRateLimitError } }) export const run11 = { post: post59, } /** - * Full workflow debug when the start node is a trigger - * * Full workflow debug when the start node is a trigger */ export const post60 = oc @@ -4526,7 +4441,6 @@ export const post60 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftTriggerRunAll', path: '/apps/{app_id}/workflows/draft/trigger/run-all', - summary: 'Full workflow debug when the start node is a trigger', tags: ['console'], }) .input( @@ -4536,10 +4450,7 @@ export const post60 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftTriggerRunAllResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, - }) + .errors({ rate_limit_error: { status: 429, data: zRateLimitError } }) export const runAll = { post: post60, @@ -4674,8 +4585,6 @@ export const variables2 = { } /** - * Get draft workflow - * * Get draft workflow for an application */ export const get80 = oc @@ -4685,19 +4594,13 @@ export const get80 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDraft', path: '/apps/{app_id}/workflows/draft', - summary: 'Get draft workflow', tags: ['console'], }) .input(z.object({ params: zGetAppsByAppIdWorkflowsDraftPath })) .output(zGetAppsByAppIdWorkflowsDraftResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, - }) + .errors({ draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError } }) /** - * Sync draft workflow - * * Sync draft workflow configuration */ export const post61 = oc @@ -4707,7 +4610,6 @@ export const post61 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraft', path: '/apps/{app_id}/workflows/draft', - summary: 'Sync draft workflow', tags: ['console'], }) .input( @@ -4717,10 +4619,7 @@ export const post61 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError }, - }) + .errors({ draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError } }) export const draft2 = { get: get80, @@ -4740,8 +4639,6 @@ export const draft2 = { } /** - * Get published workflow - * * Get published workflow for an application */ export const get81 = oc @@ -4751,23 +4648,17 @@ export const get81 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsPublish', path: '/apps/{app_id}/workflows/publish', - summary: 'Get published workflow', tags: ['console'], }) .input(z.object({ params: zGetAppsByAppIdWorkflowsPublishPath })) .output(zGetAppsByAppIdWorkflowsPublishResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) -/** - * Publish workflow - */ export const post62 = oc .route({ inputStructure: 'detailed', method: 'POST', operationId: 'postAppsByAppIdWorkflowsPublish', path: '/apps/{app_id}/workflows/publish', - summary: 'Publish workflow', tags: ['console'], }) .input( @@ -4777,7 +4668,6 @@ export const post62 = oc }), ) .output(zPostAppsByAppIdWorkflowsPublishResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const publish = { get: get81, @@ -4952,15 +4842,11 @@ export const post63 = oc }) .input(z.object({ params: zPostAppsByAppIdWorkflowsByWorkflowIdRestorePath })) .output(zPostAppsByAppIdWorkflowsByWorkflowIdRestoreResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const restore = { post: post63, } -/** - * Delete workflow - */ export const delete14 = oc .route({ inputStructure: 'detailed', @@ -4968,16 +4854,12 @@ export const delete14 = oc operationId: 'deleteAppsByAppIdWorkflowsByWorkflowId', path: '/apps/{app_id}/workflows/{workflow_id}', successStatus: 204, - summary: 'Delete workflow', tags: ['console'], }) .input(z.object({ params: zDeleteAppsByAppIdWorkflowsByWorkflowIdPath })) .output(zDeleteAppsByAppIdWorkflowsByWorkflowIdResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) /** - * Update workflow attributes - * * Update workflow by ID */ export const patch3 = oc @@ -4987,7 +4869,6 @@ export const patch3 = oc method: 'PATCH', operationId: 'patchAppsByAppIdWorkflowsByWorkflowId', path: '/apps/{app_id}/workflows/{workflow_id}', - summary: 'Update workflow attributes', tags: ['console'], }) .input( @@ -4997,7 +4878,6 @@ export const patch3 = oc }), ) .output(zPatchAppsByAppIdWorkflowsByWorkflowIdResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const byWorkflowId = { delete: delete14, @@ -5006,8 +4886,6 @@ export const byWorkflowId = { } /** - * Get published workflows - * * Get all published workflows for an application */ export const get87 = oc @@ -5017,7 +4895,6 @@ export const get87 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflows', path: '/apps/{app_id}/workflows', - summary: 'Get published workflows', tags: ['console'], }) .input( @@ -5027,7 +4904,6 @@ export const get87 = oc }), ) .output(zGetAppsByAppIdWorkflowsResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const workflows3 = { get: get87, diff --git a/packages/contracts/generated/api/console/errors.gen.ts b/packages/contracts/generated/api/console/errors.gen.ts index d9a043807f415c..f716e5ae8844b5 100644 --- a/packages/contracts/generated/api/console/errors.gen.ts +++ b/packages/contracts/generated/api/console/errors.gen.ts @@ -1,32 +1,24 @@ // This file is auto-generated by scripts/add-orpc-error-maps.ts export { delete10Errors as 'DELETE /apps/{app_id}/workflow/comments/{comment_id}' } from './apps/errors.gen.ts' export { delete10Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/research-tasks/{task_id}' } from './knowledge-fs/errors.gen.ts' -export { delete11Errors as 'DELETE /apps/{app_id}/workflows/draft/nodes/{node_id}/variables' } from './apps/errors.gen.ts' export { delete11Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/sources/{source_id}' } from './knowledge-fs/errors.gen.ts' -export { delete12Errors as 'DELETE /apps/{app_id}/workflows/draft/variables/{variable_id}' } from './apps/errors.gen.ts' export { delete12Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}' } from './knowledge-fs/errors.gen.ts' -export { delete13Errors as 'DELETE /apps/{app_id}/workflows/draft/variables' } from './apps/errors.gen.ts' export { delete13Errors as 'DELETE /knowledge-fs/uploads/{upload_id}' } from './knowledge-fs/errors.gen.ts' -export { delete14Errors as 'DELETE /apps/{app_id}/workflows/{workflow_id}' } from './apps/errors.gen.ts' export { delete16Errors as 'DELETE /workspaces/current/skills/{skill_id}' } from './workspaces/errors.gen.ts' export { delete2Errors as 'DELETE /agent/{agent_id}/build-draft' } from './agent/errors.gen.ts' export { delete2Errors as 'DELETE /apps/{app_id}/agent/config/skills/{name}' } from './apps/errors.gen.ts' export { delete2Errors as 'DELETE /datasets/external-knowledge-api/{external_knowledge_api_id}' } from './datasets/errors.gen.ts' export { delete2Errors as 'DELETE /installed-apps/{installed_app_id}/saved-messages/{message_id}' } from './installed-apps/errors.gen.ts' export { delete2Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/app-bindings/{caller_kind}/{app_id}' } from './knowledge-fs/errors.gen.ts' -export { delete2Errors as 'DELETE /rag/pipelines/{pipeline_id}/workflows/draft/nodes/{node_id}/variables' } from './rag/errors.gen.ts' export { delete3Errors as 'DELETE /agent/{agent_id}/config/files/{name}' } from './agent/errors.gen.ts' export { delete3Errors as 'DELETE /datasets/{dataset_id}/documents/{document_id}/segments/{segment_id}/child_chunks/{child_chunk_id}' } from './datasets/errors.gen.ts' export { delete3Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/documents/bulk' } from './knowledge-fs/errors.gen.ts' -export { delete3Errors as 'DELETE /rag/pipelines/{pipeline_id}/workflows/draft/variables/{variable_id}' } from './rag/errors.gen.ts' export { delete4Errors as 'DELETE /agent/{agent_id}/config/skills/{name}' } from './agent/errors.gen.ts' export { delete4Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/documents/{document_id}' } from './knowledge-fs/errors.gen.ts' -export { delete4Errors as 'DELETE /rag/pipelines/{pipeline_id}/workflows/draft/variables' } from './rag/errors.gen.ts' export { delete4Errors as 'DELETE /workspaces/current/members/{member_id}' } from './workspaces/errors.gen.ts' export { delete5Errors as 'DELETE /agent/{agent_id}/trace-config' } from './agent/errors.gen.ts' export { delete5Errors as 'DELETE /apps/{app_id}/chat-conversations/{conversation_id}' } from './apps/errors.gen.ts' export { delete5Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/golden-questions/{question_id}' } from './knowledge-fs/errors.gen.ts' -export { delete5Errors as 'DELETE /rag/pipelines/{pipeline_id}/workflows/{workflow_id}' } from './rag/errors.gen.ts' export { delete6Errors as 'DELETE /agent/{agent_id}' } from './agent/errors.gen.ts' export { delete6Errors as 'DELETE /apps/{app_id}/completion-conversations/{conversation_id}' } from './apps/errors.gen.ts' export { delete6Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/jobs/{job_id}' } from './knowledge-fs/errors.gen.ts' @@ -62,7 +54,6 @@ export { get13Errors as 'GET /trial-apps/{app_id}/messages/{message_id}/suggeste export { get14Errors as 'GET /agent/{agent_id}/config/skills/{name}/files/content' } from './agent/errors.gen.ts' export { get14Errors as 'GET /apps/{app_id}/agent/config/skills/{name}/inspect' } from './apps/errors.gen.ts' export { get14Errors as 'GET /knowledge-fs/spaces/{control_space_id}/documents/{document_id}/revisions' } from './knowledge-fs/errors.gen.ts' -export { get14Errors as 'GET /rag/pipelines/{pipeline_id}/workflows/draft/nodes/{node_id}/variables' } from './rag/errors.gen.ts' export { get14Errors as 'GET /snippets/{snippet_id}/workflows/draft/variables' } from './snippets/errors.gen.ts' export { get14Errors as 'GET /trial-apps/{app_id}/parameters' } from './trial-apps/errors.gen.ts' export { get15Errors as 'GET /agent/{agent_id}/config/skills/{name}/files/download' } from './agent/errors.gen.ts' @@ -72,19 +63,16 @@ export { get15Errors as 'GET /rag/pipelines/{pipeline_id}/workflows/draft/pre-pr export { get15Errors as 'GET /snippets/{snippet_id}/workflows/draft' } from './snippets/errors.gen.ts' export { get15Errors as 'GET /trial-apps/{app_id}/site' } from './trial-apps/errors.gen.ts' export { get16Errors as 'GET /agent/{agent_id}/config/skills/{name}/files/preview' } from './agent/errors.gen.ts' -export { get16Errors as 'GET /apps/{app_id}/agent/logs' } from './apps/errors.gen.ts' export { get16Errors as 'GET /datasets/{dataset_id}/documents/{document_id}/indexing-estimate' } from './datasets/errors.gen.ts' export { get16Errors as 'GET /knowledge-fs/spaces/{control_space_id}/documents' } from './knowledge-fs/errors.gen.ts' export { get16Errors as 'GET /rag/pipelines/{pipeline_id}/workflows/draft/processing/parameters' } from './rag/errors.gen.ts' export { get16Errors as 'GET /trial-apps/{app_id}/workflows' } from './trial-apps/errors.gen.ts' export { get17Errors as 'GET /agent/{agent_id}/config/skills/{name}/inspect' } from './agent/errors.gen.ts' export { get17Errors as 'GET /knowledge-fs/spaces/{control_space_id}/external-access' } from './knowledge-fs/errors.gen.ts' -export { get17Errors as 'GET /rag/pipelines/{pipeline_id}/workflows/draft/system-variables' } from './rag/errors.gen.ts' export { get17Errors as 'GET /trial-apps/{app_id}' } from './trial-apps/errors.gen.ts' export { get18Errors as 'GET /agent/{agent_id}/config/skills' } from './agent/errors.gen.ts' export { get18Errors as 'GET /datasets/{dataset_id}/documents/{document_id}/notion/sync' } from './datasets/errors.gen.ts' export { get18Errors as 'GET /knowledge-fs/spaces/{control_space_id}/golden-questions' } from './knowledge-fs/errors.gen.ts' -export { get18Errors as 'GET /rag/pipelines/{pipeline_id}/workflows/draft/variables/{variable_id}' } from './rag/errors.gen.ts' export { get19Errors as 'GET /agent/{agent_id}/log-sources' } from './agent/errors.gen.ts' export { get19Errors as 'GET /knowledge-fs/spaces/{control_space_id}/jobs/{job_id}' } from './knowledge-fs/errors.gen.ts' export { get19Errors as 'GET /rag/pipelines/{pipeline_id}/workflows/draft/variables' } from './rag/errors.gen.ts' @@ -225,9 +213,6 @@ export { get61Errors as 'GET /apps/{app_id}/workflow/statistics/average-app-inte export { get62Errors as 'GET /apps/{app_id}/workflow/statistics/daily-conversations' } from './apps/errors.gen.ts' export { get63Errors as 'GET /apps/{app_id}/workflow/statistics/daily-terminals' } from './apps/errors.gen.ts' export { get64Errors as 'GET /apps/{app_id}/workflow/statistics/token-costs' } from './apps/errors.gen.ts' -export { get65Errors as 'GET /apps/{app_id}/workflows/default-workflow-block-configs/{block_type}' } from './apps/errors.gen.ts' -export { get66Errors as 'GET /apps/{app_id}/workflows/default-workflow-block-configs' } from './apps/errors.gen.ts' -export { get67Errors as 'GET /apps/{app_id}/workflows/draft/conversation-variables' } from './apps/errors.gen.ts' export { get68Errors as 'GET /apps/{app_id}/workflows/draft/environment-variables' } from './apps/errors.gen.ts' export { get69Errors as 'GET /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/candidates' } from './apps/errors.gen.ts' export { get6Errors as 'GET /agent/{agent_id}/chat-messages' } from './agent/errors.gen.ts' @@ -237,14 +222,10 @@ export { get6Errors as 'GET /knowledge-fs/spaces/{control_space_id}/background-t export { get6Errors as 'GET /rag/pipelines/{pipeline_id}/exports' } from './rag/errors.gen.ts' export { get6Errors as 'GET /trial-apps/{app_id}/agent/config/skills/{name}/files/content' } from './trial-apps/errors.gen.ts' export { get70Errors as 'GET /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer' } from './apps/errors.gen.ts' -export { get71Errors as 'GET /apps/{app_id}/workflows/draft/nodes/{node_id}/last-run' } from './apps/errors.gen.ts' -export { get72Errors as 'GET /apps/{app_id}/workflows/draft/nodes/{node_id}/variables' } from './apps/errors.gen.ts' export { get73Errors as 'GET /apps/{app_id}/workflows/draft/runs/{run_id}/node-outputs/events' } from './apps/errors.gen.ts' export { get74Errors as 'GET /apps/{app_id}/workflows/draft/runs/{run_id}/node-outputs/{node_id}/{output_name}/preview' } from './apps/errors.gen.ts' export { get75Errors as 'GET /apps/{app_id}/workflows/draft/runs/{run_id}/node-outputs/{node_id}' } from './apps/errors.gen.ts' export { get76Errors as 'GET /apps/{app_id}/workflows/draft/runs/{run_id}/node-outputs' } from './apps/errors.gen.ts' -export { get77Errors as 'GET /apps/{app_id}/workflows/draft/system-variables' } from './apps/errors.gen.ts' -export { get78Errors as 'GET /apps/{app_id}/workflows/draft/variables/{variable_id}' } from './apps/errors.gen.ts' export { get79Errors as 'GET /apps/{app_id}/workflows/draft/variables' } from './apps/errors.gen.ts' export { get7Errors as 'GET /account/profile' } from './account/errors.gen.ts' export { get7Errors as 'GET /agent/{agent_id}/composer/candidates' } from './agent/errors.gen.ts' @@ -255,7 +236,6 @@ export { get7Errors as 'GET /rag/pipelines/{pipeline_id}/workflow-runs/{run_id}/ export { get7Errors as 'GET /snippets/{snippet_id}/workflows/draft/environment-variables' } from './snippets/errors.gen.ts' export { get7Errors as 'GET /trial-apps/{app_id}/agent/config/skills/{name}/files/download' } from './trial-apps/errors.gen.ts' export { get80Errors as 'GET /apps/{app_id}/workflows/draft' } from './apps/errors.gen.ts' -export { get81Errors as 'GET /apps/{app_id}/workflows/publish' } from './apps/errors.gen.ts' export { get81Errors as 'GET /workspaces/current/skills/{skill_id}/files/content' } from './workspaces/errors.gen.ts' export { get82Errors as 'GET /apps/{app_id}/workflows/published/runs/{run_id}/node-outputs/events' } from './apps/errors.gen.ts' export { get82Errors as 'GET /workspaces/current/skills/{skill_id}/files/preview' } from './workspaces/errors.gen.ts' @@ -263,7 +243,6 @@ export { get83Errors as 'GET /apps/{app_id}/workflows/published/runs/{run_id}/no export { get84Errors as 'GET /apps/{app_id}/workflows/published/runs/{run_id}/node-outputs/{node_id}' } from './apps/errors.gen.ts' export { get85Errors as 'GET /apps/{app_id}/workflows/published/runs/{run_id}/node-outputs' } from './apps/errors.gen.ts' export { get86Errors as 'GET /apps/{app_id}/workflows/triggers/webhook' } from './apps/errors.gen.ts' -export { get87Errors as 'GET /apps/{app_id}/workflows' } from './apps/errors.gen.ts' export { get88Errors as 'GET /workspaces/current/summary' } from './workspaces/errors.gen.ts' export { get8Errors as 'GET /agent/{agent_id}/composer' } from './agent/errors.gen.ts' export { get8Errors as 'GET /apps/{app_id}/agent/config/files' } from './apps/errors.gen.ts' @@ -286,12 +265,9 @@ export { getErrors as 'GET /installed-apps/{installed_app_id}/conversations' } f export { getErrors as 'GET /remote-files/{url}' } from './remote-files/errors.gen.ts' export { getErrors as 'GET /trial-apps/{app_id}/agent-composer' } from './trial-apps/errors.gen.ts' export { getErrors as 'GET /website/crawl/status/{job_id}' } from './website/errors.gen.ts' -export { patch2Errors as 'PATCH /apps/{app_id}/workflows/draft/variables/{variable_id}' } from './apps/errors.gen.ts' export { patch2Errors as 'PATCH /datasets/{dataset_id}/documents/status/{action}/batch' } from './datasets/errors.gen.ts' export { patch2Errors as 'PATCH /installed-apps/{installed_app_id}/conversations/{c_id}/unpin' } from './installed-apps/errors.gen.ts' export { patch2Errors as 'PATCH /knowledge-fs/spaces/{control_space_id}/golden-questions/{question_id}' } from './knowledge-fs/errors.gen.ts' -export { patch2Errors as 'PATCH /rag/pipelines/{pipeline_id}/workflows/draft/variables/{variable_id}' } from './rag/errors.gen.ts' -export { patch3Errors as 'PATCH /apps/{app_id}/workflows/{workflow_id}' } from './apps/errors.gen.ts' export { patch3Errors as 'PATCH /datasets/{dataset_id}/documents/{document_id}/processing/pause' } from './datasets/errors.gen.ts' export { patch3Errors as 'PATCH /installed-apps/{installed_app_id}' } from './installed-apps/errors.gen.ts' export { patch3Errors as 'PATCH /knowledge-fs/spaces/{control_space_id}/logical-documents/{document_id}' } from './knowledge-fs/errors.gen.ts' @@ -356,7 +332,6 @@ export { post15Errors as 'POST /workspaces/current/members/{member_id}/owner-tra export { post16Errors as 'POST /agent/{agent_id}/sandbox/files/download' } from './agent/errors.gen.ts' export { post16Errors as 'POST /datasets/{dataset_id}/external-hit-testing' } from './datasets/errors.gen.ts' export { post16Errors as 'POST /knowledge-fs/spaces/{control_space_id}/queries/admission' } from './knowledge-fs/errors.gen.ts' -export { post16Errors as 'POST /rag/pipelines/{pipeline_id}/workflows/publish' } from './rag/errors.gen.ts' export { post17Errors as 'POST /agent/{agent_id}/text-to-audio' } from './agent/errors.gen.ts' export { post17Errors as 'POST /apps/{app_id}/audio-to-text' } from './apps/errors.gen.ts' export { post17Errors as 'POST /datasets/{dataset_id}/hit-testing' } from './datasets/errors.gen.ts' @@ -373,9 +348,7 @@ export { post19Errors as 'POST /rag/pipelines/{pipeline_id}/workflows/published/ export { post20Errors as 'POST /agent/{agent_id}/trace-runs/{message_id}/retry' } from './agent/errors.gen.ts' export { post20Errors as 'POST /apps/{app_id}/completion-messages' } from './apps/errors.gen.ts' export { post20Errors as 'POST /knowledge-fs/spaces/{control_space_id}/research-tasks' } from './knowledge-fs/errors.gen.ts' -export { post20Errors as 'POST /rag/pipelines/{pipeline_id}/workflows/{workflow_id}/restore' } from './rag/errors.gen.ts' export { post21Errors as 'POST /agent/{agent_id}/versions/{version_id}/restore' } from './agent/errors.gen.ts' -export { post21Errors as 'POST /apps/{app_id}/convert-to-workflow' } from './apps/errors.gen.ts' export { post21Errors as 'POST /knowledge-fs/spaces/{control_space_id}/source-connections/{connection_id}/refresh' } from './knowledge-fs/errors.gen.ts' export { post22Errors as 'POST /agent' } from './agent/errors.gen.ts' export { post22Errors as 'POST /knowledge-fs/spaces/{control_space_id}/source-connections' } from './knowledge-fs/errors.gen.ts' @@ -429,7 +402,6 @@ export { post37Errors as 'POST /workspaces/current/plugin/tasks/{task_id}/delete export { post38Errors as 'POST /apps/{app_id}/trigger-enable' } from './apps/errors.gen.ts' export { post38Errors as 'POST /knowledge-fs/spaces/{control_space_id}/upload-sessions/{upload_session_id}/small-file' } from './knowledge-fs/errors.gen.ts' export { post38Errors as 'POST /workspaces/current/plugin/tasks/{task_id}/delete' } from './workspaces/errors.gen.ts' -export { post39Errors as 'POST /apps/{app_id}/workflow-runs/tasks/{task_id}/stop' } from './apps/errors.gen.ts' export { post39Errors as 'POST /knowledge-fs/spaces/{control_space_id}/upload-sessions' } from './knowledge-fs/errors.gen.ts' export { post39Errors as 'POST /workspaces/current/plugin/uninstall' } from './workspaces/errors.gen.ts' export { post3Errors as 'POST /account/change-email/reset' } from './account/errors.gen.ts' @@ -451,25 +423,16 @@ export { post42Errors as 'POST /apps/{app_id}/workflow/comments/{comment_id}/res export { post42Errors as 'POST /knowledge-fs/uploads' } from './knowledge-fs/errors.gen.ts' export { post43Errors as 'POST /apps/{app_id}/workflow/comments' } from './apps/errors.gen.ts' export { post43Errors as 'POST /workspaces/current/plugin/upload/github' } from './workspaces/errors.gen.ts' -export { post44Errors as 'POST /apps/{app_id}/workflows/draft/conversation-variables' } from './apps/errors.gen.ts' export { post44Errors as 'POST /workspaces/current/plugin/upload/pkg' } from './workspaces/errors.gen.ts' -export { post45Errors as 'POST /apps/{app_id}/workflows/draft/environment-variables' } from './apps/errors.gen.ts' -export { post46Errors as 'POST /apps/{app_id}/workflows/draft/features' } from './apps/errors.gen.ts' -export { post47Errors as 'POST /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/delivery-test' } from './apps/errors.gen.ts' -export { post48Errors as 'POST /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/preview' } from './apps/errors.gen.ts' -export { post49Errors as 'POST /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/run' } from './apps/errors.gen.ts' export { post49Errors as 'POST /workspaces/current/skills/files/upload' } from './workspaces/errors.gen.ts' export { post4Errors as 'POST /account/change-email/validity' } from './account/errors.gen.ts' export { post4Errors as 'POST /agent/{agent_id}/build-chat/finalize' } from './agent/errors.gen.ts' -export { post4Errors as 'POST /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/preview' } from './apps/errors.gen.ts' export { post4Errors as 'POST /datasets/indexing-estimate' } from './datasets/errors.gen.ts' export { post4Errors as 'POST /installed-apps/{installed_app_id}/completion-messages/{task_id}/stop' } from './installed-apps/errors.gen.ts' export { post4Errors as 'POST /knowledge-fs/spaces/{control_space_id}/background-tasks/{task_kind}/{task_id}/cancel' } from './knowledge-fs/errors.gen.ts' export { post4Errors as 'POST /snippets/{snippet_id}/workflows/draft/nodes/{node_id}/agent-composer/copy-from-roster' } from './snippets/errors.gen.ts' export { post4Errors as 'POST /trial-apps/{app_id}/completion-messages' } from './trial-apps/errors.gen.ts' -export { post50Errors as 'POST /apps/{app_id}/workflows/draft/iteration/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post50Errors as 'POST /workspaces/current/skills/import' } from './workspaces/errors.gen.ts' -export { post51Errors as 'POST /apps/{app_id}/workflows/draft/loop/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post51Errors as 'POST /workspaces/current/skills/{skill_id}/assist/messages' } from './workspaces/errors.gen.ts' export { post52Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/copy-from-roster' } from './apps/errors.gen.ts' export { post53Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/impact' } from './apps/errors.gen.ts' @@ -478,25 +441,19 @@ export { post54Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/ag export { post54Errors as 'POST /workspaces/current/skills/{skill_id}/publish' } from './workspaces/errors.gen.ts' export { post55Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/validate' } from './apps/errors.gen.ts' export { post55Errors as 'POST /workspaces/current/skills/{skill_id}/restore' } from './workspaces/errors.gen.ts' -export { post56Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post56Errors as 'POST /workspaces/current/skills' } from './workspaces/errors.gen.ts' -export { post57Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/trigger/run' } from './apps/errors.gen.ts' export { post58Errors as 'POST /apps/{app_id}/workflows/draft/run' } from './apps/errors.gen.ts' export { post59Errors as 'POST /apps/{app_id}/workflows/draft/trigger/run' } from './apps/errors.gen.ts' export { post5Errors as 'POST /account/change-email' } from './account/errors.gen.ts' export { post5Errors as 'POST /agent/{agent_id}/build-draft/apply' } from './agent/errors.gen.ts' -export { post5Errors as 'POST /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/run' } from './apps/errors.gen.ts' export { post5Errors as 'POST /datasets/init' } from './datasets/errors.gen.ts' export { post5Errors as 'POST /installed-apps/{installed_app_id}/completion-messages' } from './installed-apps/errors.gen.ts' export { post5Errors as 'POST /knowledge-fs/spaces/{control_space_id}/background-tasks/{task_kind}/{task_id}/retry' } from './knowledge-fs/errors.gen.ts' export { post5Errors as 'POST /trial-apps/{app_id}/files/upload' } from './trial-apps/errors.gen.ts' export { post60Errors as 'POST /apps/{app_id}/workflows/draft/trigger/run-all' } from './apps/errors.gen.ts' export { post61Errors as 'POST /apps/{app_id}/workflows/draft' } from './apps/errors.gen.ts' -export { post62Errors as 'POST /apps/{app_id}/workflows/publish' } from './apps/errors.gen.ts' -export { post63Errors as 'POST /apps/{app_id}/workflows/{workflow_id}/restore' } from './apps/errors.gen.ts' export { post64Errors as 'POST /apps/{resource_id}/api-keys' } from './apps/errors.gen.ts' export { post6Errors as 'POST /agent/{agent_id}/build-draft/checkout' } from './agent/errors.gen.ts' -export { post6Errors as 'POST /apps/{app_id}/advanced-chat/workflows/draft/iteration/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post6Errors as 'POST /datasets/notion-indexing-estimate' } from './datasets/errors.gen.ts' export { post6Errors as 'POST /installed-apps/{installed_app_id}/conversations/{c_id}/name' } from './installed-apps/errors.gen.ts' export { post6Errors as 'POST /knowledge-fs/spaces/{control_space_id}/documents/reindex' } from './knowledge-fs/errors.gen.ts' @@ -504,7 +461,6 @@ export { post6Errors as 'POST /snippets/{snippet_id}/workflows/draft/nodes/{node export { post6Errors as 'POST /trial-apps/{app_id}/remote-files/upload' } from './trial-apps/errors.gen.ts' export { post7Errors as 'POST /account/delete' } from './account/errors.gen.ts' export { post7Errors as 'POST /agent/{agent_id}/chat-messages/{task_id}/stop' } from './agent/errors.gen.ts' -export { post7Errors as 'POST /apps/{app_id}/advanced-chat/workflows/draft/loop/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post7Errors as 'POST /installed-apps/{installed_app_id}/messages/{message_id}/feedbacks' } from './installed-apps/errors.gen.ts' export { post7Errors as 'POST /knowledge-fs/spaces/{control_space_id}/documents' } from './knowledge-fs/errors.gen.ts' export { post7Errors as 'POST /rag/pipelines/{pipeline_id}/customized/publish' } from './rag/errors.gen.ts' @@ -558,10 +514,8 @@ export { put4Errors as 'PUT /apps/{app_id}/workflow/comments/{comment_id}' } fro export { put4Errors as 'PUT /knowledge-fs/spaces/{control_space_id}/sources/{source_id}/sync-policy' } from './knowledge-fs/errors.gen.ts' export { put5Errors as 'PUT /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer' } from './apps/errors.gen.ts' export { put5Errors as 'PUT /knowledge-fs/spaces/{control_space_id}/tags' } from './knowledge-fs/errors.gen.ts' -export { put6Errors as 'PUT /apps/{app_id}/workflows/draft/variables/{variable_id}/reset' } from './apps/errors.gen.ts' export { putErrors as 'PUT /agent/{agent_id}/build-draft' } from './agent/errors.gen.ts' export { putErrors as 'PUT /apps/{app_id}/network-access-group' } from './apps/errors.gen.ts' export { putErrors as 'PUT /knowledge-fs/spaces/{control_space_id}/app-bindings' } from './knowledge-fs/errors.gen.ts' -export { putErrors as 'PUT /rag/pipelines/{pipeline_id}/workflows/draft/variables/{variable_id}/reset' } from './rag/errors.gen.ts' export { putErrors as 'PUT /snippets/{snippet_id}/workflows/draft/nodes/{node_id}/agent-composer' } from './snippets/errors.gen.ts' export { putErrors as 'PUT /workspaces/current/agents/{agent_id}/skills' } from './workspaces/errors.gen.ts' diff --git a/packages/contracts/generated/api/console/rag/errors.gen.ts b/packages/contracts/generated/api/console/rag/errors.gen.ts index f87498e70afbc6..6ba400dabbb13c 100644 --- a/packages/contracts/generated/api/console/rag/errors.gen.ts +++ b/packages/contracts/generated/api/console/rag/errors.gen.ts @@ -78,7 +78,6 @@ export const post10Errors = { pipeline_not_found: { status: 404, data: zPipeline export const get12Errors = { draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, - pipeline_not_found: { status: 404, data: zPipelineNotFoundError }, } export const post11Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } @@ -89,10 +88,6 @@ export const get13Errors = { pipeline_not_found: { status: 404, data: zPipelineN export const post13Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } -export const delete2Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - -export const get14Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - export const get15Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } export const get16Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } @@ -102,21 +97,8 @@ export const post14Errors = { rate_limit_error: { status: 429, data: zRateLimitError }, } -export const get17Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - -export const putErrors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - -export const delete3Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - -export const get18Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - -export const patch2Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - -export const delete4Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - export const get19Errors = { draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, - pipeline_not_found: { status: 404, data: zPipelineNotFoundError }, } export const get20Errors = { @@ -126,13 +108,10 @@ export const get20Errors = { export const post15Errors = { draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError }, - pipeline_not_found: { status: 404, data: zPipelineNotFoundError }, } export const get21Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } -export const post16Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - export const post17Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } export const post18Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } @@ -146,10 +125,6 @@ export const post19Errors = { rate_limit_error: { status: 429, data: zRateLimitError }, } -export const post20Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - -export const delete5Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - export const patch3Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } export const get24Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } From 63f40656437a42f953c6ddd009e37c817c7712da Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Mon, 26 Oct 2026 22:25:27 +0800 Subject: [PATCH 12/20] Complete canonical test migration and preserve validation coverage --- .github/workflows/knowledge-fs-ci.yml | 4 +- .../app/test_message_suggested_questions.py | 4 + .../console/snippets/test_snippet_workflow.py | 32 +- .../openapi/auth/test_resource_access.py | 7 +- .../nodes/agent_v2/test_binding_resolver.py | 8 +- .../test_workflow_tool_container_handler.py | 48 ++- .../test_app_definition_query_repository.py | 18 +- ...t_app_definition_query_repository_agent.py | 394 ------------------ .../test_runner_retrieval_boundary.py | 3 +- .../unit_tests/services/test_agent_service.py | 170 ++++---- ..._generate_service_streaming_integration.py | 221 ---------- .../services/test_snippet_dsl_service.py | 9 +- .../services/test_snippet_generate_service.py | 4 +- .../test_resource_cleanup_service.py | 2 +- 14 files changed, 162 insertions(+), 762 deletions(-) delete mode 100644 api/tests/unit_tests/repositories/test_app_definition_query_repository_agent.py delete mode 100644 api/tests/unit_tests/services/test_app_generate_service_streaming_integration.py diff --git a/.github/workflows/knowledge-fs-ci.yml b/.github/workflows/knowledge-fs-ci.yml index 2e1740d1c6e545..ac45764d8bfddb 100644 --- a/.github/workflows/knowledge-fs-ci.yml +++ b/.github/workflows/knowledge-fs-ci.yml @@ -92,7 +92,7 @@ jobs: - 'api/tests/unit_tests/core/workflow/test_node_runtime.py' - 'api/tests/unit_tests/services/enterprise/test_rbac_service.py' - 'api/tests/unit_tests/repositories/workspace/test_workspace_repository.py' - - 'api/tests/unit_tests/services/test_agent_tool_inner_service.py' + - 'api/tests/unit_tests/services/agent/test_tool_invocation_service.py' - 'api/tests/unit_tests/services/test_vector_space_admission_service.py' - 'api/tests/unit_tests/services/test_knowledge_fs_capability.py' - 'api/tests/unit_tests/services/knowledge_fs/test_product_operations.py' @@ -268,7 +268,7 @@ jobs: api/tests/unit_tests/core/workflow/test_node_runtime.py api/tests/unit_tests/services/enterprise/test_rbac_service.py api/tests/unit_tests/repositories/workspace/test_workspace_repository.py - api/tests/unit_tests/services/test_agent_tool_inner_service.py + api/tests/unit_tests/services/agent/test_tool_invocation_service.py api/tests/unit_tests/services/test_vector_space_admission_service.py api/tests/unit_tests/services/test_tidb_binding_service.py ) diff --git a/api/tests/e2e_tests/contracts/controllers/console/app/test_message_suggested_questions.py b/api/tests/e2e_tests/contracts/controllers/console/app/test_message_suggested_questions.py index 87b21be992eecf..19b1778b17541c 100644 --- a/api/tests/e2e_tests/contracts/controllers/console/app/test_message_suggested_questions.py +++ b/api/tests/e2e_tests/contracts/controllers/console/app/test_message_suggested_questions.py @@ -45,6 +45,8 @@ from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository from repositories.message_repository import MessageRepository from repositories.recommended_app_catalog_repository import DatabaseRecommendedAppCatalogRepository +from repositories.tools.provider_repository import ToolProviderRepository +from repositories.tools.workflow_repository import WorkflowToolRepository from services.agent.roster_package_exporter import RosterAgentPackageExporter from services.app.agent_app_service import AgentAppAccessService from services.app_tracing_config_gateway import OpsTraceManagerGateway @@ -194,6 +196,8 @@ def track_session(session: Session, _transaction: SessionTransaction, _connectio sources=DatabaseRecommendedAppCatalogRepository(sqlite_session_factory, redis=redis), exporter=RosterAgentPackageExporter(), ), + tool_providers=ToolProviderRepository(sqlite_session_factory), + workflow_queries=WorkflowToolRepository(sqlite_session_factory), ) flask_app = Flask(__name__) flask_app.config.update(TESTING=True, RESTX_ERROR_404_HELP=False, SQLALCHEMY_DATABASE_URI=str(sqlite_engine.url)) diff --git a/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py b/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py index 73fc1dd76f119b..1c00ff7d6ea896 100644 --- a/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py +++ b/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py @@ -2,7 +2,7 @@ import json from inspect import unwrap -from typing import NoReturn +from typing import NoReturn, TypedDict, Unpack from unittest.mock import Mock import pytest @@ -32,17 +32,25 @@ def _account(account_id: str = "account-1") -> Account: ) -def _snippet(**overrides: object) -> CustomizedSnippet: - data: dict[str, object] = { - "id": "snippet-1", - "tenant_id": "tenant-1", - "name": "Snippet", - "description": "Description", - "type": "node", - "created_by": "account-1", - } - data.update(overrides) - return CustomizedSnippet(**data) +class _SnippetOverrides(TypedDict, total=False): + is_published: bool + input_fields: str | None + + +def _snippet(**overrides: Unpack[_SnippetOverrides]) -> CustomizedSnippet: + snippet = CustomizedSnippet( + id="snippet-1", + tenant_id="tenant-1", + name="Snippet", + description="Description", + type="node", + created_by="account-1", + ) + if "is_published" in overrides: + snippet.is_published = overrides["is_published"] + if "input_fields" in overrides: + snippet.input_fields = overrides["input_fields"] + return snippet def _workflow(**overrides: object) -> Workflow: diff --git a/api/tests/unit_tests/controllers/openapi/auth/test_resource_access.py b/api/tests/unit_tests/controllers/openapi/auth/test_resource_access.py index 0e98ac53a2803a..ad33808115bb09 100644 --- a/api/tests/unit_tests/controllers/openapi/auth/test_resource_access.py +++ b/api/tests/unit_tests/controllers/openapi/auth/test_resource_access.py @@ -10,6 +10,7 @@ from sqlalchemy.orm import Session from werkzeug.exceptions import Forbidden, Unauthorized +from controllers.common.errors import NotFoundError from controllers.openapi._catalog import CATALOG_HEADER, catalog_for from controllers.openapi.auth.context import Context from controllers.openapi.auth.requirements import CheckScope, CheckSubject @@ -259,8 +260,6 @@ def test_stop_task_rejects_other_token_owner( ) -> None: import inspect - from werkzeug.exceptions import NotFound - from controllers.openapi.app_run import AppRunTaskStopApi flask_app, app_id, _, _, end_user = resource_fixture @@ -274,8 +273,10 @@ def test_stop_task_rejects_other_token_owner( "controllers.openapi.app_run.application_services", return_value=replace(account_application_services, app_tasks=AppTaskControlService(redis_client=redis)), ): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError, match="Task not found") as raised: inspect.unwrap(AppRunTaskStopApi.post)(AppRunTaskStopApi(), data, app_id, "task") + assert raised.value.code == 404 + assert raised.value.error_code == "not_found" assert redis.operations == [] redis.values["generate_task_belong:task"] = b"end-user-machine-user" inspect.unwrap(AppRunTaskStopApi.post)(AppRunTaskStopApi(), data, app_id, "task") diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py index 0f9c7d018f6160..8c2924337af91e 100644 --- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py +++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py @@ -50,7 +50,7 @@ def test_node_binding_resolution_uses_injected_database_through_nested_factories from enums import DeploymentEdition from extensions.ext_application_services import build_application_services from extensions.ext_redis import RedisClientWrapper - from graphon.runtime import GraphRuntimeState, VariablePool + from graphon.runtime import RuntimeState, VariablePool from models.base import TypeBase from services.workflow.execution.adapters.agent_v2.agent_node import DifyAgentNode from services.workflow.execution.adapters.node_factory import DifyGraphInitContext, DifyNodeFactory @@ -109,11 +109,13 @@ def test_node_binding_resolution_uses_injected_database_through_nested_factories with app.app_context(): factory = DifyNodeFactory.from_graph_init_context( graph_init_context=context, - graph_runtime_state=GraphRuntimeState(variable_pool=VariablePool(), start_at=0), + runtime_state=RuntimeState(workflow_id=ids["workflow_id"], variable_pool=VariablePool(), start_at=0), agent_binding_resolver=resolver, workflow_runtime=services.workflow_runtime, ) - nested = factory.with_runtime_state(GraphRuntimeState(variable_pool=VariablePool(), start_at=0)) + nested = factory.with_runtime_state( + RuntimeState(workflow_id=ids["workflow_id"], variable_pool=VariablePool(), start_at=0) + ) for owner in (factory, nested): node = owner.create_node( {"id": ids["node_id"], "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}} diff --git a/api/tests/unit_tests/core/workflow/test_workflow_tool_container_handler.py b/api/tests/unit_tests/core/workflow/test_workflow_tool_container_handler.py index 6bf6490207b652..29ab7a427b1875 100644 --- a/api/tests/unit_tests/core/workflow/test_workflow_tool_container_handler.py +++ b/api/tests/unit_tests/core/workflow/test_workflow_tool_container_handler.py @@ -10,6 +10,7 @@ from threading import Event, get_ident from types import TracebackType from typing import override +from unittest.mock import create_autospec import pytest from flask import current_app, g @@ -41,25 +42,18 @@ from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_entities import ToolEntity, ToolIdentity from core.tools.workflow_as_tool.repository import WorkflowToolSource -from core.tools.workflow_as_tool.tool import WorkflowTool from core.workflow.node_execution_process_data import ( WORKFLOW_TOOL_INVOCATION_ID_KEY, ) -from core.workflow.node_factory import DifyNodeFactory -from core.workflow.node_runtime import DifyToolFileManager, DifyToolNodeRuntime -from core.workflow.nodes.agent_v2.binding_resolver import WorkflowAgentBindingResolver from core.workflow.nodes.human_input.boundary import resolve_human_input_node_id from core.workflow.nodes.human_input.callback import DifyHITLCallback from core.workflow.system_variables import SystemVariableKey, system_variable_selector from core.workflow.workflow_entry import WorkflowEntry -from core.workflow.workflow_tool_container_handler import ( - WorkflowToolContainerHandler, - WorkflowToolNestedContainerHandler, -) from core.workflow.workflow_tool_container_types import WorkflowToolContainerPayload from core.workflow.workflow_tool_node import DifyWorkflowToolNode from dify_app import DifyApp from enums import WorkflowKind +from extensions.application_services.workflow import WorkflowExecutionDependencies from extensions.storage.storage_type import StorageType from graphon.engine import Engine from graphon.engine.command import AbortCommand, InMemoryChannel @@ -139,7 +133,16 @@ from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom from models.human_input_entities import HumanInputNodeData, UserActionConfig from models.model import AppMode +from models.tool_runtime_contracts import WorkflowToolQueries from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom +from repositories.agent.runtime_repository import WorkflowAgentBindingResolver +from services.tools.workflow.tool import WorkflowTool +from services.workflow.execution.adapters.node_factory import DifyNodeFactory +from services.workflow.execution.adapters.node_runtime import DifyToolFileManager, DifyToolNodeRuntime +from services.workflow.execution.adapters.workflow_tool_container_handler import ( + WorkflowToolContainerHandler, + WorkflowToolNestedContainerHandler, +) from services.workflow_persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer from services.workflow_run_index import WorkflowRunIndex from tests.helpers.unit.config_override import apply_config_overrides @@ -212,6 +215,8 @@ def _workflow_tool_container_workflow_tool_node( has_runtime_parameters=False, ), runtime=ToolRuntime(tenant_id="tenant", invoke_from=InvokeFrom.DEBUGGER), + queries=create_autospec(WorkflowToolQueries, instance=True, spec_set=True), + draft_variable_saver=None, ) tool.prepare_container_inputs = lambda *_args, **_kwargs: (payload.inputs, []) runtime = DifyToolNodeRuntime(init_params.run_context) @@ -899,7 +904,9 @@ def execute(**_kwargs: object) -> dict[str, str]: raise CodeExecutionError("source code failed") return {"output": "retried output"} - monkeypatch.setattr("core.workflow.node_factory.CodeExecutor.execute_workflow_code_template", execute) + monkeypatch.setattr( + "services.workflow.execution.adapters.node_factory.CodeExecutor.execute_workflow_code_template", execute + ) source = _workflow_tool_container_workflow_tool_source() source_graph = dict(source.graph_config) source_graph["nodes"].insert( @@ -1044,7 +1051,7 @@ def build_human_input_callback( node_data: HumanInputNodeData, execution_id_getter: Callable[[], str | None], ) -> DifyHITLCallback: - human_input_app_ids.append(factory._human_input_runtime._run_context.app_id) + human_input_app_ids.append(factory.human_input_run_context.app_id) return DifyHITLCallback( form_repository=form_repository, node_data=node_data, @@ -1416,7 +1423,9 @@ def edge(source: str, target: str, handle: str = "source") -> dict[str, str]: return {"id": f"{source}-{target}", "source": source, "target": target, "sourceHandle": handle} tool_node, runtime, _ = _workflow_tool_event_delivery_workflow_tool_node() - monkeypatch.setattr("core.workflow.node_factory.DifyToolNodeRuntime", lambda _, **_kwargs: runtime) + monkeypatch.setattr( + "services.workflow.execution.adapters.node_factory.DifyToolNodeRuntime", lambda _, **_kwargs: runtime + ) outer_graph = { "nodes": [ node("start", "start", variables=[]), @@ -1660,9 +1669,10 @@ def listener_factory( def test_workflow_tool_agent_finds_its_persisted_caller_before_resolving_binding( monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], + workflow_runtime: WorkflowExecutionDependencies, caller_save_fails: bool, ) -> None: - binding_resolver = WorkflowAgentBindingResolver() + binding_resolver = WorkflowAgentBindingResolver(sqlite_session_factory) resolution_calls: list[dict[str, object]] = [] if caller_save_fails: @@ -1671,7 +1681,6 @@ def reject_resolution(**kwargs: object) -> None: pytest.fail("Binding resolution must wait for a persisted caller") monkeypatch.setattr(binding_resolver, "resolve", reject_resolution) - monkeypatch.setattr("core.workflow.node_factory.WorkflowAgentBindingResolver", lambda: binding_resolver) save = SQLAlchemyWorkflowNodeExecutionRepository.save def fail_caller_save( @@ -1757,8 +1766,15 @@ def fail_caller_save( index.seed_source(source.app_id, source.workflow_id, ()) layer.set_node_run_indices(index.indices) node.runtime_state.variable_pool.add(("sys", "workflow_run_id"), "outer-run") + graph = _workflow_tool_event_delivery_outer_graph(node) + graph.node_factory = DifyNodeFactory( + node.init_params, + node.runtime_state, + agent_binding_resolver=binding_resolver, + workflow_runtime=workflow_runtime, + ) engine = Engine( - graph=_workflow_tool_event_delivery_outer_graph(node), + graph=graph, runtime_state=node.runtime_state, command_channel=InMemoryChannel(), workers=1, @@ -1880,7 +1896,9 @@ def test_workflow_tool_dispatcher_enforces_file_ownership( start_at=1, ) template_node, runtime, payload = _workflow_tool_file_access_workflow_tool_node(runtime_state) - monkeypatch.setattr("core.workflow.node_factory.DifyToolNodeRuntime", lambda _, **_kwargs: runtime) + monkeypatch.setattr( + "services.workflow.execution.adapters.node_factory.DifyToolNodeRuntime", lambda _, **_kwargs: runtime + ) tool_data = template_node.node_data.model_dump(mode="python") nodes = [ {"id": "start", "data": {"type": "start", "title": "Start", "variables": list[object]()}}, diff --git a/api/tests/unit_tests/repositories/test_app_definition_query_repository.py b/api/tests/unit_tests/repositories/test_app_definition_query_repository.py index 83017e755f9589..35273847331209 100644 --- a/api/tests/unit_tests/repositories/test_app_definition_query_repository.py +++ b/api/tests/unit_tests/repositories/test_app_definition_query_repository.py @@ -6,7 +6,6 @@ from sqlalchemy.orm import Session, sessionmaker from core.app.app_config.common.parameters_mapping import get_parameters_from_feature_dict -from core.app.apps.agent_app.errors import AgentAppGeneratorError, AgentAppNotPublishedError from core.tools.entities.tool_entities import ApiProviderSchemaType from machinery.context import AppRequestContext from models.account import Account, Tenant, TenantStatus @@ -28,6 +27,7 @@ AppDefinitionQueryRepository, _get_public_agent_parameter_config, ) +from services.app.generation.errors import AgentAppGeneratorError, AgentAppNotPublishedError from services.app_definition_query_service import ( AppDefinitionSummary, AppParameterConfig, @@ -837,7 +837,7 @@ def test_published_agent_app_parameters_use_soul_file_upload( @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_requires_bound_agent(sqlite_session: Session): +def test_published_agent_app_parameters_requires_bound_agent(sqlite_session: Session) -> None: tenant_id = _stable_uuid("tenant:unbound") app_model = _app_model(tenant_id=tenant_id) @@ -846,7 +846,7 @@ def test_published_agent_app_parameters_requires_bound_agent(sqlite_session: Ses @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_requires_existing_active_agent(sqlite_session: Session): +def test_published_agent_app_parameters_requires_existing_active_agent(sqlite_session: Session) -> None: requested_tenant_id = _stable_uuid("tenant:requested") agent_id = _stable_uuid("agent:cross-tenant") app_model = _app_model(tenant_id=requested_tenant_id) @@ -867,7 +867,7 @@ def test_published_agent_app_parameters_requires_existing_active_agent(sqlite_se @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) def test_published_agent_app_parameters_requires_published_agent( active_config_is_published: bool, sqlite_session: Session -): +) -> None: tenant_id = _stable_uuid(f"tenant:published:{active_config_is_published}") agent_id = _stable_uuid(f"agent:published:{active_config_is_published}") app_model = _app_model(tenant_id=tenant_id) @@ -885,7 +885,9 @@ def test_published_agent_app_parameters_requires_published_agent( @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_allows_unpublished_draft_with_active_snapshot(sqlite_session: Session): +def test_published_agent_app_parameters_allows_unpublished_draft_with_active_snapshot( + sqlite_session: Session, +) -> None: tenant_id = _stable_uuid("tenant:unpublished-draft") agent_id = _stable_uuid("agent:unpublished-draft") snapshot_id = _stable_uuid("snapshot:unpublished-draft") @@ -916,7 +918,7 @@ def test_published_agent_app_parameters_allows_unpublished_draft_with_active_sna @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_rejects_seeded_unpublished_snapshot(sqlite_session: Session): +def test_published_agent_app_parameters_rejects_seeded_unpublished_snapshot(sqlite_session: Session) -> None: tenant_id = _stable_uuid("tenant:never-published") agent_id = _stable_uuid("agent:never-published") snapshot_id = _stable_uuid("snapshot:never-published") @@ -943,7 +945,7 @@ def test_published_agent_app_parameters_rejects_seeded_unpublished_snapshot(sqli @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_requires_published_snapshot(sqlite_session: Session): +def test_published_agent_app_parameters_requires_published_snapshot(sqlite_session: Session) -> None: tenant_id = _stable_uuid("tenant:missing-snapshot") agent_id = _stable_uuid("agent:missing-snapshot") app_model = _app_model(tenant_id=tenant_id) @@ -967,7 +969,7 @@ def test_published_agent_app_parameters_requires_published_snapshot(sqlite_sessi @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_allows_missing_legacy_app_model_config(sqlite_session: Session): +def test_published_agent_app_parameters_allows_missing_legacy_app_model_config(sqlite_session: Session) -> None: tenant_id = _stable_uuid("tenant:no-legacy-config") agent_id = _stable_uuid("agent:no-legacy-config") snapshot_id = _stable_uuid("snapshot:no-legacy-config") diff --git a/api/tests/unit_tests/repositories/test_app_definition_query_repository_agent.py b/api/tests/unit_tests/repositories/test_app_definition_query_repository_agent.py deleted file mode 100644 index 85da3b271b0758..00000000000000 --- a/api/tests/unit_tests/repositories/test_app_definition_query_repository_agent.py +++ /dev/null @@ -1,394 +0,0 @@ -import json -from uuid import NAMESPACE_URL, uuid5 - -import pytest -from sqlalchemy.orm import Session, sessionmaker - -from core.app.app_config.common.parameters_mapping import get_parameters_from_feature_dict -from models.agent import ( - Agent, - AgentConfigRevision, - AgentConfigRevisionOperation, - AgentConfigSnapshot, - AgentScope, - AgentSource, - AgentStatus, -) -from models.model import App, AppAnnotationSetting, AppMode, AppModelConfig -from repositories.app_definition_query_repository import ( - AppDefinitionQueryRepository, - _get_public_agent_parameter_config, -) -from services.app.generation.errors import AgentAppGeneratorError, AgentAppNotPublishedError - - -def _stable_uuid(value: str) -> str: - return str(uuid5(NAMESPACE_URL, value)) - - -def _app_model(*, tenant_id: str, app_model_config: AppModelConfig | None = None) -> App: - return App( - id=_stable_uuid(f"app:{tenant_id}"), - tenant_id=tenant_id, - name="Agent App", - mode=AppMode.AGENT, - app_model_config_id=app_model_config.id if app_model_config else None, - ) - - -def _persist_agent( - session: Session, - *, - tenant_id: str, - agent_id: str, - active_config_snapshot_id: str | None, - active_config_is_published: bool, - app_id: str | None = None, -) -> Agent: - agent = Agent( - id=agent_id, - tenant_id=tenant_id, - name="Agent", - scope=AgentScope.ROSTER, - source=AgentSource.AGENT_APP, - status=AgentStatus.ACTIVE, - active_config_snapshot_id=active_config_snapshot_id, - active_config_is_published=active_config_is_published, - app_id=app_id, - ) - session.add(agent) - session.commit() - return agent - - -def _persist_snapshot( - session: Session, - *, - snapshot_id: str, - tenant_id: str, - agent_id: str, - config_snapshot: dict[str, object], - publish_visible: bool = True, -) -> AgentConfigSnapshot: - snapshot = AgentConfigSnapshot( - id=snapshot_id, - tenant_id=tenant_id, - agent_id=agent_id, - version=1, - home_snapshot_id=_stable_uuid(f"home-snapshot:{snapshot_id}"), - config_snapshot=config_snapshot, - ) - session.add(snapshot) - if publish_visible: - _persist_publish_revision( - session, - snapshot_id=snapshot_id, - tenant_id=tenant_id, - agent_id=agent_id, - commit=False, - ) - session.commit() - return snapshot - - -def _persist_publish_revision( - session: Session, - *, - snapshot_id: str, - tenant_id: str, - agent_id: str, - commit: bool = True, -) -> None: - session.add( - AgentConfigRevision( - tenant_id=tenant_id, - agent_id=agent_id, - current_snapshot_id=snapshot_id, - revision=1, - operation=AgentConfigRevisionOperation.PUBLISH_DRAFT, - ) - ) - if commit: - session.commit() - - -def test_get_public_parameter_config_loads_agent_snapshot( - sqlite_session_factory: sessionmaker[Session], -) -> None: - tenant_id = _stable_uuid("tenant:repository") - app_id = _stable_uuid("app:repository") - agent_id = _stable_uuid("agent:repository") - snapshot_id = _stable_uuid("snapshot:repository") - with sqlite_session_factory() as session: - session.add( - App( - id=app_id, - tenant_id=tenant_id, - name="Agent App", - description="", - mode=AppMode.AGENT, - icon_type=None, - icon=None, - icon_background=None, - enable_site=True, - enable_api=True, - max_active_requests=None, - ) - ) - session.commit() - _persist_agent( - session, - tenant_id=tenant_id, - agent_id=agent_id, - active_config_snapshot_id=snapshot_id, - active_config_is_published=True, - app_id=app_id, - ) - _persist_snapshot( - session, - snapshot_id=snapshot_id, - tenant_id=tenant_id, - agent_id=agent_id, - config_snapshot={"app_variables": [{"name": "topic", "type": "string", "required": True}]}, - ) - - result = AppDefinitionQueryRepository(session_factory=sqlite_session_factory).get_published_parameter_config( - app_id, public_runtime=True - ) - - assert result is not None - assert result.user_input_form == [{"text-input": {"label": "topic", "variable": "topic", "required": True}}] - - -@pytest.mark.parametrize( - "sqlite_session", - [(Agent, AgentConfigSnapshot, AgentConfigRevision, AppAnnotationSetting, AppModelConfig)], - indirect=True, -) -@pytest.mark.parametrize("foreign_model_config", [False, True]) -def test_published_agent_app_parameters_use_soul_file_upload( - sqlite_session: Session, foreign_model_config: bool -) -> None: - tenant_id = _stable_uuid("tenant:one") - agent_id = _stable_uuid("agent:one") - snapshot_id = _stable_uuid("snapshot:one") - app_model_config = AppModelConfig( - app_id=_stable_uuid("foreign-app") if foreign_model_config else _stable_uuid(f"app:{tenant_id}"), - opening_statement="Hi from legacy presentation config", - file_upload=json.dumps({"enabled": False, "image": {"enabled": False}}), - ) - sqlite_session.add(app_model_config) - sqlite_session.commit() - app_model = _app_model(tenant_id=tenant_id, app_model_config=app_model_config) - _persist_agent( - sqlite_session, - tenant_id=tenant_id, - agent_id=agent_id, - active_config_snapshot_id=snapshot_id, - active_config_is_published=True, - app_id=app_model.id, - ) - _persist_snapshot( - sqlite_session, - snapshot_id=snapshot_id, - tenant_id=tenant_id, - agent_id=agent_id, - config_snapshot={ - "app_features": { - "file_upload": { - "enabled": True, - "allowed_file_extensions": ["PNG"], - "allowed_file_types": ["image"], - "allowed_file_upload_methods": ["local_file"], - "image": {"enabled": True}, - "number_limits": 2, - } - }, - "app_variables": [{"name": "topic", "type": "string", "required": True}], - }, - ) - - features_dict, user_input_form = _get_public_agent_parameter_config( - app_model, - session=sqlite_session, - ) - parameters = get_parameters_from_feature_dict(features_dict=features_dict, user_input_form=user_input_form) - - assert parameters["opening_statement"] == (None if foreign_model_config else "Hi from legacy presentation config") - assert parameters["file_upload"] == { - "enabled": True, - "allowed_file_extensions": ["PNG"], - "allowed_file_types": ["image"], - "allowed_file_upload_methods": ["local_file"], - "image": {"enabled": True}, - "number_limits": 2, - } - assert parameters["user_input_form"] == [{"text-input": {"label": "topic", "variable": "topic", "required": True}}] - - -@pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_requires_bound_agent(sqlite_session: Session): - tenant_id = _stable_uuid("tenant:unbound") - app_model = _app_model(tenant_id=tenant_id) - - with pytest.raises(AgentAppGeneratorError, match="no bound Agent"): - _get_public_agent_parameter_config(app_model, session=sqlite_session) - - -@pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_requires_existing_active_agent(sqlite_session: Session): - requested_tenant_id = _stable_uuid("tenant:requested") - agent_id = _stable_uuid("agent:cross-tenant") - app_model = _app_model(tenant_id=requested_tenant_id) - _persist_agent( - sqlite_session, - tenant_id=_stable_uuid("tenant:other"), - agent_id=agent_id, - active_config_snapshot_id=None, - active_config_is_published=False, - app_id=app_model.id, - ) - - with pytest.raises(AgentAppGeneratorError, match="no bound Agent"): - _get_public_agent_parameter_config(app_model, session=sqlite_session) - - -@pytest.mark.parametrize("active_config_is_published", [True, False]) -@pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_requires_published_agent( - active_config_is_published: bool, sqlite_session: Session -): - tenant_id = _stable_uuid(f"tenant:published:{active_config_is_published}") - agent_id = _stable_uuid(f"agent:published:{active_config_is_published}") - app_model = _app_model(tenant_id=tenant_id) - _persist_agent( - sqlite_session, - tenant_id=tenant_id, - agent_id=agent_id, - active_config_snapshot_id=None, - active_config_is_published=active_config_is_published, - app_id=app_model.id, - ) - - with pytest.raises(AgentAppNotPublishedError, match="not been published"): - _get_public_agent_parameter_config(app_model, session=sqlite_session) - - -@pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_allows_unpublished_draft_with_active_snapshot(sqlite_session: Session): - tenant_id = _stable_uuid("tenant:unpublished-draft") - agent_id = _stable_uuid("agent:unpublished-draft") - snapshot_id = _stable_uuid("snapshot:unpublished-draft") - app_model = _app_model(tenant_id=tenant_id) - _persist_agent( - sqlite_session, - tenant_id=tenant_id, - agent_id=agent_id, - active_config_snapshot_id=snapshot_id, - active_config_is_published=False, - app_id=app_model.id, - ) - _persist_snapshot( - sqlite_session, - snapshot_id=snapshot_id, - tenant_id=tenant_id, - agent_id=agent_id, - config_snapshot={}, - ) - - features_dict, user_input_form = _get_public_agent_parameter_config( - app_model, - session=sqlite_session, - ) - - assert features_dict["file_upload"]["enabled"] is True - assert user_input_form == [] - - -@pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_rejects_seeded_unpublished_snapshot(sqlite_session: Session): - tenant_id = _stable_uuid("tenant:never-published") - agent_id = _stable_uuid("agent:never-published") - snapshot_id = _stable_uuid("snapshot:never-published") - app_model = _app_model(tenant_id=tenant_id) - _persist_agent( - sqlite_session, - tenant_id=tenant_id, - agent_id=agent_id, - active_config_snapshot_id=snapshot_id, - active_config_is_published=False, - app_id=app_model.id, - ) - _persist_snapshot( - sqlite_session, - snapshot_id=snapshot_id, - tenant_id=tenant_id, - agent_id=agent_id, - config_snapshot={}, - publish_visible=False, - ) - - with pytest.raises(AgentAppNotPublishedError, match="not been published"): - _get_public_agent_parameter_config(app_model, session=sqlite_session) - - -@pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_requires_published_snapshot(sqlite_session: Session): - tenant_id = _stable_uuid("tenant:missing-snapshot") - agent_id = _stable_uuid("agent:missing-snapshot") - app_model = _app_model(tenant_id=tenant_id) - _persist_agent( - sqlite_session, - tenant_id=tenant_id, - agent_id=agent_id, - active_config_snapshot_id=_stable_uuid("snapshot:missing"), - active_config_is_published=True, - app_id=app_model.id, - ) - _persist_publish_revision( - sqlite_session, - snapshot_id=_stable_uuid("snapshot:missing"), - tenant_id=tenant_id, - agent_id=agent_id, - ) - - with pytest.raises(AgentAppGeneratorError, match="published version not found"): - _get_public_agent_parameter_config(app_model, session=sqlite_session) - - -@pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_allows_missing_legacy_app_model_config(sqlite_session: Session): - tenant_id = _stable_uuid("tenant:no-legacy-config") - agent_id = _stable_uuid("agent:no-legacy-config") - snapshot_id = _stable_uuid("snapshot:no-legacy-config") - app_model = _app_model(tenant_id=tenant_id) - _persist_agent( - sqlite_session, - tenant_id=tenant_id, - agent_id=agent_id, - active_config_snapshot_id=snapshot_id, - active_config_is_published=True, - app_id=app_model.id, - ) - _persist_snapshot( - sqlite_session, - snapshot_id=snapshot_id, - tenant_id=tenant_id, - agent_id=agent_id, - config_snapshot={}, - ) - - features_dict, user_input_form = _get_public_agent_parameter_config( - app_model, - session=sqlite_session, - ) - - assert features_dict["file_upload"] == { - "allowed_file_extensions": ["JPG", "JPEG", "PNG", "GIF", "WEBP", "SVG"], - "allowed_file_types": ["document", "image", "audio", "video"], - "allowed_file_upload_methods": ["local_file", "remote_url"], - "enabled": True, - "image": {"enabled": True}, - "number_limits": 3, - } - assert user_input_form == [] diff --git a/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py b/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py index 4d5f89af69c786..51c4a381b41f52 100644 --- a/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py +++ b/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py @@ -297,7 +297,8 @@ def organize_prompt(**kwargs: object) -> tuple[list[object], list[object]]: # Stop after retrieval and prompt preparation; the assertions cover all retrieval I/O. monkeypatch.setattr(runner, "check_hosting_moderation", lambda **_kwargs: True) queue = Queue() - app = App(id=app_id, tenant_id=tenant_id, mode=mode) + app_mode = mode if isinstance(mode, AppMode) else AppMode.AGENT_CHAT + app = App(id=app_id, tenant_id=tenant_id, mode=app_mode) message = Message(id=str(uuid4())) try: with Flask(__name__).app_context(): diff --git a/api/tests/unit_tests/services/test_agent_service.py b/api/tests/unit_tests/services/test_agent_service.py index 80d4c13ac05182..5bf89eff51f0a0 100644 --- a/api/tests/unit_tests/services/test_agent_service.py +++ b/api/tests/unit_tests/services/test_agent_service.py @@ -2,127 +2,107 @@ from __future__ import annotations -from collections.abc import Iterator from dataclasses import dataclass, field from datetime import UTC, datetime from typing import cast import pytest -from sqlalchemy.orm import Session -from models import Account, App -from services import agent_service +from core.tools.entities.tool_entities import ToolProviderType +from machinery.context import RequestContext +from models.model import AppModelConfigDict +from services.agent.log_contracts import AgentLogSnapshot, AgentLogThought +from services.agent.log_service import AgentLogService _NOW = datetime(2026, 1, 1, tzinfo=UTC) +_CONTEXT = RequestContext("request", None, "account-id", "tenant-id") @dataclass(frozen=True) -class _AgentThought: - tools: list[str] - tool_meta: dict[str, object] - tool_labels: dict[str, str] = field(default_factory=dict) - tool_inputs_dict: dict[str, object] = field(default_factory=dict) - tool_outputs_dict: dict[str, object] = field(default_factory=dict) - tokens: int = 1 - tool_input: str = "{}" - observation: str = "{}" - thought: str = "historical thought" - created_at: datetime = _NOW - files: list[str] = field(default_factory=list) - - -@dataclass -class _Message: - agent_thoughts: list[_AgentThought] - created_at: datetime = _NOW - provider_response_latency: float = 0.1 - answer_tokens: int = 1 - message_tokens: int = 1 - - def agent_thoughts_with_session(self, session: object) -> list[_AgentThought]: - del session - return self.agent_thoughts - - def message_files_with_session(self, session: object) -> list[object]: - del session +class _Records: + snapshot: AgentLogSnapshot + + def agent_log( + self, *, tenant_id: str, account_id: str, app_id: str, conversation_id: str, message_id: str + ) -> AgentLogSnapshot: + assert (tenant_id, account_id, app_id, conversation_id, message_id) == ( + "tenant-id", + "account-id", + "app-id", + "conversation-id", + "message-id", + ) + return self.snapshot + + +class _Files: + def resolve(self, *, tenant_id: str, files: list[dict[str, object]]) -> list[dict[str, object]]: + assert tenant_id == "tenant-id" + assert files == [] return [] -@dataclass(frozen=True) -class _Conversation: - from_end_user_id: str | None = None - from_account_id: str = "account-id" - - -class _AppModelConfig: - agent_mode_dict = {"strategy": "react"} - - def to_dict(self, *, annotation_reply: object = None) -> dict[str, object]: - del annotation_reply - return {} - - -@dataclass(frozen=True) -class _App: - id: str = "app-id" - tenant_id: str = "tenant-id" - - def app_model_config_with_session(self, session: object) -> _AppModelConfig: - del session - return _AppModelConfig() - - -class _ScriptedSession: - """Answers each `scalar` read in the order the log query issues them.""" - - def __init__(self, *results: object) -> None: - self._results: Iterator[object] = iter(results) - - def scalar(self, statement: object) -> object: - del statement - return next(self._results) - - -@dataclass(frozen=True) -class _AgentTool: - tool_name: str - provider_type: str - provider_id: str - - -@dataclass(frozen=True) -class _AgentConfig: - tools: list[_AgentTool] - - @dataclass class _ToolIcons: calls: list[dict[str, str]] = field(default_factory=list) - def get_tool_icon(self, *, tenant_id: str, provider_type: str, provider_id: str) -> str: - self.calls.append({"tenant_id": tenant_id, "provider_type": provider_type, "provider_id": provider_id}) + def __call__(self, *, tenant_id: str, provider_type: ToolProviderType, provider_id: str) -> str: + self.calls.append({"tenant_id": tenant_id, "provider_type": provider_type.value, "provider_id": provider_id}) return "icon-url" +def _snapshot(*, tool_still_configured: bool) -> AgentLogSnapshot: + tools: list[dict[str, object]] = ( + [{"enabled": True, "provider_type": "api", "provider_id": "provider-id", "tool_name": "old_tool"}] + if tool_still_configured + else [] + ) + model_config = cast( + AppModelConfigDict, + { + "model": {"provider": "openai", "name": "gpt", "mode": "chat"}, + "agent_mode": {"enabled": True, "strategy": "react", "tools": tools}, + }, + ) + thought = AgentLogThought( + tokens=1, + tools=["old_tool"], + labels={}, + metadata={"old_tool": {"tool_config": {}}}, + inputs={}, + outputs={}, + raw_input="{}", + raw_output="{}", + thought="historical thought", + created_at=_NOW, + files=[], + ) + return AgentLogSnapshot( + executor="Executor", + timezone="UTC", + created_at=_NOW, + elapsed_time=0.1, + total_tokens=2, + model_config=model_config, + thoughts=[thought], + files=[], + ) + + @pytest.mark.parametrize("tool_still_configured", [False, True]) def test_legacy_tool_log_without_provider_metadata_uses_configured_tool_or_blank_icon( - monkeypatch: pytest.MonkeyPatch, tool_still_configured: bool + tool_still_configured: bool, ) -> None: - account = Account(name="Executor", email="executor@example.test") - account.timezone = "UTC" - monkeypatch.setattr(agent_service, "current_user", account) - - thought = _AgentThought(tools=["old_tool"], tool_meta={"old_tool": {"tool_config": {}}}) - session = _ScriptedSession(_Conversation(), _Message(agent_thoughts=[thought]), "Executor") - tool = _AgentTool(tool_name="old_tool", provider_type="api", provider_id="provider-id") - agent_config = _AgentConfig(tools=[tool] if tool_still_configured else []) - monkeypatch.setattr(agent_service.AgentConfigManager, "convert", lambda _: agent_config) - monkeypatch.setattr(agent_service, "load_annotation_reply_config", lambda *_: None) icons = _ToolIcons() - monkeypatch.setattr(agent_service.ToolManager, "get_tool_icon", icons.get_tool_icon) + service = AgentLogService( + records=_Records(_snapshot(tool_still_configured=tool_still_configured)), icons=icons, files=_Files() + ) - result = agent_service.AgentService.get_agent_logs( - cast(App, _App()), "conversation-id", "message-id", cast(Session, session) + result = service.get( + _CONTEXT, + app_id="app-id", + conversation_id="conversation-id", + message_id="message-id", ) assert result["iterations"][0]["tool_calls"][0]["tool_icon"] == ("icon-url" if tool_still_configured else "") diff --git a/api/tests/unit_tests/services/test_app_generate_service_streaming_integration.py b/api/tests/unit_tests/services/test_app_generate_service_streaming_integration.py deleted file mode 100644 index f18544362472d6..00000000000000 --- a/api/tests/unit_tests/services/test_app_generate_service_streaming_integration.py +++ /dev/null @@ -1,221 +0,0 @@ -import json -import uuid -from collections import defaultdict, deque - -import pytest - -from models.model import AppMode -from services.app_generate_service import AppGenerateService -from services.workflow.execution.adapters.response_stream import WorkflowEventStream -from tests.helpers.unit.config_override import apply_config_overrides - - -# ----------------------------- -# Fakes for Redis Pub/Sub flow -# ----------------------------- -class _FakePubSub: - def __init__(self, store: dict[str, deque[bytes]]): - self._store = store - self._subs: set[str] = set() - self._closed = False - - def subscribe(self, topic: str) -> None: - self._subs.add(topic) - - def unsubscribe(self, topic: str) -> None: - self._subs.discard(topic) - - def close(self) -> None: - self._closed = True - - def get_message(self, ignore_subscribe_messages: bool = True, timeout: int | float | None = 1): - _ = ignore_subscribe_messages, timeout - # simulate a non-blocking poll; return first available - if self._closed: - return None - for t in list(self._subs): - q = self._store.get(t) - if q and len(q) > 0: - payload = q.popleft() - return {"type": "message", "channel": t, "data": payload} - # no message - return None - - -class _FakeRedisClient: - def __init__(self, store: dict[str, deque[bytes]]): - self._store = store - - def pubsub(self): - return _FakePubSub(self._store) - - def publish(self, topic: str, payload: bytes) -> None: - self._store.setdefault(topic, deque()).append(payload) - - -# ------------------------------------ -# Fakes for Redis Streams (XADD/XREAD) -# ------------------------------------ -class _FakeStreams: - def __init__(self) -> None: - # key -> list[(id, {field: value})] - self._data: dict[str, list[tuple[str, dict]]] = defaultdict(list) - self._seq: dict[str, int] = defaultdict(int) - # Mirrors real Redis: "$" resolves once, to the tail of the stream at the moment - # the first xread naming it is issued -- entries added earlier are never seen. - self._dollar_snapshots: dict[str, int] = {} - - def xadd(self, key: str, fields: dict[str, object], *, maxlen: int | None = None) -> str: - # maxlen is accepted for API compatibility with redis-py; ignored in this test double - _ = maxlen - self._seq[key] += 1 - eid = f"{self._seq[key]}-0" - self._data[key].append((eid, fields)) - return eid - - def expire(self, key: str, seconds: int) -> None: - # no-op for tests - _ = key, seconds - - def xrevrange(self, key: str, count: int | None = None): - entries = list(reversed(self._data.get(key, []))) - if count is not None: - entries = entries[:count] - return entries - - def xread(self, streams: dict[str, object], block: int | None = None, count: int | None = None): - _ = block - assert len(streams) == 1 - key, last_id = next(iter(streams.items())) - entries = self._data.get(key, []) - start = 0 - if last_id == "$": - start = self._dollar_snapshots.setdefault(key, len(entries)) - elif last_id != "0-0": - for i, (eid, _f) in enumerate(entries): - if eid == last_id: - start = i + 1 - break - if start >= len(entries): - return [] - end = len(entries) if count is None else min(len(entries), start + count) - return [(key, entries[start:end])] - - -@pytest.fixture -def _patch_get_channel_streams(monkeypatch: pytest.MonkeyPatch): - from libs.broadcast_channel.redis.streams_channel import StreamsBroadcastChannel - - fake = _FakeStreams() - chan = StreamsBroadcastChannel(fake, retention_seconds=60) - - def _get_channel(): - return chan - - # Patch the source and the imported aliases used by WorkflowEventStream / MessageBasedAppGenerator - monkeypatch.setattr("extensions.ext_redis.get_pubsub_broadcast_channel", lambda: chan) - monkeypatch.setattr( - "services.workflow.execution.adapters.response_stream.get_pubsub_broadcast_channel", lambda: chan - ) - monkeypatch.setattr( - "services.workflow.execution.adapters.response_stream.get_pubsub_broadcast_channel", lambda: chan - ) - apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE="streams") - - -@pytest.fixture -def _patch_get_channel_pubsub(monkeypatch: pytest.MonkeyPatch): - from libs.broadcast_channel.redis.pubsub_channel import BroadcastChannel as RedisBroadcastChannel - - store: dict[str, deque[bytes]] = defaultdict(deque) - client = _FakeRedisClient(store) - chan = RedisBroadcastChannel(client) - - def _get_channel(): - return chan - - # Patch both the source and the imported alias used by WorkflowEventStream - monkeypatch.setattr("extensions.ext_redis.get_pubsub_broadcast_channel", lambda: chan) - monkeypatch.setattr( - "services.workflow.execution.adapters.response_stream.get_pubsub_broadcast_channel", lambda: chan - ) - apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE="pubsub") - - -def _publish_events(app_mode: AppMode, run_id: str, events: list[dict]): - # Publish events to the same topic used by WorkflowEventStream - topic = WorkflowEventStream.get_response_topic(app_mode, run_id) - for ev in events: - topic.publish(json.dumps(ev).encode()) - - -@pytest.mark.usefixtures("_patch_get_channel_streams") -def test_streams_full_flow_prepublish_and_replay(): - """Regression test for dify#40948: `workflow_started` dropped for concurrent streaming - runs on the streams transport. - - `start_task` is invoked via `on_subscribe` after entering the subscription and publishes - both events synchronously, before the listener thread necessarily gets a chance to run its - first `xread`. `_StreamsSubscription` fixes its read boundary synchronously before spawning - that thread, so neither event is dropped. - """ - app_mode = AppMode.WORKFLOW - run_id = str(uuid.uuid4()) - - events = [{"event": "workflow_started"}, {"event": "workflow_finished"}] - - def start_task(): - _publish_events(app_mode, run_id, events) - - # MessageBasedAppGenerator is what AppGenerateService actually dispatches WORKFLOW / - # ADVANCED_CHAT streaming runs through in production. - on_subscribe = AppGenerateService._build_streaming_task_on_subscribe(start_task) - - gen = WorkflowEventStream.retrieve_events(app_mode, run_id, idle_timeout=2.0, on_subscribe=on_subscribe) - - received = [] - for msg in gen: - if isinstance(msg, str): - # skip ping events - continue - received.append(msg) - if msg.get("event") == "workflow_finished": - break - - assert [m.get("event") for m in received] == ["workflow_started", "workflow_finished"] - - -@pytest.mark.usefixtures("_patch_get_channel_pubsub") -def test_pubsub_full_flow_start_on_subscribe_gated(monkeypatch: pytest.MonkeyPatch): - # Speed up any potential timer if it accidentally triggers - monkeypatch.setattr("services.app_generate_service.SSE_TASK_START_FALLBACK_MS", 50) - - app_mode = AppMode.WORKFLOW - run_id = str(uuid.uuid4()) - - published_order: list[str] = [] - - def start_task(): - # When called (on subscribe), publish both events - events = [{"event": "workflow_started"}, {"event": "workflow_finished"}] - _publish_events(app_mode, run_id, events) - published_order.extend([e["event"] for e in events]) - - on_subscribe = AppGenerateService._build_streaming_task_on_subscribe(start_task) - - # Producer not started yet; only when subscribe happens - assert published_order == [] - - gen = WorkflowEventStream.retrieve_events(app_mode, run_id, idle_timeout=2.0, on_subscribe=on_subscribe) - - received = [] - for msg in gen: - if isinstance(msg, str): - continue - received.append(msg) - if msg.get("event") == "workflow_finished": - break - - # Verify publish happened and consumer received in order - assert published_order == ["workflow_started", "workflow_finished"] - assert [m.get("event") for m in received] == ["workflow_started", "workflow_finished"] diff --git a/api/tests/unit_tests/services/test_snippet_dsl_service.py b/api/tests/unit_tests/services/test_snippet_dsl_service.py index 1ce89a105c7239..0519ca3b475173 100644 --- a/api/tests/unit_tests/services/test_snippet_dsl_service.py +++ b/api/tests/unit_tests/services/test_snippet_dsl_service.py @@ -10,7 +10,7 @@ from sqlalchemy import event, select from sqlalchemy.orm import Session -from extensions.application_services.snippets import build_snippet_dsl_service +from extensions.application_services.snippets import build_snippet_dsl_service, build_snippet_service from graphon.nodes import BuiltinNodeTypes from models import Account from models.agent import WorkflowAgentBindingType, WorkflowAgentNodeBinding @@ -25,7 +25,6 @@ SnippetPendingData, _check_version_compatibility, ) -from services.snippet_service import SnippetService from tests.helpers.unit.model_factories import make_account, make_tenant, make_workflow SQLITE_MODELS = (CustomizedSnippet,) @@ -554,7 +553,7 @@ def forbidden_yaml_load(*_args: object, **_kwargs: object) -> NoReturn: assert snippet.created_by == account.id assert snippet.name == "Override name" assert snippet.description == "Override description" - assert SnippetService(session=sqlite_session).get_draft_workflow(snippet) is not None + assert build_snippet_service(session=sqlite_session).get_draft_workflow(snippet) is not None assert commands[-1] == ("DEL", redis_key) assert redis_key not in values @@ -724,7 +723,7 @@ def retire_unowned( persisted = sqlite_session.get(CustomizedSnippet, snippet.id) assert persisted is not None assert persisted.name == "New" - workflow = SnippetService(session=sqlite_session).get_draft_workflow(snippet) + workflow = build_snippet_service(session=sqlite_session).get_draft_workflow(snippet) assert workflow is draft_workflow assert workflow.graph_dict == {"nodes": [], "edges": []} assert sqlite_session.get(WorkflowAgentNodeBinding, "retired-binding") is None @@ -756,7 +755,7 @@ def test_create_or_update_snippet_creates_new_snippet_and_flushes(service: Snipp assert result.type == "group" assert not sqlite_session.in_transaction() assert sqlite_session.get(CustomizedSnippet, result.id) is result - workflow = SnippetService(session=sqlite_session).get_draft_workflow(result) + workflow = build_snippet_service(session=sqlite_session).get_draft_workflow(result) assert workflow is not None assert workflow.graph_dict == {"nodes": [], "edges": []} assert workflow.created_by == "account-1" diff --git a/api/tests/unit_tests/services/test_snippet_generate_service.py b/api/tests/unit_tests/services/test_snippet_generate_service.py index c802979defe43f..008048c04a9404 100644 --- a/api/tests/unit_tests/services/test_snippet_generate_service.py +++ b/api/tests/unit_tests/services/test_snippet_generate_service.py @@ -93,8 +93,8 @@ def test_debug_generators_use_injected_variable_dependencies( calls: list[str] = [] def execute(generator: WorkflowAppGenerator, **_kwargs: object) -> dict[str, str]: - assert generator._draft_variable_loader == workflow_variables.workflow_loader - assert generator._draft_variable_saver == workflow_variables.saver_factory + assert generator.draft_variable_loader == workflow_variables.workflow_loader + assert generator.draft_variable_saver == workflow_variables.saver_factory saver = generator._get_draft_var_saver_factory(InvokeFrom.DEBUGGER, user, tenant_id=workflow.tenant_id) saver(workflow.app_id, "node", BuiltinNodeTypes.CODE, "execution").save( process_data=None, outputs={"text": "injected"} diff --git a/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.py b/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.py index 6eb7b78b02710f..b75dd60eefa7dc 100644 --- a/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.py +++ b/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.py @@ -276,7 +276,7 @@ def test_direct_cleanup_does_not_resolve_a_console_login( _service(sqlite_session_factory).cleanup("target") sqlite_session.expire_all() assert sqlite_session.get(App, "app") is None - assert retirement.retire_unowned.call_args.kwargs["account_id"] is None + retirement.return_value.retire_unowned.assert_called_once_with(tenant_id="target", agent_ids=set(), account_id=None) user.__bool__.assert_not_called() From 3fdfacc2965b675ac9f958cce4dec54c2961c8c4 Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Mon, 26 Oct 2026 22:29:43 +0800 Subject: [PATCH 13/20] Type the publication flush spy without weakening session assertions --- .../unit_tests/services/agent/test_knowledge_spaces.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api/tests/unit_tests/services/agent/test_knowledge_spaces.py b/api/tests/unit_tests/services/agent/test_knowledge_spaces.py index e538f74e101486..e2d65fc78f7de2 100644 --- a/api/tests/unit_tests/services/agent/test_knowledge_spaces.py +++ b/api/tests/unit_tests/services/agent/test_knowledge_spaces.py @@ -48,12 +48,15 @@ def _patch_runtime(monkeypatch: pytest.MonkeyPatch, bindings: _RecordingAppBindi def test_sync_workflow_bindings_unions_direct_and_frozen_agent_spaces( monkeypatch: pytest.MonkeyPatch, ) -> None: - session = Mock(spec=Session) + flush = Mock() + session = Mock(spec=Session, flush=flush) bindings = _RecordingAppBindings() session_maker = _patch_runtime(monkeypatch, bindings) + owning_session = session def collect_agent_spaces(*, session: Session, workflow: Workflow) -> tuple[str, ...]: - assert session.flush.call_count == 1 + assert session is owning_session + assert flush.call_count == 1 assert workflow.id == "published-workflow" return ("space-b", "space-c") From 506ca7813b36fe87355f373f6c017c4eab4d58e3 Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Mon, 26 Oct 2026 23:18:19 +0800 Subject: [PATCH 14/20] Preserve Agent context and tracing semantics when service execution activates --- api/repositories/app/generation_repository.py | 4 +- api/services/agent/chat/base_runner.py | 12 ++++ .../app/generation/adapters/agent_app.py | 11 ++++ .../generation/adapters/message_pipeline.py | 30 ++++++++-- .../core/agent/test_base_agent_runner.py | 31 +++++++++++ .../app/apps/agent_app/test_app_generator.py | 46 +++++++++++++++- .../services/tracing/test_recovery.py | 55 +++++++++++++------ 7 files changed, 164 insertions(+), 25 deletions(-) diff --git a/api/repositories/app/generation_repository.py b/api/repositories/app/generation_repository.py index cf0659260b3c65..fcdff3af2dc995 100644 --- a/api/repositories/app/generation_repository.py +++ b/api/repositories/app/generation_repository.py @@ -392,7 +392,9 @@ def save_message(self, identity: MessageIdentity, update: MessageUpdate) -> Mess for fields in update.files ) session.flush() - session.refresh(message) + # Every field consumed by post-commit message subscribers is assigned + # above. A full refresh would expire unloaded relationships before the + # row is detached, leaving subscribers with an unusable detached model. session.expunge(message) return message diff --git a/api/services/agent/chat/base_runner.py b/api/services/agent/chat/base_runner.py index a21a5e9546efbc..e6451ac4d1dcaa 100644 --- a/api/services/agent/chat/base_runner.py +++ b/api/services/agent/chat/base_runner.py @@ -12,7 +12,10 @@ from core.app.apps.draft_variable_saver import DraftVariableSaverFactory from core.app.entities.app_invoke_entities import ( AgentChatAppGenerateEntity, + DifyRunContext, + InvokeFrom, ModelConfigWithCredentialsEntity, + UserFrom, ) from core.app.file_access import DatabaseFileAccessController from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler @@ -148,6 +151,15 @@ def _convert_tool_to_prompt_message_tool(self, tool: AgentToolEntity) -> tuple[P workflow_queries=self._workflow_runtime.tools, workflow_runtime=self._workflow_runtime, ) + invoke_from = self.application_generate_entity.invoke_from + if isinstance(invoke_from, InvokeFrom): + tool_entity.runtime.dify_run_context = DifyRunContext( + tenant_id=self.tenant_id, + app_id=self.app_config.app_id, + user_id=self.user_id, + user_from=(UserFrom.ACCOUNT if invoke_from.runs_as_account() else UserFrom.END_USER), + invoke_from=invoke_from, + ) assert tool_entity.entity.description message_tool = PromptMessageTool( name=tool.tool_name, diff --git a/api/services/app/generation/adapters/agent_app.py b/api/services/app/generation/adapters/agent_app.py index 97c66a73fc186d..8799085e098c9a 100644 --- a/api/services/app/generation/adapters/agent_app.py +++ b/api/services/app/generation/adapters/agent_app.py @@ -316,6 +316,17 @@ def _generate_worker( except GenerateTaskStoppedError: pass except Exception as e: + from core.ops.base_trace_instance import application_tracing_hooks + + tracing_hooks = application_tracing_hooks() + if tracing_hooks is not None: + tracing_hooks.capture_agent_app_terminal( + tenant_id=application_generate_entity.app_config.tenant_id, + app_id=application_generate_entity.app_config.app_id, + message_id=message_id, + outcome="failed", + output=None, + ) logger.exception("Unknown Error in Agent App generate worker") queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER) diff --git a/api/services/app/generation/adapters/message_pipeline.py b/api/services/app/generation/adapters/message_pipeline.py index 7d4fdbb4eef8ac..bb29023b12b722 100644 --- a/api/services/app/generation/adapters/message_pipeline.py +++ b/api/services/app/generation/adapters/message_pipeline.py @@ -324,9 +324,19 @@ def _process_stream_response( # A cancelled Agent can report authoritative usage after the SSE # stream stops. The repository preserves it atomically with the answer. - self._save_message( + saved_message = self._save_message( trace_manager=trace_manager, preserve_existing_usage=isinstance(event, QueueStopEvent) ) + if ( + isinstance(event, QueueStopEvent) + and self._application_generate_entity.app_config.app_mode == AppMode.AGENT + ): + # Provider/billing subscribers may perform external I/O. + # The repository has released the stop-save row lock before this notification. + message_was_created.send( + saved_message, + application_generate_entity=self._application_generate_entity, + ) message_end_resp = self._message_end_to_stream_response() yield message_end_resp case QueueRetrieverResourcesEvent(): @@ -423,11 +433,19 @@ def _save_message( *, trace_manager: TraceQueueManager | None = None, preserve_existing_usage: bool = False, - ): + ) -> Message: llm_result = self._task_state.llm_result usage = llm_result.usage usage.latency = time.perf_counter() - self.start_at self._task_state.metadata.usage = usage + agent_stop_requested = ( + preserve_existing_usage and self._application_generate_entity.app_config.app_mode == AppMode.AGENT + ) + metadata = self._task_state.metadata.model_dump(mode="json") + if agent_stop_requested: + # Closing the consumer records a request, not a runtime outcome. + # The Agent worker owns the terminal after cancellation/cleanup. + metadata["agent_stop_requested"] = True saved = self._records.save_message( self._identity, MessageUpdate( @@ -437,7 +455,7 @@ def _save_message( else "" ), latency=usage.latency, - metadata=self._task_state.metadata.model_dump(mode="json"), + metadata=metadata, files=[], usage=usage, prompt=PromptMessageUtil.prompt_messages_to_prompt_for_saving( @@ -449,7 +467,7 @@ def _save_message( metadata = json.loads(saved.message_metadata) if saved.message_metadata else {} if "usage" in metadata: self._task_state.llm_result.usage = LLMUsage.from_metadata(metadata["usage"]) - if trace_manager: + if trace_manager and not agent_stop_requested: trace_manager.add_trace_task( TraceTask( TraceTaskName.MESSAGE_TRACE, @@ -459,7 +477,9 @@ def _save_message( ) ) # Consumers observe a committed, detached message. - message_was_created.send(saved, application_generate_entity=self._application_generate_entity) + if not agent_stop_requested: + message_was_created.send(saved, application_generate_entity=self._application_generate_entity) + return saved def _handle_stop(self, event: QueueStopEvent): """ diff --git a/api/tests/unit_tests/core/agent/test_base_agent_runner.py b/api/tests/unit_tests/core/agent/test_base_agent_runner.py index dc8fab1fdec28b..121b6aaf0201e0 100644 --- a/api/tests/unit_tests/core/agent/test_base_agent_runner.py +++ b/api/tests/unit_tests/core/agent/test_base_agent_runner.py @@ -22,6 +22,7 @@ from core.app.entities.app_invoke_entities import ( AgentChatAppGenerateEntity, InvokeFrom, + UserFrom, ) from core.model_manager import ModelInstance from core.tools.__base.tool import Tool @@ -722,6 +723,36 @@ def test_convert_tool_to_prompt_message_tool(runner: BaseAgentRunner, mocker: Mo assert prompt_tool.parameters == schema +@pytest.mark.parametrize( + ("invoke_from", "expected_user_from"), + [ + (InvokeFrom.DEBUGGER, UserFrom.ACCOUNT), + (InvokeFrom.SERVICE_API, UserFrom.END_USER), + ], +) +def test_convert_tool_to_prompt_message_tool_attaches_run_context( + runner: BaseAgentRunner, + mocker: MockerFixture, + invoke_from: InvokeFrom, + expected_user_from: UserFrom, +) -> None: + runner.application_generate_entity.invoke_from = invoke_from + tool_entity = _tool("tool1", []) + mocker.patch.object(module.ToolManager, "get_agent_tool_runtime", return_value=tool_entity) + + _, converted = runner._convert_tool_to_prompt_message_tool(_agent_tool("tool1")) + + run_context = converted.runtime.dify_run_context + assert run_context is not None + assert ( + run_context.tenant_id, + run_context.app_id, + run_context.user_id, + run_context.user_from, + run_context.invoke_from, + ) == ("tenant", "app1", "user", expected_user_from, invoke_from) + + def test_convert_dataset_retriever_tool( runner: BaseAgentRunner, mocker: MockerFixture, sqlite_session_factory: sessionmaker[Session] ) -> None: diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py index 9807bb3806bff4..e697de88f13ccd 100644 --- a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py +++ b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py @@ -282,6 +282,32 @@ def test_generate_loads_existing_conversation(self, generator: AgentAppGenerator assert generator._agent_configs.resolve.call_args.kwargs["conversation_id"] == "conv" assert "session" not in generator._init_generate_records.call_args.kwargs + def test_record_initialization_failure_does_not_capture_a_worker_terminal( + self, generator: AgentAppGenerator, mocker: MockerFixture + ) -> None: + generator._prepare_user_inputs = mocker.MagicMock(return_value={}) + generator._init_generate_records = mocker.MagicMock(side_effect=ValueError("record initialization failed")) + mocker.patch( + f"{MODULE}.AgentAppConfigManager.get_app_config", + return_value=mocker.MagicMock(variables=[], tenant_id="tenant", app_id="app1"), + ) + mocker.patch(f"{MODULE}.ModelConfigConverter.convert", return_value=mocker.MagicMock(model="gpt-4o-mini")) + mocker.patch(f"{MODULE}.TraceQueueManager", return_value=mocker.MagicMock()) + mocker.patch(f"{MODULE}.AgentAppGenerateEntity", return_value=mocker.MagicMock()) + thread_constructor = mocker.patch(f"{MODULE}.threading.Thread") + tracing_hooks = mocker.patch("core.ops.base_trace_instance.application_tracing_hooks") + + with pytest.raises(ValueError, match="record initialization failed"): + generator.generate( + app_model=_app(), + user=_account(), + args={"query": "query", "inputs": {}}, + invoke_from=InvokeFrom.DEBUGGER, + ) + + thread_constructor.assert_not_called() + tracing_hooks.assert_not_called() + def test_generate_does_not_include_trace_session_id_in_extras( self, generator: AgentAppGenerator, mocker: MockerFixture ): @@ -467,8 +493,22 @@ def test_generate_task_stopped_is_swallowed(self, generator, mocker: MockerFixtu self._call(generator, mocker, queue_manager) queue_manager.publish_error.assert_not_called() - def test_unexpected_error_is_published(self, generator: AgentAppGenerator, mocker: MockerFixture): - self._wire(generator, mocker, run_side_effect=ValueError("boom")) + @pytest.mark.parametrize("failure_stage", ["record_load", "runner"]) + def test_unexpected_error_is_published_and_captures_failed_terminal( + self, generator: AgentAppGenerator, mocker: MockerFixture, failure_stage: str + ) -> None: + self._wire(generator, mocker, run_side_effect=ValueError("boom") if failure_stage == "runner" else None) + if failure_stage == "record_load": + generator._records.load.side_effect = ValueError("boom") + hooks = mocker.MagicMock() + mocker.patch("core.ops.base_trace_instance.application_tracing_hooks", return_value=hooks) queue_manager = mocker.MagicMock() self._call(generator, mocker, queue_manager) - assert queue_manager.publish_error.called + hooks.capture_agent_app_terminal.assert_called_once_with( + tenant_id="tenant", + app_id="app1", + message_id="msg", + outcome="failed", + output=None, + ) + queue_manager.publish_error.assert_called_once() diff --git a/api/tests/unit_tests/services/tracing/test_recovery.py b/api/tests/unit_tests/services/tracing/test_recovery.py index a00e7fe7eccf40..92eac91698bcda 100644 --- a/api/tests/unit_tests/services/tracing/test_recovery.py +++ b/api/tests/unit_tests/services/tracing/test_recovery.py @@ -125,9 +125,9 @@ def save_agent_stop( ) from core.app.entities.app_invoke_entities import InvokeFrom from core.app.entities.queue_entities import QueueStopEvent - from core.app.task_pipeline.easy_ui_based_generate_task_pipeline import EasyUIBasedGenerateTaskPipeline from core.ops import ops_trace_manager from core.ops.base_trace_instance import bind_application_tracing_hooks + from services.app.generation.adapters.message_pipeline import EasyUIBasedGenerateTaskPipeline from services.tracing import runtime_bridge from tests.helpers.unit.config_override import apply_config_overrides from tests.helpers.unit.core.model_fixtures import make_model_config @@ -145,9 +145,9 @@ def send(message: Message, *, application_generate_entity: ChatAppGenerateEntity if notify is not None: notify(message, application_generate_entity) - monkeypatch.setattr("core.app.task_pipeline.easy_ui_based_generate_task_pipeline.message_was_created.send", send) + monkeypatch.setattr("services.app.generation.adapters.message_pipeline.message_was_created.send", send) monkeypatch.setattr( - "core.app.task_pipeline.easy_ui_based_generate_task_pipeline.naive_utc_now", + "repositories.app.generation_repository.naive_utc_now", lambda: case.now - timedelta(seconds=10), ) with case.collection.sessions.begin() as session: @@ -179,7 +179,14 @@ def send(message: Message, *, application_generate_entity: ChatAppGenerateEntity invoke_from=InvokeFrom.DEBUGGER, ) queue_manager = _QueueManager() - pipeline = EasyUIBasedGenerateTaskPipeline(entity, queue_manager, conversation, message, stream=True) + pipeline = EasyUIBasedGenerateTaskPipeline( + entity, + queue_manager, + conversation, + message, + stream=True, + records=AppGenerationRepository(case.collection.sessions), + ) pipeline._task_state.llm_result.message.content = answer application = Flask(__name__) bind_application_tracing_hooks(application, runtime_bridge.RuntimeApplicationTracingHooks()) @@ -192,11 +199,9 @@ def send(message: Message, *, application_generate_entity: ChatAppGenerateEntity monkeypatch.setattr(pipeline, "_handle_stop", lambda _event: None) monkeypatch.setattr(pipeline, "handle_output_moderation_when_task_finished", lambda _answer: None) list(pipeline._process_stream_response(publisher=None, trace_manager=trace_manager)) - elif consumer_session is None: - with case.collection.sessions.begin() as session: - pipeline._save_message(session=session, trace_manager=trace_manager, preserve_existing_usage=True) else: - pipeline._save_message(session=consumer_session, trace_manager=trace_manager, preserve_existing_usage=True) + pipeline._save_message(trace_manager=trace_manager, preserve_existing_usage=True) + if consumer_session is not None: consumer_session.commit() return trace_manager.collect_tasks() @@ -370,10 +375,13 @@ def test_classic_stopped_consumer_keeps_its_existing_trace_notification( def test_worker_usage_flush_preserves_stop_intent_from_a_newer_consumer_save( - case: RecoveryCase, monkeypatch: pytest.MonkeyPatch + case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, app_records: AppGenerationRepository ) -> None: - from core.app.apps.agent_app.app_runner import AgentAppRunner + from clients.agent_backend import FakeAgentBackendRunClient from graphon.model_runtime.entities.llm_entities import LLMUsage + from services.app.generation.adapters.agent_runner import AgentAppRunner + from services.app.generation.ports import MessageIdentity + from tests.unit_tests.core.app.apps.agent_app.test_app_runner import _FakeSessionStore, _runner # The worker's scoped session may already have loaded this Message before # the consumer commits its independent stop transaction. @@ -381,8 +389,14 @@ def test_worker_usage_flush_preserves_stop_intent_from_a_newer_consumer_save( assert cached_message is not None assert cached_message.message_metadata is None save_agent_stop(case, monkeypatch, "partial answer") - AgentAppRunner._persist_message_usage( - message_id=cached_message.id, + runner: AgentAppRunner = _runner(FakeAgentBackendRunClient(), _FakeSessionStore(), app_records=app_records) + runner._persist_message_usage( + identity=MessageIdentity( + case.collection.run.tenant_id, + case.collection.run.app_id, + cached_message.conversation_id, + cached_message.id, + ), usage=LLMUsage.from_metadata({"prompt_tokens": 13, "completion_tokens": 8, "total_price": "0.21"}), ) with case.collection.sessions() as session: @@ -396,10 +410,13 @@ def test_worker_usage_flush_preserves_stop_intent_from_a_newer_consumer_save( def test_stop_save_preserves_worker_usage_committed_after_its_cached_message_read( - case: RecoveryCase, monkeypatch: pytest.MonkeyPatch + case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, app_records: AppGenerationRepository ) -> None: - from core.app.apps.agent_app.app_runner import AgentAppRunner + from clients.agent_backend import FakeAgentBackendRunClient from graphon.model_runtime.entities.llm_entities import LLMUsage + from services.app.generation.adapters.agent_runner import AgentAppRunner + from services.app.generation.ports import MessageIdentity + from tests.unit_tests.core.app.apps.agent_app.test_app_runner import _FakeSessionStore, _runner with case.collection.sessions.begin() as session: initial = session.get(Message, case.collection.run.source_id) @@ -410,8 +427,14 @@ def test_stop_save_preserves_worker_usage_committed_after_its_cached_message_rea cached = consumer.get(Message, case.collection.run.source_id) assert cached is not None assert cached.message_tokens == 0 - AgentAppRunner._persist_message_usage( - message_id=cached.id, + runner: AgentAppRunner = _runner(FakeAgentBackendRunClient(), _FakeSessionStore(), app_records=app_records) + runner._persist_message_usage( + identity=MessageIdentity( + case.collection.run.tenant_id, + case.collection.run.app_id, + cached.conversation_id, + cached.id, + ), usage=LLMUsage.from_metadata({"prompt_tokens": 13, "completion_tokens": 8, "total_price": "0.21"}), ) save_agent_stop(case, monkeypatch, "partial answer", consumer_session=consumer) From 8daba3e51d9b6a26a708cddc733b5d97248ef052 Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Sat, 31 Oct 2026 22:12:40 +0800 Subject: [PATCH 15/20] Regenerate API contracts for application execution entry points --- api/openapi/markdown/console-openapi.md | 106 +++----------- api/openapi/markdown/service-openapi.md | 2 +- .../generated/api/console/apps/errors.gen.ts | 68 +-------- .../generated/api/console/apps/orpc.gen.ts | 138 +----------------- .../generated/api/console/apps/types.gen.ts | 40 ++--- .../generated/api/console/errors.gen.ts | 26 ---- .../generated/api/console/rag/errors.gen.ts | 7 - .../generated/api/console/rag/orpc.gen.ts | 16 +- .../generated/api/console/rag/types.gen.ts | 4 - .../api/console/snippets/orpc.gen.ts | 4 +- 10 files changed, 50 insertions(+), 361 deletions(-) diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md index c76ad5f0d8295c..33f4fbfc56d37a 100644 --- a/api/openapi/markdown/console-openapi.md +++ b/api/openapi/markdown/console-openapi.md @@ -2049,8 +2049,6 @@ Get advanced chat workflow run list | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/preview -**Preview human input form content and placeholders** - Get human input form preview for advanced chat workflow #### Parameters @@ -2071,12 +2069,9 @@ Get human input form preview for advanced chat workflow | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Human input form preview | **application/json**: [HumanInputFormPreviewResponse](#humaninputformpreviewresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/run -**Submit human input form preview** - Submit human input form preview for advanced chat workflow #### Parameters @@ -2097,7 +2092,6 @@ Submit human input form preview for advanced chat workflow | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Human input form submission result | **application/json**: [HumanInputFormSubmitResponse](#humaninputformsubmitresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/message-template/test @@ -2123,8 +2117,6 @@ Submit human input form preview for advanced chat workflow | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/iteration/nodes/{node_id}/run -**Run draft workflow iteration node** - Run draft workflow iteration node for advanced chat #### Parameters @@ -2146,12 +2138,10 @@ Run draft workflow iteration node for advanced chat | ---- | ----------- | ------ | | 200 | Iteration node run started successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Node not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Node not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/loop/nodes/{node_id}/run -**Run draft workflow loop node** - Run draft workflow loop node for advanced chat #### Parameters @@ -2173,12 +2163,10 @@ Run draft workflow loop node for advanced chat | ---- | ----------- | ------ | | 200 | Loop node run started successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Node not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Node not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/run -**Run draft workflow** - Run draft workflow for advanced chat application #### Parameters @@ -2200,7 +2188,6 @@ Run draft workflow for advanced chat application | 200 | Workflow run started successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 400 | Invalid request parameters | **application/json**: [ErrorResponse](#errorresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | 429 | Too Many Requests | **application/json**: [RateLimitError](#ratelimiterror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| @@ -3126,11 +3113,7 @@ Get conversation variables for an application | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/convert-to-workflow -**Convert basic mode of chatbot app to workflow mode** - Convert application to workflow mode -Convert expert mode of chatbot app to workflow mode -Convert Completion App to Workflow App #### Parameters @@ -3151,7 +3134,6 @@ Convert Completion App to Workflow App | 200 | Application converted to workflow successfully | **application/json**: [NewAppResponse](#newappresponse)
| | 400 | Application cannot be converted | **application/json**: [ErrorResponse](#errorresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/copy @@ -4085,8 +4067,6 @@ Get workflow application execution logs | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflow-runs/tasks/{task_id}/stop -**Stop workflow task** - Stop running workflow task #### Parameters @@ -4102,7 +4082,7 @@ Stop running workflow task | ---- | ----------- | ------ | | 200 | Task stopped successfully | **application/json**: [SimpleResultResponse](#simpleresultresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Task not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Task not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflow-runs/{run_id} @@ -4523,8 +4503,6 @@ Get workflow daily token cost statistics | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows -**Get published workflows** - Get all published workflows for an application #### Parameters @@ -4542,12 +4520,9 @@ Get all published workflows for an application | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Published workflows retrieved successfully | **application/json**: [WorkflowPaginationResponse](#workflowpaginationresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows/default-workflow-block-configs -**Get default block config** - Get default block configurations for workflow #### Parameters @@ -4561,12 +4536,9 @@ Get default block configurations for workflow | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Default block configurations retrieved successfully | **application/json**: [DefaultBlockConfigsResponse](#defaultblockconfigsresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows/default-workflow-block-configs/{block_type} -**Get default block config** - Get default block configuration by type #### Parameters @@ -4582,12 +4554,10 @@ Get default block configuration by type | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Default block configuration retrieved successfully | **application/json**: [DefaultBlockConfigResponse](#defaultblockconfigresponse)
| -| 404 | Block type not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Block type not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows/draft -**Get draft workflow** - Get draft workflow for an application #### Parameters @@ -4601,12 +4571,10 @@ Get draft workflow for an application | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Draft workflow retrieved successfully | **application/json**: [WorkflowResponse](#workflowresponse)
| -| 404 | Draft workflow not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[DraftWorkflowNotExistError](#draftworkflownotexisterror)
[ErrorResponse](#errorresponse)
| +| 404 | Draft workflow not found | **application/json**: [DraftWorkflowNotExistError](#draftworkflownotexisterror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft -**Sync draft workflow** - Sync draft workflow configuration #### Parameters @@ -4628,7 +4596,6 @@ Sync draft workflow configuration | 200 | Draft workflow synced successfully | **application/json**: [SyncDraftWorkflowResponse](#syncdraftworkflowresponse)
| | 400 | Invalid workflow configuration | **application/json**: [ErrorResponse](#errorresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | 409 | Conflict | **application/json**: [DraftWorkflowNotSyncError](#draftworkflownotsyncerror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| @@ -4732,12 +4699,9 @@ Update draft workflow features | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Workflow features updated successfully | **application/json**: [SimpleResultResponse](#simpleresultresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/delivery-test -**Test human input delivery** - Test human input delivery for workflow #### Parameters @@ -4758,12 +4722,9 @@ Test human input delivery for workflow | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Human input delivery test result | **application/json**: [EmptyObjectResponse](#emptyobjectresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/preview -**Preview human input form content and placeholders** - Get human input form preview for workflow #### Parameters @@ -4784,12 +4745,9 @@ Get human input form preview for workflow | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Human input form preview | **application/json**: [HumanInputFormPreviewResponse](#humaninputformpreviewresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/run -**Submit human input form preview** - Submit human input form preview for workflow #### Parameters @@ -4810,7 +4768,6 @@ Submit human input form preview for workflow | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Human input form submission result | **application/json**: [HumanInputFormSubmitResponse](#humaninputformsubmitresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/message-template/test @@ -4836,7 +4793,7 @@ Submit human input form preview for workflow | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/iteration/nodes/{node_id}/run -**Run draft workflow iteration node** +Run draft workflow iteration node #### Parameters @@ -4857,11 +4814,11 @@ Submit human input form preview for workflow | ---- | ----------- | ------ | | 200 | Workflow iteration node run started successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Node not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Node not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/loop/nodes/{node_id}/run -**Run draft workflow loop node** +Run draft workflow loop node #### Parameters @@ -4882,7 +4839,7 @@ Submit human input form preview for workflow | ---- | ----------- | ------ | | 200 | Workflow loop node run started successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Node not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Node not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer @@ -5051,11 +5008,11 @@ Get last run result for draft workflow node | ---- | ----------- | ------ | | 200 | Node last run retrieved successfully | **application/json**: [WorkflowRunNodeExecutionResponse](#workflowrunnodeexecutionresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Node last run not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Node last run not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/nodes/{node_id}/run -**Run draft workflow node** +Run draft workflow node #### Parameters @@ -5076,11 +5033,11 @@ Get last run result for draft workflow node | ---- | ----------- | ------ | | 200 | Node run started successfully | **application/json**: [WorkflowRunNodeExecutionResponse](#workflowrunnodeexecutionresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Node not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Node not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/nodes/{node_id}/trigger/run -**Poll for trigger events and execute single node when event arrives** +Poll for trigger events and execute single node when event arrives #### Parameters @@ -5096,7 +5053,6 @@ Get last run result for draft workflow node | 200 | Trigger event received and node executed successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 400 | Trigger run failed | **application/json**: [WorkflowTriggerRunErrorResponse](#workflowtriggerrunerrorresponse)
[ErrorResponse](#errorresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | 500 | Internal server error | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| @@ -5135,7 +5091,7 @@ Get variables for a specific node | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/run -**Run draft workflow** +Run draft workflow #### Parameters @@ -5155,7 +5111,6 @@ Get variables for a specific node | ---- | ----------- | ------ | | 200 | Draft workflow run started successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | 429 | Too Many Requests | **application/json**: [RateLimitError](#ratelimiterror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| @@ -5251,7 +5206,7 @@ Get system variables for workflow | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/trigger/run -**Poll for trigger events and execute full workflow when event arrives** +Poll for trigger events and execute full workflow when event arrives #### Parameters @@ -5272,13 +5227,12 @@ Get system variables for workflow | 200 | Trigger event received and workflow executed successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 400 | Trigger run failed | **application/json**: [WorkflowTriggerRunErrorResponse](#workflowtriggerrunerrorresponse)
[ErrorResponse](#errorresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | 429 | Too Many Requests | **application/json**: [RateLimitError](#ratelimiterror)
[ErrorResponse](#errorresponse)
| | 500 | Internal server error | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/trigger/run-all -**Full workflow debug when the start node is a trigger** +Full workflow debug when the start node is a trigger #### Parameters @@ -5299,7 +5253,6 @@ Get system variables for workflow | 200 | Workflow executed successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 400 | Trigger run failed | **application/json**: [WorkflowTriggerRunErrorResponse](#workflowtriggerrunerrorresponse)
[ErrorResponse](#errorresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | 429 | Too Many Requests | **application/json**: [RateLimitError](#ratelimiterror)
[ErrorResponse](#errorresponse)
| | 500 | Internal server error | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| @@ -5421,8 +5374,6 @@ Reset a workflow variable to its default value | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows/publish -**Get published workflow** - Get published workflow for an application #### Parameters @@ -5436,12 +5387,9 @@ Get published workflow for an application | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Published workflow retrieved successfully, or null if not found | **application/json**: [WorkflowResponse](#workflowresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/publish -**Publish workflow** - #### Parameters | Name | Located in | Description | Required | Schema | @@ -5459,7 +5407,6 @@ Get published workflow for an application | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Workflow published successfully | **application/json**: [WorkflowPublishResponse](#workflowpublishresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows/published/runs/{run_id}/node-outputs @@ -5556,8 +5503,6 @@ Full value for one declared output of a published run. | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [DELETE] /apps/{app_id}/workflows/{workflow_id} -**Delete workflow** - #### Parameters | Name | Located in | Description | Required | Schema | @@ -5570,12 +5515,9 @@ Full value for one declared output of a published run. | Code | Description | Schema | | ---- | ----------- | ------ | | 204 | Workflow deleted successfully | | -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [PATCH] /apps/{app_id}/workflows/{workflow_id} -**Update workflow attributes** - Update workflow by ID #### Parameters @@ -5597,7 +5539,7 @@ Update workflow by ID | ---- | ----------- | ------ | | 200 | Workflow updated successfully | **application/json**: [WorkflowResponse](#workflowresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Workflow not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Workflow not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/{workflow_id}/restore @@ -5616,7 +5558,7 @@ Restore a published workflow version into the draft workflow | ---- | ----------- | ------ | | 200 | Workflow restored successfully | **application/json**: [WorkflowRestoreResponse](#workflowrestoreresponse)
| | 400 | Source workflow must be published | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Workflow not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Workflow not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{resource_id}/api-keys @@ -11869,7 +11811,6 @@ Update account-level Step-by-step Tour state | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Success | **application/json**: [RagPipelineWorkflowSyncResponse](#ragpipelineworkflowsyncresponse)
| -| 404 | Not Found | **application/json**: [PipelineNotFoundError](#pipelinenotfounderror)
[ErrorResponse](#errorresponse)
| | 409 | Conflict | **application/json**: [DraftWorkflowNotSyncError](#draftworkflownotsyncerror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| @@ -12247,7 +12188,7 @@ Update account-level Step-by-step Tour state | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /rag/pipelines/{pipeline_id}/workflows/publish -**Publish workflow** +**Publish a prepared Pipeline and its dataset settings atomically** #### Parameters @@ -12260,7 +12201,6 @@ Update account-level Step-by-step Tour state | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Success | **application/json**: [RagPipelineWorkflowPublishResponse](#ragpipelineworkflowpublishresponse)
| -| 404 | Not Found | **application/json**: [PipelineNotFoundError](#pipelinenotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /rag/pipelines/{pipeline_id}/workflows/published/datasource/nodes/{node_id}/preview @@ -12372,8 +12312,6 @@ Update account-level Step-by-step Tour state | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [DELETE] /rag/pipelines/{pipeline_id}/workflows/{workflow_id} -**Delete a published workflow version that is not currently active on the pipeline** - #### Parameters | Name | Located in | Description | Required | Schema | @@ -12386,7 +12324,6 @@ Update account-level Step-by-step Tour state | Code | Description | Schema | | ---- | ----------- | ------ | | 204 | Workflow deleted successfully | | -| 404 | Not Found | **application/json**: [PipelineNotFoundError](#pipelinenotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [PATCH] /rag/pipelines/{pipeline_id}/workflows/{workflow_id} @@ -12428,7 +12365,6 @@ Update account-level Step-by-step Tour state | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Success | **application/json**: [RagPipelineWorkflowSyncResponse](#ragpipelineworkflowsyncresponse)
| -| 404 | Not Found | **application/json**: [PipelineNotFoundError](#pipelinenotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /refresh-token @@ -13292,7 +13228,9 @@ Reset a draft workflow variable to its default value (snippet scope) | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [DELETE] /snippets/{snippet_id}/workflows/{workflow_id} -**Delete a published snippet workflow version** +**Delete a version and retire its unowned Agents through the shared use case** + +Delete a published snippet workflow version #### Parameters diff --git a/api/openapi/markdown/service-openapi.md b/api/openapi/markdown/service-openapi.md index 0962c702afd98b..e717aae3a90d03 100644 --- a/api/openapi/markdown/service-openapi.md +++ b/api/openapi/markdown/service-openapi.md @@ -2746,7 +2746,7 @@ Execute the full knowledge pipeline for a knowledge base. Published runs are que | 200 | Pipeline execution result. Published runs return a JSON object containing `batch`, `dataset`, and `documents`. Draft runs return `text/event-stream` for streaming mode or a workflow result JSON object for blocking mode. | **application/json**: [PipelineRunJsonResponse](#pipelinerunjsonresponse)
**text/event-stream**: string
| | 400 | Bad request - invalid payload or pipeline is not configured | **application/json**: [ErrorResponse](#errorresponse)
| | 401 | Unauthorized - invalid API token | **application/json**: [ErrorResponse](#errorresponse)
| -| 403 | `forbidden` : Forbidden. | **application/json**: [ErrorResponse](#errorresponse)
| +| 403 | `forbidden` : AccessDeniedError. | **application/json**: [ErrorResponse](#errorresponse)
| | 404 | `not_found` : Dataset not found. | **application/json**: [ErrorResponse](#errorresponse)
| | 500 | `pipeline_run_error` : Pipeline execution failed. | **application/json**: [PipelineRunError](#pipelinerunerror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| diff --git a/packages/contracts/generated/api/console/apps/errors.gen.ts b/packages/contracts/generated/api/console/apps/errors.gen.ts index 4aa9548e94fe18..3652108ea43858 100644 --- a/packages/contracts/generated/api/console/apps/errors.gen.ts +++ b/packages/contracts/generated/api/console/apps/errors.gen.ts @@ -604,20 +604,9 @@ export const get4Errors = { app_not_found: { status: 404, data: zAppNotFoundErro export const get5Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const post4Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post5Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const post6Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const post7Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post8Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post9Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, -} +export const post9Errors = { rate_limit_error: { status: 429, data: zRateLimitError } } export const post10Errors = { agent_not_bound: { status: 400, data: zAgentNotBoundError }, @@ -791,8 +780,6 @@ export const post22Errors = { export const get30Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const post23Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get31Errors = { roster_agent_package_export_failed: { status: 500, data: zRosterAgentPackageExportFailedError }, } @@ -971,8 +958,6 @@ export const get50Errors = { app_not_found: { status: 404, data: zAppNotFoundErr export const get51Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const post41Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get52Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } export const get53Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } @@ -1061,28 +1046,12 @@ export const get63Errors = { app_not_found: { status: 404, data: zAppNotFoundErr export const get64Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const get65Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const get66Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get68Errors = { draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, } -export const post48Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post49Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post50Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post51Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const post52Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const post53Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post54Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get69Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } export const post55Errors = { @@ -1128,16 +1097,7 @@ export const put5Errors = { plaintext_secret_not_allowed_error: { status: 400, data: zPlaintextSecretNotAllowedError }, } -export const get71Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post59Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post60Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post61Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, -} +export const post61Errors = { rate_limit_error: { status: 429, data: zRateLimitError } } export const get73Errors = { app_not_found: { status: 404, data: zAppNotFoundError }, @@ -1162,34 +1122,22 @@ export const get76Errors = { workflow_run_not_found: { status: 404, data: zWorkflowRunNotFoundError }, } -export const post62Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, -} +export const post62Errors = { rate_limit_error: { status: 429, data: zRateLimitError } } -export const post63Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, -} +export const post63Errors = { rate_limit_error: { status: 429, data: zRateLimitError } } export const get79Errors = { draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, } export const get80Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, } export const post64Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError }, } -export const get81Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post65Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get82Errors = { app_not_found: { status: 404, data: zAppNotFoundError }, workflow_run_not_found: { status: 404, data: zWorkflowRunNotFoundError }, @@ -1218,14 +1166,6 @@ export const get86Errors = { webhook_trigger_not_found: { status: 404, data: zWebhookTriggerNotFoundError }, } -export const post66Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const delete14Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const patch3Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const get87Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const post67Errors = { agent_not_published: { status: 409, data: zAgentNotPublishedError }, max_keys_exceeded: { status: 400, data: zMaxKeysExceededError }, diff --git a/packages/contracts/generated/api/console/apps/orpc.gen.ts b/packages/contracts/generated/api/console/apps/orpc.gen.ts index f22bf4f76a1e38..64c9308be01ca5 100644 --- a/packages/contracts/generated/api/console/apps/orpc.gen.ts +++ b/packages/contracts/generated/api/console/apps/orpc.gen.ts @@ -747,8 +747,6 @@ export const workflowRuns = { } /** - * Preview human input form content and placeholders - * * Get human input form preview for advanced chat workflow */ export const post4 = oc @@ -758,7 +756,6 @@ export const post4 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormPreview', path: '/apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/preview', - summary: 'Preview human input form content and placeholders', tags: ['console'], }) .input( @@ -768,15 +765,12 @@ export const post4 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormPreviewResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const preview = { post: post4, } /** - * Submit human input form preview - * * Submit human input form preview for advanced chat workflow */ export const post5 = oc @@ -786,7 +780,6 @@ export const post5 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/run', - summary: 'Submit human input form preview', tags: ['console'], }) .input( @@ -796,7 +789,6 @@ export const post5 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run = { post: post5, @@ -850,8 +842,6 @@ export const humanInput = { } /** - * Run draft workflow iteration node - * * Run draft workflow iteration node for advanced chat */ export const post7 = oc @@ -861,7 +851,6 @@ export const post7 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftIterationNodesByNodeIdRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/iteration/nodes/{node_id}/run', - summary: 'Run draft workflow iteration node', tags: ['console'], }) .input( @@ -871,7 +860,6 @@ export const post7 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftIterationNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run2 = { post: post7, @@ -890,8 +878,6 @@ export const iteration = { } /** - * Run draft workflow loop node - * * Run draft workflow loop node for advanced chat */ export const post8 = oc @@ -901,7 +887,6 @@ export const post8 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftLoopNodesByNodeIdRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/loop/nodes/{node_id}/run', - summary: 'Run draft workflow loop node', tags: ['console'], }) .input( @@ -911,7 +896,6 @@ export const post8 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftLoopNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run3 = { post: post8, @@ -930,8 +914,6 @@ export const loop = { } /** - * Run draft workflow - * * Run draft workflow for advanced chat application */ export const post9 = oc @@ -941,7 +923,6 @@ export const post9 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/run', - summary: 'Run draft workflow', tags: ['console'], }) .input( @@ -951,10 +932,7 @@ export const post9 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftRunResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, - }) + .errors({ rate_limit_error: { status: 429, data: zRateLimitError } }) export const run4 = { post: post9, @@ -2077,21 +2055,15 @@ export const conversationVariables = { } /** - * Convert basic mode of chatbot app to workflow mode - * * Convert application to workflow mode - * Convert expert mode of chatbot app to workflow mode - * Convert Completion App to Workflow App */ export const post23 = oc .route({ - description: - 'Convert application to workflow mode\nConvert expert mode of chatbot app to workflow mode\nConvert Completion App to Workflow App', + description: 'Convert application to workflow mode', inputStructure: 'detailed', method: 'POST', operationId: 'postAppsByAppIdConvertToWorkflow', path: '/apps/{app_id}/convert-to-workflow', - summary: 'Convert basic mode of chatbot app to workflow mode', tags: ['console'], }) .input( @@ -2101,7 +2073,6 @@ export const post23 = oc }), ) .output(zPostAppsByAppIdConvertToWorkflowResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const convertToWorkflow = { post: post23, @@ -3180,8 +3151,6 @@ export const count3 = { } /** - * Stop workflow task - * * Stop running workflow task */ export const post41 = oc @@ -3191,12 +3160,10 @@ export const post41 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowRunsTasksByTaskIdStop', path: '/apps/{app_id}/workflow-runs/tasks/{task_id}/stop', - summary: 'Stop workflow task', tags: ['console'], }) .input(z.object({ params: zPostAppsByAppIdWorkflowRunsTasksByTaskIdStopPath })) .output(zPostAppsByAppIdWorkflowRunsTasksByTaskIdStopResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const stop3 = { post: post41, @@ -3818,8 +3785,6 @@ export const workflow = { } /** - * Get default block config - * * Get default block configuration by type */ export const get65 = oc @@ -3829,7 +3794,6 @@ export const get65 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsByBlockType', path: '/apps/{app_id}/workflows/default-workflow-block-configs/{block_type}', - summary: 'Get default block config', tags: ['console'], }) .input( @@ -3839,15 +3803,12 @@ export const get65 = oc }), ) .output(zGetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsByBlockTypeResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const byBlockType = { get: get65, } /** - * Get default block config - * * Get default block configurations for workflow */ export const get66 = oc @@ -3857,12 +3818,10 @@ export const get66 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDefaultWorkflowBlockConfigs', path: '/apps/{app_id}/workflows/default-workflow-block-configs', - summary: 'Get default block config', tags: ['console'], }) .input(z.object({ params: zGetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsPath })) .output(zGetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const defaultWorkflowBlockConfigs = { get: get66, @@ -3972,15 +3931,12 @@ export const post48 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftFeaturesResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const features = { post: post48, } /** - * Test human input delivery - * * Test human input delivery for workflow */ export const post49 = oc @@ -3990,7 +3946,6 @@ export const post49 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdDeliveryTest', path: '/apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/delivery-test', - summary: 'Test human input delivery', tags: ['console'], }) .input( @@ -4000,15 +3955,12 @@ export const post49 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdDeliveryTestResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const deliveryTest = { post: post49, } /** - * Preview human input form content and placeholders - * * Get human input form preview for workflow */ export const post50 = oc @@ -4018,7 +3970,6 @@ export const post50 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormPreview', path: '/apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/preview', - summary: 'Preview human input form content and placeholders', tags: ['console'], }) .input( @@ -4028,15 +3979,12 @@ export const post50 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormPreviewResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const preview4 = { post: post50, } /** - * Submit human input form preview - * * Submit human input form preview for workflow */ export const post51 = oc @@ -4046,7 +3994,6 @@ export const post51 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormRun', path: '/apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/run', - summary: 'Submit human input form preview', tags: ['console'], }) .input( @@ -4056,7 +4003,6 @@ export const post51 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run5 = { post: post51, @@ -4107,8 +4053,6 @@ export const humanInput2 = { } /** - * Run draft workflow iteration node - * * Run draft workflow iteration node */ export const post53 = oc @@ -4118,7 +4062,6 @@ export const post53 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftIterationNodesByNodeIdRun', path: '/apps/{app_id}/workflows/draft/iteration/nodes/{node_id}/run', - summary: 'Run draft workflow iteration node', tags: ['console'], }) .input( @@ -4128,7 +4071,6 @@ export const post53 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftIterationNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run6 = { post: post53, @@ -4147,8 +4089,6 @@ export const iteration2 = { } /** - * Run draft workflow loop node - * * Run draft workflow loop node */ export const post54 = oc @@ -4158,7 +4098,6 @@ export const post54 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftLoopNodesByNodeIdRun', path: '/apps/{app_id}/workflows/draft/loop/nodes/{node_id}/run', - summary: 'Run draft workflow loop node', tags: ['console'], }) .input( @@ -4168,7 +4107,6 @@ export const post54 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftLoopNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run7 = { post: post54, @@ -4377,15 +4315,12 @@ export const get71 = oc }) .input(z.object({ params: zGetAppsByAppIdWorkflowsDraftNodesByNodeIdLastRunPath })) .output(zGetAppsByAppIdWorkflowsDraftNodesByNodeIdLastRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const lastRun = { get: get71, } /** - * Run draft workflow node - * * Run draft workflow node */ export const post59 = oc @@ -4395,7 +4330,6 @@ export const post59 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftNodesByNodeIdRun', path: '/apps/{app_id}/workflows/draft/nodes/{node_id}/run', - summary: 'Run draft workflow node', tags: ['console'], }) .input( @@ -4405,15 +4339,12 @@ export const post59 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run8 = { post: post59, } /** - * Poll for trigger events and execute single node when event arrives - * * Poll for trigger events and execute single node when event arrives */ export const post60 = oc @@ -4423,12 +4354,10 @@ export const post60 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRun', path: '/apps/{app_id}/workflows/draft/nodes/{node_id}/trigger/run', - summary: 'Poll for trigger events and execute single node when event arrives', tags: ['console'], }) .input(z.object({ params: zPostAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRunPath })) .output(zPostAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run9 = { post: post60, @@ -4487,8 +4416,6 @@ export const nodes7 = { } /** - * Run draft workflow - * * Run draft workflow */ export const post61 = oc @@ -4498,7 +4425,6 @@ export const post61 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftRun', path: '/apps/{app_id}/workflows/draft/run', - summary: 'Run draft workflow', tags: ['console'], }) .input( @@ -4508,10 +4434,7 @@ export const post61 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftRunResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, - }) + .errors({ rate_limit_error: { status: 429, data: zRateLimitError } }) export const run10 = { post: post61, @@ -4651,8 +4574,6 @@ export const systemVariables = { } /** - * Poll for trigger events and execute full workflow when event arrives - * * Poll for trigger events and execute full workflow when event arrives */ export const post62 = oc @@ -4662,7 +4583,6 @@ export const post62 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftTriggerRun', path: '/apps/{app_id}/workflows/draft/trigger/run', - summary: 'Poll for trigger events and execute full workflow when event arrives', tags: ['console'], }) .input( @@ -4672,18 +4592,13 @@ export const post62 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftTriggerRunResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, - }) + .errors({ rate_limit_error: { status: 429, data: zRateLimitError } }) export const run11 = { post: post62, } /** - * Full workflow debug when the start node is a trigger - * * Full workflow debug when the start node is a trigger */ export const post63 = oc @@ -4693,7 +4608,6 @@ export const post63 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftTriggerRunAll', path: '/apps/{app_id}/workflows/draft/trigger/run-all', - summary: 'Full workflow debug when the start node is a trigger', tags: ['console'], }) .input( @@ -4703,10 +4617,7 @@ export const post63 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftTriggerRunAllResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, - }) + .errors({ rate_limit_error: { status: 429, data: zRateLimitError } }) export const runAll = { post: post63, @@ -4841,8 +4752,6 @@ export const variables2 = { } /** - * Get draft workflow - * * Get draft workflow for an application */ export const get80 = oc @@ -4852,19 +4761,13 @@ export const get80 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDraft', path: '/apps/{app_id}/workflows/draft', - summary: 'Get draft workflow', tags: ['console'], }) .input(z.object({ params: zGetAppsByAppIdWorkflowsDraftPath })) .output(zGetAppsByAppIdWorkflowsDraftResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, - }) + .errors({ draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError } }) /** - * Sync draft workflow - * * Sync draft workflow configuration */ export const post64 = oc @@ -4874,7 +4777,6 @@ export const post64 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraft', path: '/apps/{app_id}/workflows/draft', - summary: 'Sync draft workflow', tags: ['console'], }) .input( @@ -4884,10 +4786,7 @@ export const post64 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError }, - }) + .errors({ draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError } }) export const draft2 = { get: get80, @@ -4907,8 +4806,6 @@ export const draft2 = { } /** - * Get published workflow - * * Get published workflow for an application */ export const get81 = oc @@ -4918,23 +4815,17 @@ export const get81 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsPublish', path: '/apps/{app_id}/workflows/publish', - summary: 'Get published workflow', tags: ['console'], }) .input(z.object({ params: zGetAppsByAppIdWorkflowsPublishPath })) .output(zGetAppsByAppIdWorkflowsPublishResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) -/** - * Publish workflow - */ export const post65 = oc .route({ inputStructure: 'detailed', method: 'POST', operationId: 'postAppsByAppIdWorkflowsPublish', path: '/apps/{app_id}/workflows/publish', - summary: 'Publish workflow', tags: ['console'], }) .input( @@ -4944,7 +4835,6 @@ export const post65 = oc }), ) .output(zPostAppsByAppIdWorkflowsPublishResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const publish = { get: get81, @@ -5119,15 +5009,11 @@ export const post66 = oc }) .input(z.object({ params: zPostAppsByAppIdWorkflowsByWorkflowIdRestorePath })) .output(zPostAppsByAppIdWorkflowsByWorkflowIdRestoreResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const restore = { post: post66, } -/** - * Delete workflow - */ export const delete14 = oc .route({ inputStructure: 'detailed', @@ -5135,16 +5021,12 @@ export const delete14 = oc operationId: 'deleteAppsByAppIdWorkflowsByWorkflowId', path: '/apps/{app_id}/workflows/{workflow_id}', successStatus: 204, - summary: 'Delete workflow', tags: ['console'], }) .input(z.object({ params: zDeleteAppsByAppIdWorkflowsByWorkflowIdPath })) .output(zDeleteAppsByAppIdWorkflowsByWorkflowIdResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) /** - * Update workflow attributes - * * Update workflow by ID */ export const patch3 = oc @@ -5154,7 +5036,6 @@ export const patch3 = oc method: 'PATCH', operationId: 'patchAppsByAppIdWorkflowsByWorkflowId', path: '/apps/{app_id}/workflows/{workflow_id}', - summary: 'Update workflow attributes', tags: ['console'], }) .input( @@ -5164,7 +5045,6 @@ export const patch3 = oc }), ) .output(zPatchAppsByAppIdWorkflowsByWorkflowIdResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const byWorkflowId = { delete: delete14, @@ -5173,8 +5053,6 @@ export const byWorkflowId = { } /** - * Get published workflows - * * Get all published workflows for an application */ export const get87 = oc @@ -5184,7 +5062,6 @@ export const get87 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflows', path: '/apps/{app_id}/workflows', - summary: 'Get published workflows', tags: ['console'], }) .input( @@ -5194,7 +5071,6 @@ export const get87 = oc }), ) .output(zGetAppsByAppIdWorkflowsResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const workflows3 = { get: get87, diff --git a/packages/contracts/generated/api/console/apps/types.gen.ts b/packages/contracts/generated/api/console/apps/types.gen.ts index b175d789433091..dc794fbcc29de8 100644 --- a/packages/contracts/generated/api/console/apps/types.gen.ts +++ b/packages/contracts/generated/api/console/apps/types.gen.ts @@ -5156,7 +5156,6 @@ export type PostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdForm } export type PostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormPreviewErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -5181,7 +5180,6 @@ export type PostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdForm } export type PostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormRunErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -5235,7 +5233,7 @@ export type PostAppsByAppIdAdvancedChatWorkflowsDraftIterationNodesByNodeIdRunDa export type PostAppsByAppIdAdvancedChatWorkflowsDraftIterationNodesByNodeIdRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -5261,7 +5259,7 @@ export type PostAppsByAppIdAdvancedChatWorkflowsDraftLoopNodesByNodeIdRunData = export type PostAppsByAppIdAdvancedChatWorkflowsDraftLoopNodesByNodeIdRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -5287,7 +5285,6 @@ export type PostAppsByAppIdAdvancedChatWorkflowsDraftRunData = { export type PostAppsByAppIdAdvancedChatWorkflowsDraftRunErrors = { 400: ErrorResponse 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse 429: RateLimitError | ErrorResponse default: ErrorResponse } @@ -6566,7 +6563,6 @@ export type PostAppsByAppIdConvertToWorkflowData = { export type PostAppsByAppIdConvertToWorkflowErrors = { 400: ErrorResponse 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -7806,7 +7802,7 @@ export type PostAppsByAppIdWorkflowRunsTasksByTaskIdStopData = { export type PostAppsByAppIdWorkflowRunsTasksByTaskIdStopErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -8370,7 +8366,6 @@ export type GetAppsByAppIdWorkflowsData = { } export type GetAppsByAppIdWorkflowsErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -8394,7 +8389,6 @@ export type GetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsData = { } export type GetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -8421,7 +8415,7 @@ export type GetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsByBlockTypeData = } export type GetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsByBlockTypeErrors = { - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -8445,7 +8439,7 @@ export type GetAppsByAppIdWorkflowsDraftData = { } export type GetAppsByAppIdWorkflowsDraftErrors = { - 404: AppNotFoundError | DraftWorkflowNotExistError | ErrorResponse + 404: DraftWorkflowNotExistError | ErrorResponse default: ErrorResponse } @@ -8471,7 +8465,6 @@ export type PostAppsByAppIdWorkflowsDraftData = { export type PostAppsByAppIdWorkflowsDraftErrors = { 400: ErrorResponse 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse 409: DraftWorkflowNotSyncError | ErrorResponse default: ErrorResponse } @@ -8590,7 +8583,6 @@ export type PostAppsByAppIdWorkflowsDraftFeaturesData = { } export type PostAppsByAppIdWorkflowsDraftFeaturesErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -8615,7 +8607,6 @@ export type PostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdDeliveryTestData } export type PostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdDeliveryTestErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -8640,7 +8631,6 @@ export type PostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormPreviewData } export type PostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormPreviewErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -8665,7 +8655,6 @@ export type PostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormRunData = { } export type PostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormRunErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -8716,7 +8705,7 @@ export type PostAppsByAppIdWorkflowsDraftIterationNodesByNodeIdRunData = { export type PostAppsByAppIdWorkflowsDraftIterationNodesByNodeIdRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -8742,7 +8731,7 @@ export type PostAppsByAppIdWorkflowsDraftLoopNodesByNodeIdRunData = { export type PostAppsByAppIdWorkflowsDraftLoopNodesByNodeIdRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -8964,7 +8953,7 @@ export type GetAppsByAppIdWorkflowsDraftNodesByNodeIdLastRunData = { export type GetAppsByAppIdWorkflowsDraftNodesByNodeIdLastRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -8990,7 +8979,7 @@ export type PostAppsByAppIdWorkflowsDraftNodesByNodeIdRunData = { export type PostAppsByAppIdWorkflowsDraftNodesByNodeIdRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -9017,7 +9006,6 @@ export type PostAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRunData = { export type PostAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRunErrors = { 400: WorkflowTriggerRunErrorResponse | ErrorResponse 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse 500: ErrorResponse default: ErrorResponse } @@ -9091,7 +9079,6 @@ export type PostAppsByAppIdWorkflowsDraftRunData = { export type PostAppsByAppIdWorkflowsDraftRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse 429: RateLimitError | ErrorResponse default: ErrorResponse } @@ -9250,7 +9237,6 @@ export type PostAppsByAppIdWorkflowsDraftTriggerRunData = { export type PostAppsByAppIdWorkflowsDraftTriggerRunErrors = { 400: WorkflowTriggerRunErrorResponse | ErrorResponse 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse 429: RateLimitError | ErrorResponse 500: ErrorResponse default: ErrorResponse @@ -9278,7 +9264,6 @@ export type PostAppsByAppIdWorkflowsDraftTriggerRunAllData = { export type PostAppsByAppIdWorkflowsDraftTriggerRunAllErrors = { 400: WorkflowTriggerRunErrorResponse | ErrorResponse 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse 429: RateLimitError | ErrorResponse 500: ErrorResponse default: ErrorResponse @@ -9455,7 +9440,6 @@ export type GetAppsByAppIdWorkflowsPublishData = { } export type GetAppsByAppIdWorkflowsPublishErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -9479,7 +9463,6 @@ export type PostAppsByAppIdWorkflowsPublishData = { } export type PostAppsByAppIdWorkflowsPublishErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -9641,7 +9624,6 @@ export type DeleteAppsByAppIdWorkflowsByWorkflowIdData = { } export type DeleteAppsByAppIdWorkflowsByWorkflowIdErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -9667,7 +9649,7 @@ export type PatchAppsByAppIdWorkflowsByWorkflowIdData = { export type PatchAppsByAppIdWorkflowsByWorkflowIdErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -9693,7 +9675,7 @@ export type PostAppsByAppIdWorkflowsByWorkflowIdRestoreData = { export type PostAppsByAppIdWorkflowsByWorkflowIdRestoreErrors = { 400: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } diff --git a/packages/contracts/generated/api/console/errors.gen.ts b/packages/contracts/generated/api/console/errors.gen.ts index fd47e4bbf7cf59..5f0e8fa93df185 100644 --- a/packages/contracts/generated/api/console/errors.gen.ts +++ b/packages/contracts/generated/api/console/errors.gen.ts @@ -5,7 +5,6 @@ export { delete11Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/sourc export { delete12Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}' } from './knowledge-fs/errors.gen.ts' export { delete12Errors as 'DELETE /workspaces/current/network-access-groups/{group_id}' } from './workspaces/errors.gen.ts' export { delete13Errors as 'DELETE /knowledge-fs/uploads/{upload_id}' } from './knowledge-fs/errors.gen.ts' -export { delete14Errors as 'DELETE /apps/{app_id}/workflows/{workflow_id}' } from './apps/errors.gen.ts' export { delete20Errors as 'DELETE /workspaces/current/skills/{skill_id}' } from './workspaces/errors.gen.ts' export { delete2Errors as 'DELETE /agent/{agent_id}/build-draft' } from './agent/errors.gen.ts' export { delete2Errors as 'DELETE /apps/{app_id}/agent/config/skills/{name}' } from './apps/errors.gen.ts' @@ -21,7 +20,6 @@ export { delete4Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/docume export { delete5Errors as 'DELETE /agent/{agent_id}/trace-config' } from './agent/errors.gen.ts' export { delete5Errors as 'DELETE /apps/{app_id}/chat-conversations/{conversation_id}' } from './apps/errors.gen.ts' export { delete5Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/golden-questions/{question_id}' } from './knowledge-fs/errors.gen.ts' -export { delete5Errors as 'DELETE /rag/pipelines/{pipeline_id}/workflows/{workflow_id}' } from './rag/errors.gen.ts' export { delete6Errors as 'DELETE /agent/{agent_id}' } from './agent/errors.gen.ts' export { delete6Errors as 'DELETE /apps/{app_id}/completion-conversations/{conversation_id}' } from './apps/errors.gen.ts' export { delete6Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/jobs/{job_id}' } from './knowledge-fs/errors.gen.ts' @@ -227,8 +225,6 @@ export { get61Errors as 'GET /apps/{app_id}/workflow/statistics/average-app-inte export { get62Errors as 'GET /apps/{app_id}/workflow/statistics/daily-conversations' } from './apps/errors.gen.ts' export { get63Errors as 'GET /apps/{app_id}/workflow/statistics/daily-terminals' } from './apps/errors.gen.ts' export { get64Errors as 'GET /apps/{app_id}/workflow/statistics/token-costs' } from './apps/errors.gen.ts' -export { get65Errors as 'GET /apps/{app_id}/workflows/default-workflow-block-configs/{block_type}' } from './apps/errors.gen.ts' -export { get66Errors as 'GET /apps/{app_id}/workflows/default-workflow-block-configs' } from './apps/errors.gen.ts' export { get68Errors as 'GET /apps/{app_id}/workflows/draft/environment-variables' } from './apps/errors.gen.ts' export { get69Errors as 'GET /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/candidates' } from './apps/errors.gen.ts' export { get6Errors as 'GET /agent/{agent_id}/chat-messages' } from './agent/errors.gen.ts' @@ -238,7 +234,6 @@ export { get6Errors as 'GET /knowledge-fs/spaces/{control_space_id}/background-t export { get6Errors as 'GET /rag/pipelines/{pipeline_id}/exports' } from './rag/errors.gen.ts' export { get6Errors as 'GET /trial-apps/{app_id}/agent/config/skills/{name}/files/content' } from './trial-apps/errors.gen.ts' export { get70Errors as 'GET /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer' } from './apps/errors.gen.ts' -export { get71Errors as 'GET /apps/{app_id}/workflows/draft/nodes/{node_id}/last-run' } from './apps/errors.gen.ts' export { get73Errors as 'GET /apps/{app_id}/workflows/draft/runs/{run_id}/node-outputs/events' } from './apps/errors.gen.ts' export { get74Errors as 'GET /apps/{app_id}/workflows/draft/runs/{run_id}/node-outputs/{node_id}/{output_name}/preview' } from './apps/errors.gen.ts' export { get75Errors as 'GET /apps/{app_id}/workflows/draft/runs/{run_id}/node-outputs/{node_id}' } from './apps/errors.gen.ts' @@ -253,13 +248,11 @@ export { get7Errors as 'GET /rag/pipelines/{pipeline_id}/workflow-runs/{run_id}/ export { get7Errors as 'GET /snippets/{snippet_id}/workflows/draft/environment-variables' } from './snippets/errors.gen.ts' export { get7Errors as 'GET /trial-apps/{app_id}/agent/config/skills/{name}/files/download' } from './trial-apps/errors.gen.ts' export { get80Errors as 'GET /apps/{app_id}/workflows/draft' } from './apps/errors.gen.ts' -export { get81Errors as 'GET /apps/{app_id}/workflows/publish' } from './apps/errors.gen.ts' export { get82Errors as 'GET /apps/{app_id}/workflows/published/runs/{run_id}/node-outputs/events' } from './apps/errors.gen.ts' export { get83Errors as 'GET /apps/{app_id}/workflows/published/runs/{run_id}/node-outputs/{node_id}/{output_name}/preview' } from './apps/errors.gen.ts' export { get84Errors as 'GET /apps/{app_id}/workflows/published/runs/{run_id}/node-outputs/{node_id}' } from './apps/errors.gen.ts' export { get85Errors as 'GET /apps/{app_id}/workflows/published/runs/{run_id}/node-outputs' } from './apps/errors.gen.ts' export { get86Errors as 'GET /apps/{app_id}/workflows/triggers/webhook' } from './apps/errors.gen.ts' -export { get87Errors as 'GET /apps/{app_id}/workflows' } from './apps/errors.gen.ts' export { get8Errors as 'GET /agent/{agent_id}/composer' } from './agent/errors.gen.ts' export { get8Errors as 'GET /apps/{app_id}/agent/config/files' } from './apps/errors.gen.ts' export { get8Errors as 'GET /installed-apps/{installed_app_id}' } from './installed-apps/errors.gen.ts' @@ -287,7 +280,6 @@ export { getErrors as 'GET /workflow-run-archives/downloads/{download_id}' } fro export { patch2Errors as 'PATCH /datasets/{dataset_id}/documents/status/{action}/batch' } from './datasets/errors.gen.ts' export { patch2Errors as 'PATCH /installed-apps/{installed_app_id}/conversations/{c_id}/unpin' } from './installed-apps/errors.gen.ts' export { patch2Errors as 'PATCH /knowledge-fs/spaces/{control_space_id}/golden-questions/{question_id}' } from './knowledge-fs/errors.gen.ts' -export { patch3Errors as 'PATCH /apps/{app_id}/workflows/{workflow_id}' } from './apps/errors.gen.ts' export { patch3Errors as 'PATCH /datasets/{dataset_id}/documents/{document_id}/processing/pause' } from './datasets/errors.gen.ts' export { patch3Errors as 'PATCH /installed-apps/{installed_app_id}' } from './installed-apps/errors.gen.ts' export { patch3Errors as 'PATCH /knowledge-fs/spaces/{control_space_id}/logical-documents/{document_id}' } from './knowledge-fs/errors.gen.ts' @@ -345,7 +337,6 @@ export { post15Errors as 'POST /rag/pipelines/{pipeline_id}/workflows/draft' } f export { post16Errors as 'POST /agent/{agent_id}/publish' } from './agent/errors.gen.ts' export { post16Errors as 'POST /datasets/{dataset_id}/external-hit-testing' } from './datasets/errors.gen.ts' export { post16Errors as 'POST /knowledge-fs/spaces/{control_space_id}/queries/admission' } from './knowledge-fs/errors.gen.ts' -export { post16Errors as 'POST /rag/pipelines/{pipeline_id}/workflows/publish' } from './rag/errors.gen.ts' export { post17Errors as 'POST /agent/{agent_id}/sandbox/files/download' } from './agent/errors.gen.ts' export { post17Errors as 'POST /datasets/{dataset_id}/hit-testing' } from './datasets/errors.gen.ts' export { post17Errors as 'POST /knowledge-fs/spaces/{control_space_id}/queries' } from './knowledge-fs/errors.gen.ts' @@ -360,7 +351,6 @@ export { post19Errors as 'POST /rag/pipelines/{pipeline_id}/workflows/published/ export { post20Errors as 'POST /agent/{agent_id}/trace-config' } from './agent/errors.gen.ts' export { post20Errors as 'POST /apps/{app_id}/chat-messages/{task_id}/stop' } from './apps/errors.gen.ts' export { post20Errors as 'POST /knowledge-fs/spaces/{control_space_id}/research-tasks' } from './knowledge-fs/errors.gen.ts' -export { post20Errors as 'POST /rag/pipelines/{pipeline_id}/workflows/{workflow_id}/restore' } from './rag/errors.gen.ts' export { post21Errors as 'POST /agent/{agent_id}/trace-runs/{message_id}/retry' } from './agent/errors.gen.ts' export { post21Errors as 'POST /apps/{app_id}/completion-messages/{task_id}/stop' } from './apps/errors.gen.ts' export { post21Errors as 'POST /knowledge-fs/spaces/{control_space_id}/source-connections/{connection_id}/refresh' } from './knowledge-fs/errors.gen.ts' @@ -368,7 +358,6 @@ export { post22Errors as 'POST /agent/{agent_id}/versions/{version_id}/restore' export { post22Errors as 'POST /apps/{app_id}/completion-messages' } from './apps/errors.gen.ts' export { post22Errors as 'POST /knowledge-fs/spaces/{control_space_id}/source-connections' } from './knowledge-fs/errors.gen.ts' export { post23Errors as 'POST /agent' } from './agent/errors.gen.ts' -export { post23Errors as 'POST /apps/{app_id}/convert-to-workflow' } from './apps/errors.gen.ts' export { post23Errors as 'POST /datasets/{resource_id}/api-keys' } from './datasets/errors.gen.ts' export { post23Errors as 'POST /knowledge-fs/spaces/{control_space_id}/source-workflows/{run_id}/cancel' } from './knowledge-fs/errors.gen.ts' export { post24Errors as 'POST /datasets' } from './datasets/errors.gen.ts' @@ -427,7 +416,6 @@ export { post3Errors as 'POST /trial-apps/{app_id}/chat-messages' } from './tria export { post3Errors as 'POST /workflow-generate' } from './workflow-generate/errors.gen.ts' export { post40Errors as 'POST /apps/{app_id}/trigger-enable' } from './apps/errors.gen.ts' export { post40Errors as 'POST /knowledge-fs/spaces' } from './knowledge-fs/errors.gen.ts' -export { post41Errors as 'POST /apps/{app_id}/workflow-runs/tasks/{task_id}/stop' } from './apps/errors.gen.ts' export { post41Errors as 'POST /knowledge-fs/tasks/{task_id}/stream-capability' } from './knowledge-fs/errors.gen.ts' export { post41Errors as 'POST /workspaces/current/plugin/install/github' } from './workspaces/errors.gen.ts' export { post42Errors as 'POST /apps/{app_id}/workflow-runs/{workflow_run_id}/agent-nodes/{node_id}/sandbox/files/download' } from './apps/errors.gen.ts' @@ -440,41 +428,31 @@ export { post44Errors as 'POST /workspaces/current/plugin/list/installations/ids export { post45Errors as 'POST /apps/{app_id}/workflow/comments' } from './apps/errors.gen.ts' export { post45Errors as 'POST /workspaces/current/plugin/list/latest-versions' } from './workspaces/errors.gen.ts' export { post46Errors as 'POST /workspaces/current/plugin/parameters/dynamic-options-with-credentials' } from './workspaces/errors.gen.ts' -export { post48Errors as 'POST /apps/{app_id}/workflows/draft/features' } from './apps/errors.gen.ts' export { post48Errors as 'POST /workspaces/current/plugin/tasks/delete_all' } from './workspaces/errors.gen.ts' -export { post49Errors as 'POST /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/delivery-test' } from './apps/errors.gen.ts' export { post49Errors as 'POST /workspaces/current/plugin/tasks/{task_id}/delete/{identifier}' } from './workspaces/errors.gen.ts' export { post4Errors as 'POST /account/change-email' } from './account/errors.gen.ts' export { post4Errors as 'POST /agent/{agent_id}/build-chat/finalize' } from './agent/errors.gen.ts' -export { post4Errors as 'POST /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/preview' } from './apps/errors.gen.ts' export { post4Errors as 'POST /datasets/indexing-estimate' } from './datasets/errors.gen.ts' export { post4Errors as 'POST /installed-apps/{installed_app_id}/completion-messages/{task_id}/stop' } from './installed-apps/errors.gen.ts' export { post4Errors as 'POST /knowledge-fs/spaces/{control_space_id}/background-tasks/{task_kind}/{task_id}/cancel' } from './knowledge-fs/errors.gen.ts' export { post4Errors as 'POST /snippets/{snippet_id}/workflows/draft/nodes/{node_id}/agent-composer/copy-from-roster' } from './snippets/errors.gen.ts' export { post4Errors as 'POST /trial-apps/{app_id}/completion-messages' } from './trial-apps/errors.gen.ts' -export { post50Errors as 'POST /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/preview' } from './apps/errors.gen.ts' export { post50Errors as 'POST /workspaces/current/plugin/tasks/{task_id}/delete' } from './workspaces/errors.gen.ts' -export { post51Errors as 'POST /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/run' } from './apps/errors.gen.ts' export { post51Errors as 'POST /workspaces/current/plugin/uninstall' } from './workspaces/errors.gen.ts' export { post52Errors as 'POST /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/message-template/test' } from './apps/errors.gen.ts' export { post52Errors as 'POST /workspaces/current/plugin/upgrade/github' } from './workspaces/errors.gen.ts' -export { post53Errors as 'POST /apps/{app_id}/workflows/draft/iteration/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post53Errors as 'POST /workspaces/current/plugin/upgrade/marketplace' } from './workspaces/errors.gen.ts' -export { post54Errors as 'POST /apps/{app_id}/workflows/draft/loop/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post55Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/copy-from-roster' } from './apps/errors.gen.ts' export { post55Errors as 'POST /workspaces/current/plugin/upload/github' } from './workspaces/errors.gen.ts' export { post56Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/impact' } from './apps/errors.gen.ts' export { post56Errors as 'POST /workspaces/current/plugin/upload/pkg' } from './workspaces/errors.gen.ts' export { post57Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/save-to-roster' } from './apps/errors.gen.ts' export { post58Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/validate' } from './apps/errors.gen.ts' -export { post59Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post5Errors as 'POST /agent/{agent_id}/build-draft/apply' } from './agent/errors.gen.ts' -export { post5Errors as 'POST /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/run' } from './apps/errors.gen.ts' export { post5Errors as 'POST /datasets/init' } from './datasets/errors.gen.ts' export { post5Errors as 'POST /installed-apps/{installed_app_id}/completion-messages' } from './installed-apps/errors.gen.ts' export { post5Errors as 'POST /knowledge-fs/spaces/{control_space_id}/background-tasks/{task_kind}/{task_id}/retry' } from './knowledge-fs/errors.gen.ts' export { post5Errors as 'POST /trial-apps/{app_id}/files/upload' } from './trial-apps/errors.gen.ts' -export { post60Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/trigger/run' } from './apps/errors.gen.ts' export { post61Errors as 'POST /apps/{app_id}/workflows/draft/run' } from './apps/errors.gen.ts' export { post61Errors as 'POST /workspaces/current/skills/files/upload' } from './workspaces/errors.gen.ts' export { post62Errors as 'POST /apps/{app_id}/workflows/draft/trigger/run' } from './apps/errors.gen.ts' @@ -482,9 +460,7 @@ export { post62Errors as 'POST /workspaces/current/skills/import' } from './work export { post63Errors as 'POST /apps/{app_id}/workflows/draft/trigger/run-all' } from './apps/errors.gen.ts' export { post63Errors as 'POST /workspaces/current/skills/{skill_id}/assist/messages' } from './workspaces/errors.gen.ts' export { post64Errors as 'POST /apps/{app_id}/workflows/draft' } from './apps/errors.gen.ts' -export { post65Errors as 'POST /apps/{app_id}/workflows/publish' } from './apps/errors.gen.ts' export { post65Errors as 'POST /workspaces/current/skills/{skill_id}/files/check' } from './workspaces/errors.gen.ts' -export { post66Errors as 'POST /apps/{app_id}/workflows/{workflow_id}/restore' } from './apps/errors.gen.ts' export { post66Errors as 'POST /workspaces/current/skills/{skill_id}/publish' } from './workspaces/errors.gen.ts' export { post67Errors as 'POST /apps/{resource_id}/api-keys' } from './apps/errors.gen.ts' export { post67Errors as 'POST /workspaces/current/skills/{skill_id}/restore' } from './workspaces/errors.gen.ts' @@ -499,7 +475,6 @@ export { post6Errors as 'POST /snippets/{snippet_id}/workflows/draft/nodes/{node export { post6Errors as 'POST /trial-apps/{app_id}/remote-files/upload' } from './trial-apps/errors.gen.ts' export { post7Errors as 'POST /account/education' } from './account/errors.gen.ts' export { post7Errors as 'POST /agent/{agent_id}/chat-messages/{task_id}/stop' } from './agent/errors.gen.ts' -export { post7Errors as 'POST /apps/{app_id}/advanced-chat/workflows/draft/iteration/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post7Errors as 'POST /installed-apps/{installed_app_id}/messages/{message_id}/feedbacks' } from './installed-apps/errors.gen.ts' export { post7Errors as 'POST /knowledge-fs/spaces/{control_space_id}/documents' } from './knowledge-fs/errors.gen.ts' export { post7Errors as 'POST /rag/pipelines/{pipeline_id}/customized/publish' } from './rag/errors.gen.ts' @@ -507,7 +482,6 @@ export { post7Errors as 'POST /snippets/{snippet_id}/workflows/draft/nodes/{node export { post7Errors as 'POST /trial-apps/{app_id}/text-to-audio' } from './trial-apps/errors.gen.ts' export { post8Errors as 'POST /account/init' } from './account/errors.gen.ts' export { post8Errors as 'POST /agent/{agent_id}/composer/validate' } from './agent/errors.gen.ts' -export { post8Errors as 'POST /apps/{app_id}/advanced-chat/workflows/draft/loop/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post8Errors as 'POST /installed-apps/{installed_app_id}/saved-messages' } from './installed-apps/errors.gen.ts' export { post8Errors as 'POST /knowledge-fs/spaces/{control_space_id}/golden-questions/bulk-import' } from './knowledge-fs/errors.gen.ts' export { post8Errors as 'POST /rag/pipelines/{pipeline_id}/workflow-runs/tasks/{task_id}/stop' } from './rag/errors.gen.ts' diff --git a/packages/contracts/generated/api/console/rag/errors.gen.ts b/packages/contracts/generated/api/console/rag/errors.gen.ts index 501d339d7f2b47..6ba400dabbb13c 100644 --- a/packages/contracts/generated/api/console/rag/errors.gen.ts +++ b/packages/contracts/generated/api/console/rag/errors.gen.ts @@ -108,13 +108,10 @@ export const get20Errors = { export const post15Errors = { draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError }, - pipeline_not_found: { status: 404, data: zPipelineNotFoundError }, } export const get21Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } -export const post16Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - export const post17Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } export const post18Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } @@ -128,10 +125,6 @@ export const post19Errors = { rate_limit_error: { status: 429, data: zRateLimitError }, } -export const post20Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - -export const delete5Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - export const patch3Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } export const get24Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } diff --git a/packages/contracts/generated/api/console/rag/orpc.gen.ts b/packages/contracts/generated/api/console/rag/orpc.gen.ts index b266709612c4db..32abe153377753 100644 --- a/packages/contracts/generated/api/console/rag/orpc.gen.ts +++ b/packages/contracts/generated/api/console/rag/orpc.gen.ts @@ -1048,10 +1048,7 @@ export const post15 = oc }), ) .output(zPostRagPipelinesByPipelineIdWorkflowsDraftResponse) - .errors({ - draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError }, - pipeline_not_found: { status: 404, data: zPipelineNotFoundError }, - }) + .errors({ draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError } }) export const draft = { get: get20, @@ -1085,7 +1082,7 @@ export const get21 = oc .errors({ pipeline_not_found: { status: 404, data: zPipelineNotFoundError } }) /** - * Publish workflow + * Publish a prepared Pipeline and its dataset settings atomically */ export const post16 = oc .route({ @@ -1093,12 +1090,11 @@ export const post16 = oc method: 'POST', operationId: 'postRagPipelinesByPipelineIdWorkflowsPublish', path: '/rag/pipelines/{pipeline_id}/workflows/publish', - summary: 'Publish workflow', + summary: 'Publish a prepared Pipeline and its dataset settings atomically', tags: ['console'], }) .input(z.object({ params: zPostRagPipelinesByPipelineIdWorkflowsPublishPath })) .output(zPostRagPipelinesByPipelineIdWorkflowsPublishResponse) - .errors({ pipeline_not_found: { status: 404, data: zPipelineNotFoundError } }) export const publish2 = { get: get21, @@ -1271,15 +1267,11 @@ export const post20 = oc }) .input(z.object({ params: zPostRagPipelinesByPipelineIdWorkflowsByWorkflowIdRestorePath })) .output(zPostRagPipelinesByPipelineIdWorkflowsByWorkflowIdRestoreResponse) - .errors({ pipeline_not_found: { status: 404, data: zPipelineNotFoundError } }) export const restore = { post: post20, } -/** - * Delete a published workflow version that is not currently active on the pipeline - */ export const delete5 = oc .route({ inputStructure: 'detailed', @@ -1287,12 +1279,10 @@ export const delete5 = oc operationId: 'deleteRagPipelinesByPipelineIdWorkflowsByWorkflowId', path: '/rag/pipelines/{pipeline_id}/workflows/{workflow_id}', successStatus: 204, - summary: 'Delete a published workflow version that is not currently active on the pipeline', tags: ['console'], }) .input(z.object({ params: zDeleteRagPipelinesByPipelineIdWorkflowsByWorkflowIdPath })) .output(zDeleteRagPipelinesByPipelineIdWorkflowsByWorkflowIdResponse) - .errors({ pipeline_not_found: { status: 404, data: zPipelineNotFoundError } }) /** * Update workflow attributes diff --git a/packages/contracts/generated/api/console/rag/types.gen.ts b/packages/contracts/generated/api/console/rag/types.gen.ts index 11a7d9a559d684..ebaa560dc8f165 100644 --- a/packages/contracts/generated/api/console/rag/types.gen.ts +++ b/packages/contracts/generated/api/console/rag/types.gen.ts @@ -1549,7 +1549,6 @@ export type PostRagPipelinesByPipelineIdWorkflowsDraftData = { } export type PostRagPipelinesByPipelineIdWorkflowsDraftErrors = { - 404: PipelineNotFoundError | ErrorResponse 409: DraftWorkflowNotSyncError | ErrorResponse default: ErrorResponse } @@ -2066,7 +2065,6 @@ export type PostRagPipelinesByPipelineIdWorkflowsPublishData = { } export type PostRagPipelinesByPipelineIdWorkflowsPublishErrors = { - 404: PipelineNotFoundError | ErrorResponse default: ErrorResponse } @@ -2221,7 +2219,6 @@ export type DeleteRagPipelinesByPipelineIdWorkflowsByWorkflowIdData = { } export type DeleteRagPipelinesByPipelineIdWorkflowsByWorkflowIdErrors = { - 404: PipelineNotFoundError | ErrorResponse default: ErrorResponse } @@ -2273,7 +2270,6 @@ export type PostRagPipelinesByPipelineIdWorkflowsByWorkflowIdRestoreData = { } export type PostRagPipelinesByPipelineIdWorkflowsByWorkflowIdRestoreErrors = { - 404: PipelineNotFoundError | ErrorResponse default: ErrorResponse } diff --git a/packages/contracts/generated/api/console/snippets/orpc.gen.ts b/packages/contracts/generated/api/console/snippets/orpc.gen.ts index 635e7773a6928e..4d0b11051edc16 100644 --- a/packages/contracts/generated/api/console/snippets/orpc.gen.ts +++ b/packages/contracts/generated/api/console/snippets/orpc.gen.ts @@ -918,7 +918,7 @@ export const restore = { } /** - * Delete a published snippet workflow version + * Delete a version and retire its unowned Agents through the shared use case * * Delete a published snippet workflow version */ @@ -930,7 +930,7 @@ export const delete4 = oc operationId: 'deleteSnippetsBySnippetIdWorkflowsByWorkflowId', path: '/snippets/{snippet_id}/workflows/{workflow_id}', successStatus: 204, - summary: 'Delete a published snippet workflow version', + summary: 'Delete a version and retire its unowned Agents through the shared use case', tags: ['console'], }) .input(z.object({ params: zDeleteSnippetsBySnippetIdWorkflowsByWorkflowIdPath })) From 2396901e96c1df2c6d29ff44a12910bfd93dd038 Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Sat, 31 Oct 2026 22:22:15 +0800 Subject: [PATCH 16/20] Align migrated application fixtures with explicit runtime contracts --- .../console/explore/test_completion.py | 6 +-- .../console/explore/test_workflow.py | 4 +- .../core/app/apps/test_workflow_app_runner.py | 1 + .../nodes/human_input_v2/test_node.py | 45 ++++++++++++------- .../services/app/test_console_service.py | 4 +- .../test_form_delivery_runtime.py | 14 +++--- .../unit_tests/services/test_agent_service.py | 30 ++++++++++++- .../services/test_github_import_urls.py | 10 ++++- .../tools/test_builtin_provider_runtime.py | 17 ++++++- .../tasks/test_async_workflow_tasks.py | 3 +- 10 files changed, 98 insertions(+), 36 deletions(-) diff --git a/api/tests/unit_tests/controllers/console/explore/test_completion.py b/api/tests/unit_tests/controllers/console/explore/test_completion.py index dde8d5c0920d59..064eb703849434 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_completion.py +++ b/api/tests/unit_tests/controllers/console/explore/test_completion.py @@ -15,13 +15,13 @@ from models.enums import ConversationFromSource from repositories.installed_app_repository import SQLAlchemyInstalledAppRepository from services.account_errors import AccountNotFoundError +from services.app.generation.runtime import AppGenerationRuntime from services.app_generate_service import AppGenerateService from services.errors.app_model_config import AppModelConfigBrokenError from services.errors.conversation import ConversationCompletedError, ConversationNotExistsError from services.errors.llm import InvokeRateLimitError from services.installed_app_generation_adapters import AppGenerateServiceRuntime from services.installed_app_generation_service import GenerationResponse, InstalledAppGenerationService -from services.workflow.execution.ports import WorkflowRuntime from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.unit_tests.controllers.console.explore.test_installed_app_admission import ( _assert_json_response, @@ -428,7 +428,7 @@ def test_resource_removed_after_admission_is_revalidated_before_generation( deleted_resource: str, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ) -> None: def delete_resource() -> None: with sqlite_session_factory.begin() as session: @@ -817,7 +817,7 @@ def test_chat_conversation_preflight_returns_404_before_starting_generation_and_ rejection: str, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ) -> None: conversation_id = str(uuid4()) with sqlite_session_factory.begin() as session: diff --git a/api/tests/unit_tests/controllers/console/explore/test_workflow.py b/api/tests/unit_tests/controllers/console/explore/test_workflow.py index 10a32bdfd9397a..43dd82e027aa26 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_workflow.py +++ b/api/tests/unit_tests/controllers/console/explore/test_workflow.py @@ -19,13 +19,13 @@ from models.workflow import WorkflowType as ModelWorkflowType from repositories.installed_app_repository import SQLAlchemyInstalledAppRepository from services.account_errors import AccountNotFoundError +from services.app.generation.runtime import AppGenerationRuntime from services.app_definition_query_service import AppDefinitionUnavailableError from services.app_generate_service import AppGenerateService from services.app_task_service import AppTaskControlService from services.errors.llm import InvokeRateLimitError from services.installed_app_generation_adapters import AppGenerateServiceRuntime from services.installed_app_generation_service import GenerationResponse, InstalledAppGenerationService -from services.workflow.execution.ports import WorkflowRuntime from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.unit_tests.controllers.console.explore.test_installed_app_admission import ( _assert_json_response, @@ -416,7 +416,7 @@ def test_workflow_run_revalidates_app_while_stop_uses_admission_snapshot( action: str, *, workflow_variables: WorkflowExecutionVariables, - workflow_runtime: WorkflowRuntime, + workflow_runtime: AppGenerationRuntime, ) -> None: def remove_app() -> None: with sqlite_session_factory.begin() as session: diff --git a/api/tests/unit_tests/core/app/apps/test_workflow_app_runner.py b/api/tests/unit_tests/core/app/apps/test_workflow_app_runner.py index a4d9f2f81b257f..ef46019cbfb67d 100644 --- a/api/tests/unit_tests/core/app/apps/test_workflow_app_runner.py +++ b/api/tests/unit_tests/core/app/apps/test_workflow_app_runner.py @@ -1430,6 +1430,7 @@ def _generate_entity() -> WorkflowAppGenerateEntity: inputs={}, files=[], user_id="user", + user_from=UserFrom.ACCOUNT, stream=True, invoke_from=InvokeFrom.DEBUGGER, workflow_execution_id="run-1", diff --git a/api/tests/unit_tests/core/workflow/nodes/human_input_v2/test_node.py b/api/tests/unit_tests/core/workflow/nodes/human_input_v2/test_node.py index 9b9199c67494b4..58680087c4f9e3 100644 --- a/api/tests/unit_tests/core/workflow/nodes/human_input_v2/test_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/human_input_v2/test_node.py @@ -6,13 +6,11 @@ import pytest from pydantic import SecretStr, TypeAdapter -from core.app.apps.workflow_app_runner import WorkflowBasedAppRunner from core.app.entities.queue_entities import ( QueueHumanInputFormFilledEvent, QueueHumanInputFormTimeoutEvent, QueueNodeSucceededEvent, ) -from enums.human_input import ButtonStyle from core.human_input_v2.resolved_form import ( FileInput, FileListInput, @@ -32,8 +30,7 @@ from core.workflow.nodes.human_input_v2.events import NodeRunHumanInputV2FormRequiredEvent from core.workflow.nodes.human_input_v2.node import HumanInputNode from core.workflow.nodes.human_input_v2.runtime import HumanInputDeliveryError, HumanInputRuntime, PreparedForm -from core.workflow.workflow_entry import WorkflowEntry -from enums.human_input import HumanInputFormKind, HumanInputFormStatus, ValueSourceType +from enums.human_input import ButtonStyle, HumanInputFormKind, HumanInputFormStatus, ValueSourceType from graphon.engine_events import ( EngineEvent, NodeRunFailedEvent, @@ -56,6 +53,10 @@ StringSource, ) from repositories.human_input_v2.form_repository import Form, FormSubmission +from services.tools.provider_queries import ToolProviderIcons +from services.workflow.execution.adapters.events import WorkflowEventPublisher +from services.workflow.execution.adapters.workflow_entry import WorkflowEntry +from services.workflow.execution.ports import ExecutionContexts from tests.unit_tests.core.app.apps.common.test_workflow_response_converter import ( _workflow_response_converter_human_input_build_converter as _build_converter, ) @@ -147,10 +148,10 @@ def test_v1_event_filter_preserves_v2_completion_without_duplicate_events( form: Form, status: HumanInputFormStatus ) -> None: from core.repositories.human_input_repository import HumanInputFormSubmissionRepository - from core.workflow.nodes.human_input.boundary import HumanInputFormEventFilter from graphon.engine.filter import EngineEventFilterContext, filter_engine_events from graphon.graph import Graph from graphon.runtime import ReadOnlyRuntimeStateWrapper + from services.workflow.execution.adapters.human_input_events import HumanInputFormEventFilter form = submit(form) if status == HumanInputFormStatus.SUBMITTED else replace(form, status=status) node, _ = build_node(form) @@ -270,16 +271,23 @@ def test_reentry_consumes_the_new_persisted_outcome(form: Form) -> None: @pytest.mark.parametrize("status", [HumanInputFormStatus.SUBMITTED, HumanInputFormStatus.TIMEOUT]) -def test_actual_graph_queue_response_path_preserves_order_and_fields(form: Form, status: HumanInputFormStatus) -> None: +def test_actual_graph_queue_response_path_preserves_order_and_fields( + form: Form, + status: HumanInputFormStatus, + *, + workflow_contexts: ExecutionContexts, + tool_providers: ToolProviderIcons, +) -> None: form = submit(form) if status == HumanInputFormStatus.SUBMITTED else replace(form, status=status) node, _ = build_node(form) - runner = create_autospec(WorkflowBasedAppRunner, instance=True) - entry = WorkflowEntry({}, Graph(root_node=node), 0, node.runtime_state) + queue_manager = MagicMock() + publisher = WorkflowEventPublisher(queue_manager, notify_pause=MagicMock()) + entry = WorkflowEntry({}, Graph(root_node=node), 0, node.runtime_state, human_form_reader=MagicMock()) for event in list(engine_events(node))[1:]: - WorkflowBasedAppRunner.handle_event(runner, entry, event) - queue_events = [call.args[0] for call in runner._publish_event.call_args_list] + publisher.publish(entry, event) + queue_events = [call.args[0] for call in queue_manager.publish.call_args_list] assert isinstance(queue_events[1], QueueNodeSucceededEvent) - converter = _build_converter() + converter = _build_converter(workflow_contexts=workflow_contexts, tool_providers=tool_providers) converter.workflow_start_to_stream_response( task_id="task-1", workflow_run_id="run-1", workflow_id="wf-1", reason=WorkflowStartReason.INITIAL ) @@ -336,7 +344,9 @@ def test_dify_projection_serializes_the_runtime_token_without_a_graph_reason(for } -def test_file_segments_survive_node_graph_queue_and_response(form: Form) -> None: +def test_file_segments_survive_node_graph_queue_and_response( + form: Form, *, workflow_contexts: ExecutionContexts, tool_providers: ToolProviderIcons +) -> None: attachment = File( file_id="file-1", file_type=FileType.DOCUMENT, @@ -370,14 +380,15 @@ def test_file_segments_survive_node_graph_queue_and_response(form: Form) -> None assert isinstance(graph_events[2], NodeRunSucceededEvent) assert graph_events[1].rendered_content == "Answer: yes; files: [1 files]; attachment: [file]" assert isinstance(graph_events[2].node_run_result.outputs["attachment"], FileSegment) - runner = create_autospec(WorkflowBasedAppRunner, instance=True) - WorkflowBasedAppRunner.handle_event( - runner, WorkflowEntry({}, Graph(root_node=node), 0, node.runtime_state), graph_events[1] + queue_manager = MagicMock() + publisher = WorkflowEventPublisher(queue_manager, notify_pause=MagicMock()) + publisher.publish( + WorkflowEntry({}, Graph(root_node=node), 0, node.runtime_state, human_form_reader=MagicMock()), graph_events[1] ) - queue_event = runner._publish_event.call_args.args[0] + queue_event = queue_manager.publish.call_args.args[0] assert isinstance(queue_event.submitted_data["attachment"], FileSegment) assert isinstance(queue_event.submitted_data["files"], ArrayFileSegment) - converter = _build_converter() + converter = _build_converter(workflow_contexts=workflow_contexts, tool_providers=tool_providers) converter.workflow_start_to_stream_response( task_id="task-1", workflow_run_id="run-1", workflow_id="wf-1", reason=WorkflowStartReason.INITIAL ) diff --git a/api/tests/unit_tests/services/app/test_console_service.py b/api/tests/unit_tests/services/app/test_console_service.py index e9f473eb4614b9..7cc8f8603f5b9e 100644 --- a/api/tests/unit_tests/services/app/test_console_service.py +++ b/api/tests/unit_tests/services/app/test_console_service.py @@ -25,6 +25,7 @@ from core.tools.entities.tool_entities import ApiProviderSchemaType, ToolProviderType from events.app_event import app_was_updated from extensions.application_services.app import AppServices +from extensions.application_services.workflow import build_app_dsl_service from extensions.ext_redis import redis_client from graphon.model_runtime.entities.model_entities import ModelType from libs.datetime_utils import naive_utc_now @@ -59,7 +60,6 @@ CreatorsPlatformDisabledError, InvalidAppExportError, ) -from services.app_dsl_service import AppDslService from services.app_package_service import AppPackageService from services.app_tracing_config_service import AppTraceSettings from services.entities.app_entities import ( @@ -1645,7 +1645,7 @@ def test_planning_new_workflow_tools_checks_tool_management_under_rbac( rbac_domain.transport.response = {"allowed": allowed} gateway = AppTransferGateway( session_factory=sqlite_session_factory, - dsl_factory=AppDslService, + dsl_factory=build_app_dsl_service, packages=AppPackageService(), agent_packages=RosterAgentPackageExporter(), agent_importer=RosterAgentPackageImporter(), diff --git a/api/tests/unit_tests/services/human_input_v2/test_form_delivery_runtime.py b/api/tests/unit_tests/services/human_input_v2/test_form_delivery_runtime.py index 5f77efa55f30ba..90bee38bd4960f 100644 --- a/api/tests/unit_tests/services/human_input_v2/test_form_delivery_runtime.py +++ b/api/tests/unit_tests/services/human_input_v2/test_form_delivery_runtime.py @@ -79,6 +79,8 @@ FormExecutionContext, HumanInputFormService, ) +from services.tools.provider_queries import ToolProviderIcons +from services.workflow.execution.ports import ExecutionContexts _TENANT = TenantId("00000000-0000-0000-0000-000000000001") _APP = "00000000-0000-0000-0000-000000000002" @@ -235,11 +237,11 @@ def test_factory_runs_v2_and_reuses_persisted_approval( ) -> None: from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, build_dify_run_context from core.db.session_factory import session_factory - from core.workflow.node_factory import DifyNodeFactory from core.workflow.nodes.human_input_v2.events import NodeRunHumanInputV2FormRequiredEvent from core.workflow.system_variables import SystemVariableKey, system_variable_selector from graphon.engine_events import NodeRunPauseRequestedEvent from graphon.runtime import InitParams, RuntimeState + from services.workflow.execution.adapters.node_factory import DifyNodeFactory monkeypatch.setattr(session_factory, "create_session", context.sessions) pool = VariablePool() @@ -1521,8 +1523,8 @@ def test_pause_persistence_and_reconnect_preserve_v2_form_and_initiator_only_tok monkeypatch: pytest.MonkeyPatch, recipient: Initiator | OnetimeEmail, *, - workflow_contexts, - tool_providers, + workflow_contexts: ExecutionContexts, + tool_providers: ToolProviderIcons, ) -> None: from core.app.entities.queue_entities import QueueWorkflowPausedEvent from core.app.entities.task_entities import ( @@ -1746,7 +1748,7 @@ def test_pause_snapshot_resolves_mixed_versions_in_checkpoint_order( @pytest.mark.parametrize("initiator", [False, True]) @pytest.mark.usefixtures("providers") def test_console_pause_details_follow_checkpoint_with_mixed_form_versions( - context: Context, monkeypatch: pytest.MonkeyPatch, initiator: bool, *, tool_providers + context: Context, monkeypatch: pytest.MonkeyPatch, initiator: bool, *, tool_providers: ToolProviderIcons ) -> None: from core.app.layers.pause_state_persist_layer import WorkflowResumptionContext from core.workflow.nodes.human_input.pause_reason import HumanInputRequired @@ -1848,7 +1850,7 @@ def test_console_pause_details_follow_checkpoint_with_mixed_form_versions( @pytest.mark.parametrize("checkpoint", ["invalid", "missing_form"]) def test_console_pause_details_do_not_fall_back_when_checkpoint_is_invalid( - context: Context, monkeypatch: pytest.MonkeyPatch, checkpoint: str, *, tool_providers + context: Context, monkeypatch: pytest.MonkeyPatch, checkpoint: str, *, tool_providers: ToolProviderIcons ) -> None: from machinery.context import RequestContext from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository @@ -3050,7 +3052,6 @@ def test_initial_delivery_results_control_graph_failure_and_approval( ) -> None: from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, build_dify_run_context from core.db.session_factory import session_factory - from core.workflow.node_factory import DifyNodeFactory from core.workflow.system_variables import SystemVariableKey, system_variable_selector from graphon.engine import Engine from graphon.engine.command import InMemoryChannel @@ -3069,6 +3070,7 @@ def test_initial_delivery_results_control_graph_failure_and_approval( from graphon.runtime import InitParams, RuntimeState from repositories.human_input_v2.delivery_attempt_repository import DeliveryStatus from repositories.human_input_v2.email_channel.ports import EmailProviderOperationError + from services.workflow.execution.adapters.node_factory import DifyNodeFactory monkeypatch.setattr(session_factory, "create_session", context.sessions) provider_accept = providers[1].on_send diff --git a/api/tests/unit_tests/services/test_agent_service.py b/api/tests/unit_tests/services/test_agent_service.py index ecb72c10202afc..161754324202f5 100644 --- a/api/tests/unit_tests/services/test_agent_service.py +++ b/api/tests/unit_tests/services/test_agent_service.py @@ -5,6 +5,7 @@ import json from dataclasses import dataclass, field from datetime import datetime +from typing import TypedDict, cast import pytest from sqlalchemy.orm import Session, sessionmaker @@ -21,6 +22,26 @@ _CONTEXT = RequestContext("request", None, "account-1", "tenant-1") +class _ToolCallResult(TypedDict): + tool_icon: object + + +class _IterationResult(TypedDict): + tool_calls: list[_ToolCallResult] + thought: str | None + + +class _MetaResult(TypedDict): + executor: str + total_tokens: int + iterations: int + + +class _AgentLogResult(TypedDict): + iterations: list[_IterationResult] + meta: _MetaResult + + class _Files: def resolve(self, *, tenant_id: str, files: list[dict[str, object]]) -> list[dict[str, object]]: assert tenant_id == "tenant-1" @@ -98,8 +119,13 @@ def _persist_log(session: Session, *, tool_still_configured: bool = False, end_u session.expire_all() -def _get_log(service: AgentLogService, *, conversation_id: str = "conversation-1", message_id: str = "message-1"): - return service.get(_CONTEXT, app_id="app-1", conversation_id=conversation_id, message_id=message_id) +def _get_log( + service: AgentLogService, *, conversation_id: str = "conversation-1", message_id: str = "message-1" +) -> _AgentLogResult: + return cast( + _AgentLogResult, + service.get(_CONTEXT, app_id="app-1", conversation_id=conversation_id, message_id=message_id), + ) @pytest.mark.parametrize("tool_still_configured", [False, True]) diff --git a/api/tests/unit_tests/services/test_github_import_urls.py b/api/tests/unit_tests/services/test_github_import_urls.py index c6cbd496fd55b0..ea5bfc5ce007ad 100644 --- a/api/tests/unit_tests/services/test_github_import_urls.py +++ b/api/tests/unit_tests/services/test_github_import_urls.py @@ -1,5 +1,7 @@ """Check URL selection at the legacy YAML import download boundary.""" +from unittest.mock import create_autospec + import httpx import pytest from sqlalchemy.orm import Session @@ -7,7 +9,9 @@ from core.file import remote_fetcher from models import Account from services.app_dsl_service import AppDslService +from services.entities.dsl_entities import AppDslOverwriteStore from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineDslService +from services.workflow.draft_service import WorkflowDraftService @pytest.mark.parametrize("importer", ["app", "rag-pipeline"]) @@ -46,7 +50,11 @@ def fetch(method: str, url: str, **_kwargs: object) -> httpx.Response: monkeypatch.setattr(remote_fetcher, "make_request", fetch) account = Account(name="Test Account", email="test@example.com") if importer == "app": - result = AppDslService(session=unbound_session).import_app( + result = AppDslService( + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ).import_app( account=account, import_mode="yaml-url", yaml_url=url ) else: diff --git a/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py b/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py index a8d9c9f1b42ff4..2437468f8515aa 100644 --- a/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py +++ b/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py @@ -16,17 +16,21 @@ from core.tools.entities.tool_entities import ToolProviderEntityWithPlugin, ToolProviderType from core.tools.errors import ToolProviderCredentialValidationError, ToolProviderNotFoundError from core.tools.plugin_tool.provider import PluginToolProviderController +from extensions.application_services import tools as tool_services_module from extensions.application_services.workflow import build_workflow_execution_dependencies from extensions.ext_database import db from models.account import Account from models.base import TypeBase from models.tools import BuiltinToolProvider, MCPToolProvider, ToolOAuthSystemClient, ToolOAuthTenantClient +from services.credentials import tool_runtime_adapters as credential_runtime_module +from services.credentials.tool_runtime_adapters import ProviderToolCredentialRuntime from services.entities.agent_tool_inner import AgentToolInvokeRequest from services.errors.agent_tool_inner import AgentToolInnerServiceError from services.tools.agent_invocation_gateway import AgentToolInvocationGateway from services.tools.builtin import credentials as credentials_module from services.tools.tool_manager import ToolManager from services.workflow.variable_contracts import WorkflowExecutionVariables +from tests.helpers.unit.credential_refresh import ThreadCredentialRefreshLock TENANT = "tenant" PROVIDER = "vendor/search/search" @@ -136,6 +140,7 @@ def test_selected_oauth_credential_releases_connections_across_full_invocation( sessions, global_session = databases credential_id = seed(sessions, oauth_client=client) runtime = build_workflow_execution_dependencies(sessions) + refresh_lock = ThreadCredentialRefreshLock() calls: list[str] = [] def released() -> None: @@ -202,9 +207,14 @@ def decrypt_system(params: str) -> dict[str, str]: return {"client": "client"} monkeypatch.setattr(ToolManager, "get_builtin_provider", lambda *_args: controller()) + monkeypatch.setattr(ToolManager, "_credential_resolver", tool_services_module.resolve_tool_credentials) + monkeypatch.setattr(tool_services_module, "get_session_maker", lambda: sessions) + monkeypatch.setattr(tool_services_module, "RedisCredentialRefreshLock", lambda **_kwargs: refresh_lock) monkeypatch.setattr(credentials_module, "create_provider_encrypter", encrypter) - monkeypatch.setattr(credentials_module, "runtime_check_credential_policy_compliance", policy) - monkeypatch.setattr(credentials_module.OAuthHandler, "refresh_credentials", refresh) + monkeypatch.setattr(credential_runtime_module, "create_provider_encrypter", encrypter) + monkeypatch.setattr(credential_runtime_module, "runtime_check_credential_policy_compliance", policy) + monkeypatch.setattr(credential_runtime_module.OAuthHandler, "refresh_credentials", refresh) + monkeypatch.setattr(ProviderToolCredentialRuntime, "_cache", lambda _self, _record: Cache()) monkeypatch.setattr(credentials_module.PluginService, "is_plugin_verified", verified) monkeypatch.setattr(credentials_module, "decrypt_system_params", decrypt_system) monkeypatch.setattr("core.plugin.impl.tool.PluginToolManager.invoke", invoke) @@ -238,6 +248,7 @@ def decrypt_system(params: str) -> dict[str, str]: gateway.invoke(request) assert exc.value.error_code == "agent_tool_credential_invalid" assert "invoke" not in calls + assert refresh_lock.keys == [("tool", TENANT, credential_id)] released() with sessions() as session: persisted = session.get(BuiltinToolProvider, credential_id) @@ -268,6 +279,8 @@ def test_selected_credential_cannot_cross_owner_scope( else: record.provider = "other/provider" monkeypatch.setattr(ToolManager, "get_builtin_provider", lambda *_args: controller()) + monkeypatch.setattr(ToolManager, "_credential_resolver", tool_services_module.resolve_tool_credentials) + monkeypatch.setattr(tool_services_module, "get_session_maker", lambda: sessions) runtime = build_workflow_execution_dependencies(sessions) with pytest.raises(ToolProviderCredentialValidationError, match="has been deleted"): ToolManager.get_tool_runtime( diff --git a/api/tests/unit_tests/tasks/test_async_workflow_tasks.py b/api/tests/unit_tests/tasks/test_async_workflow_tasks.py index b2038249b4f7a7..ccd582d9a89c0d 100644 --- a/api/tests/unit_tests/tasks/test_async_workflow_tasks.py +++ b/api/tests/unit_tests/tasks/test_async_workflow_tasks.py @@ -6,7 +6,7 @@ import pytest from core.app.app_config.entities import WorkflowUIBasedAppConfig -from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity +from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, WorkflowAppGenerateEntity from models.enums import AppTriggerType, WorkflowRunTriggeredFrom from models.model import AppMode from repositories.workflow.definition_repository import workflow_snapshot @@ -109,6 +109,7 @@ def test_resume_workflow_projects_the_persisted_workflow_before_resuming( inputs={}, files=[], user_id=user.id, + user_from=UserFrom.ACCOUNT, stream=False, invoke_from=InvokeFrom.WEB_APP, workflow_execution_id=workflow_run.id, From d844e78ee3a0199251131ca59f56bc097047041b Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Sat, 31 Oct 2026 22:26:02 +0800 Subject: [PATCH 17/20] Use canonical Human Input test owners after execution migration --- .../containers/controllers/openapi/test_app_dsl.py | 6 +----- .../core/workflow/nodes/human_input_v2/test_node.py | 10 ++++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/api/tests/e2e_tests/containers/controllers/openapi/test_app_dsl.py b/api/tests/e2e_tests/containers/controllers/openapi/test_app_dsl.py index 7bafbf445a4a36..d6bdb39c72cac9 100644 --- a/api/tests/e2e_tests/containers/controllers/openapi/test_app_dsl.py +++ b/api/tests/e2e_tests/containers/controllers/openapi/test_app_dsl.py @@ -53,7 +53,6 @@ def external_deps() -> Generator[dict[str, object]]: workflow sync, dependency analysis, enterprise hooks) while leaving the DSL service and DB writes real.""" with ( - patch("services.app_dsl_service.WorkflowService") as mock_workflow_service, patch("extensions.application_services.workflow.build_workflow_draft_service") as mock_drafts, patch("services.app_dsl_service.DependenciesAnalysisService") as mock_dependencies_service, patch("services.app_dsl_service.app_was_created") as mock_app_was_created, @@ -61,9 +60,6 @@ def external_deps() -> Generator[dict[str, object]]: patch("services.app_service.SystemFeatureService") as mock_feature_service, patch("services.app_service.EnterpriseService") as mock_enterprise_service, ): - mock_workflow_service.get_draft_workflow.return_value = None - mock_workflow_service.get_published_workflow_by_id.return_value = None - mock_workflow_service.return_value.get_draft_workflow.return_value = None mock_drafts.return_value.sync.return_value = None mock_dependencies_service.generate_latest_dependencies.return_value = [] # type: ignore[assignment] mock_dependencies_service.get_leaked_dependencies.return_value = [] # type: ignore[assignment] @@ -78,7 +74,7 @@ def external_deps() -> Generator[dict[str, object]]: mock_enterprise_service.WebAppAuth.update_app_access_mode.return_value = None mock_enterprise_service.WebAppAuth.cleanup_webapp.return_value = None - yield {"workflow_service": mock_workflow_service} + yield {} def _app_and_account(db_session: Session, *, mode: str = "chat") -> tuple[App, Account]: diff --git a/api/tests/unit_tests/core/workflow/nodes/human_input_v2/test_node.py b/api/tests/unit_tests/core/workflow/nodes/human_input_v2/test_node.py index 58680087c4f9e3..3c72c205a76f26 100644 --- a/api/tests/unit_tests/core/workflow/nodes/human_input_v2/test_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/human_input_v2/test_node.py @@ -147,10 +147,10 @@ def submit(form: Form) -> Form: def test_v1_event_filter_preserves_v2_completion_without_duplicate_events( form: Form, status: HumanInputFormStatus ) -> None: - from core.repositories.human_input_repository import HumanInputFormSubmissionRepository from graphon.engine.filter import EngineEventFilterContext, filter_engine_events from graphon.graph import Graph from graphon.runtime import ReadOnlyRuntimeStateWrapper + from repositories.human_input.form_repository import HumanInputFormSubmissionRepository from services.workflow.execution.adapters.human_input_events import HumanInputFormEventFilter form = submit(form) if status == HumanInputFormStatus.SUBMITTED else replace(form, status=status) @@ -312,12 +312,14 @@ def test_actual_graph_queue_response_path_preserves_order_and_fields( @pytest.mark.parametrize("token", ["initiator-token", None]) def test_dify_projection_serializes_the_runtime_token_without_a_graph_reason(form: Form, token: str | None) -> None: - from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter from core.app.entities.task_entities import HumanInputRequiredPauseReasonPayload + from services.workflow.execution.adapters.response_converter import WorkflowResponseConverter prepared = PreparedForm(form, SecretStr(token) if token else None) with patch( - "core.app.apps.common.workflow_response_converter.Session", side_effect=AssertionError("unexpected lookup") + "services.workflow.execution.adapters.response_converter.Session", + side_effect=AssertionError("unexpected lookup"), + create=True, ): data = WorkflowResponseConverter.human_input_v2_required_data(prepared, node_id="node-1") assert data.form_token == token @@ -441,7 +443,7 @@ def test_filled_rendering_skips_missing_and_null_inputs_and_preserves_select_tex def test_waiting_projection_freezes_select_options_and_deduplicates_slots(form: Form) -> None: - from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter + from services.workflow.execution.adapters.response_converter import WorkflowResponseConverter option = SelectInput(output_variable_name="choice", options=("old-a", "old-b"), default_value="old-b") form = replace(form, resolved_form=form.resolved_form.model_copy(update={"parts": (option, option)})) From 11852ad9ced5c5752ead71571a25cd6b74b76997 Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Sat, 31 Oct 2026 22:51:09 +0800 Subject: [PATCH 18/20] Format migrated test fixtures --- api/tests/unit_tests/services/test_github_import_urls.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/tests/unit_tests/services/test_github_import_urls.py b/api/tests/unit_tests/services/test_github_import_urls.py index ea5bfc5ce007ad..46875ce47ff045 100644 --- a/api/tests/unit_tests/services/test_github_import_urls.py +++ b/api/tests/unit_tests/services/test_github_import_urls.py @@ -54,9 +54,7 @@ def fetch(method: str, url: str, **_kwargs: object) -> httpx.Response: session=unbound_session, drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), - ).import_app( - account=account, import_mode="yaml-url", yaml_url=url - ) + ).import_app(account=account, import_mode="yaml-url", yaml_url=url) else: result = RagPipelineDslService(session=unbound_session).import_rag_pipeline( account=account, import_mode="yaml-url", yaml_url=url From be0f86a8c5d9763e01df91ee7a5eb30114f9d78a Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Sat, 31 Oct 2026 22:59:34 +0800 Subject: [PATCH 19/20] Use typed billing limit fixture for workflow publication --- .../repositories/workflow/test_definition_repository.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/tests/unit_tests/repositories/workflow/test_definition_repository.py b/api/tests/unit_tests/repositories/workflow/test_definition_repository.py index 547e8474dcbd8d..24fc69413e47b2 100644 --- a/api/tests/unit_tests/repositories/workflow/test_definition_repository.py +++ b/api/tests/unit_tests/repositories/workflow/test_definition_repository.py @@ -72,6 +72,7 @@ from services.workflow.draft_service import DraftAgentRetirement, WorkflowDraftService from services.workflow.environment_variable_service import prepare_environment_variables from services.workflow_service import WorkflowService +from tests.helpers import billing_builders as builders from tests.helpers.unit.model_factories import make_account, make_app, make_tenant, make_workflow CONTEXT = RequestContext("request", None, "account-1", "tenant-1") @@ -1078,7 +1079,7 @@ def test_publication_retains_blocking_domain_validations( else: config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) monkeypatch.setattr( - workflow_module.BillingService, "get_info", lambda *_args: {"subscription": {"plan": "sandbox"}} + workflow_module.BillingService, "get_info", lambda *_args: builders.subscription_info("sandbox") ) draft.graph = json.dumps( {"nodes": [{"id": str(index), "data": {"type": "trigger-webhook"}} for index in range(3)], "edges": []} From 8e19bdd637a0e0fefe66f7d44f805cdd8fb4a68c Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Sat, 31 Oct 2026 23:02:57 +0800 Subject: [PATCH 20/20] Expose workflow variables fixture to Service API contract tests --- .../e2e_tests/contracts/controllers/service_api/test_conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/tests/e2e_tests/contracts/controllers/service_api/test_conftest.py b/api/tests/e2e_tests/contracts/controllers/service_api/test_conftest.py index f26ba315481f07..5b7027f9f74eb3 100644 --- a/api/tests/e2e_tests/contracts/controllers/service_api/test_conftest.py +++ b/api/tests/e2e_tests/contracts/controllers/service_api/test_conftest.py @@ -9,6 +9,7 @@ persist_service_api_dataset_owner, persist_service_api_tenant_owner, ) +from tests.unit_tests.conftest import workflow_variables as workflow_variables # noqa: PLC0414 from tests.unit_tests.controllers.conftest import app_query_services as app_query_services # noqa: PLC0414 from tests.unit_tests.controllers.conftest import ( datasource_application_dependencies as datasource_application_dependencies, # noqa: PLC0414