diff --git a/apps/web/i18n/messages/en-US/errors.json b/apps/web/i18n/messages/en-US/errors.json index 01ebb386c..39a35630e 100644 --- a/apps/web/i18n/messages/en-US/errors.json +++ b/apps/web/i18n/messages/en-US/errors.json @@ -268,6 +268,12 @@ "modelCatalogFetchFailed": "Failed to sync models.dev. The last synced data is kept." }, "mcp": { + "fileCreate": { + "unauthorized": "The file creation capability is invalid, expired, or revoked.", + "conflict": "This capability is bound to another file, or existing content is unavailable.", + "resultGone": "The original file creation result is no longer available.", + "invalidInput": "Exactly one bounded file is required." + }, "clientUnavailable": "MCP client is unavailable.", "imageProcessingFailed": "The MCP image processing tool failed.", "invalidAttachmentConfiguration": "Invalid MCP tool attachment configuration.", diff --git a/apps/web/i18n/messages/zh-CN/errors.json b/apps/web/i18n/messages/zh-CN/errors.json index 2a0677640..3d4ea7723 100644 --- a/apps/web/i18n/messages/zh-CN/errors.json +++ b/apps/web/i18n/messages/zh-CN/errors.json @@ -268,6 +268,12 @@ "modelCatalogFetchFailed": "同步 models.dev 失败，已保留上次同步的数据。" }, "mcp": { + "fileCreate": { + "unauthorized": "文件创建授权无效、已过期或已撤销。", + "conflict": "此授权已绑定其他文件，或已有内容不可用。", + "resultGone": "原文件创建结果已不可用。", + "invalidInput": "必须且只能提交一个符合大小限制的文件。" + }, "clientUnavailable": "MCP 客户端不可用。", "imageProcessingFailed": "MCP 图片处理工具执行失败。", "invalidAttachmentConfiguration": "MCP 工具附件配置无效。", diff --git a/backend/README.md b/backend/README.md index 0616254b3..a55ebfa34 100644 --- a/backend/README.md +++ b/backend/README.md @@ -55,6 +55,7 @@ backend/ - [项目主 README](../README.md) - [前端 README](../apps/web/README.md) - `docs/README.md`：后端文档索引 +- [MCP 文件创建委托（Draft 评审契约）](../docs/MCP_FILE_CREATE.md)：默认关闭、每次工具调用一文件的受限上传能力 - `docs/swagger.json` / `docs/swagger.yaml`：Swagger API 文档 ## 核心约束 diff --git a/backend/docs/docs.go b/backend/docs/docs.go index 4e0ed54f4..b840035ca 100644 --- a/backend/docs/docs.go +++ b/backend/docs/docs.go @@ -14765,6 +14765,94 @@ const docTemplate = `{ } } }, + "/mcp/servers/{id}/files": { + "post": { + "description": "仅接受 DEEIX 在 tools/call 签发的一文件 capability，不接受用户 JWT 或 MCP 共享 HMAC 上下文。重复相同文件返回同一 file_id；创建不代表提取或向量化已完成。", + "consumes": [ + "multipart/form-data" + ], + "produces": [ + "application/json" + ], + "tags": [ + "mcp" + ], + "summary": "创建 MCP 工具产物文件", + "parameters": [ + { + "type": "integer", + "description": "登记的 MCP 服务 ID", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Bearer \u003cDEEIX-issued file-create token\u003e", + "name": "Authorization", + "in": "header", + "required": true + }, + { + "type": "file", + "description": "唯一文件，不接受其他字段", + "name": "file", + "in": "formData", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/FileCreateResponseDoc" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + }, + "429": { + "description": "Too Many Requests", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + } + } + } + }, "/mcp/tools": { "get": { "security": [ @@ -22763,6 +22851,10 @@ const docTemplate = `{ "baseURL": { "type": "string" }, + "fileCreateEnabled": { + "description": "FileCreateEnabled authorizes automatic creation of one user file per persistent tool call.", + "type": "boolean" + }, "headersJSON": { "type": "string" }, @@ -23727,6 +23819,68 @@ const docTemplate = `{ } } }, + "FileCreateResponse": { + "type": "object", + "required": [ + "embed_status", + "extract_status", + "file_id", + "file_name", + "processing_ready", + "processing_status", + "replayed", + "reused", + "sha256", + "size_bytes" + ], + "properties": { + "embed_status": { + "type": "string" + }, + "extract_status": { + "type": "string" + }, + "file_id": { + "type": "string" + }, + "file_name": { + "type": "string" + }, + "processing_ready": { + "type": "boolean" + }, + "processing_status": { + "type": "string" + }, + "replayed": { + "type": "boolean" + }, + "reused": { + "type": "boolean" + }, + "sha256": { + "type": "string" + }, + "size_bytes": { + "type": "integer" + } + } + }, + "FileCreateResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/FileCreateResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, "FileEmbeddingSkipResponse": { "type": "object", "required": [ @@ -31062,6 +31216,7 @@ const docTemplate = `{ "activeToolCount", "baseURL", "createdAt", + "fileCreateEnabled", "headersJSON", "id", "lastError", @@ -31083,6 +31238,9 @@ const docTemplate = `{ "createdAt": { "type": "string" }, + "fileCreateEnabled": { + "type": "boolean" + }, "headersJSON": { "type": "string" }, diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index 17f5622c6..d44ba1d4a 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -14758,6 +14758,94 @@ } } }, + "/mcp/servers/{id}/files": { + "post": { + "description": "仅接受 DEEIX 在 tools/call 签发的一文件 capability，不接受用户 JWT 或 MCP 共享 HMAC 上下文。重复相同文件返回同一 file_id；创建不代表提取或向量化已完成。", + "consumes": [ + "multipart/form-data" + ], + "produces": [ + "application/json" + ], + "tags": [ + "mcp" + ], + "summary": "创建 MCP 工具产物文件", + "parameters": [ + { + "type": "integer", + "description": "登记的 MCP 服务 ID", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Bearer ", + "name": "Authorization", + "in": "header", + "required": true + }, + { + "type": "file", + "description": "唯一文件，不接受其他字段", + "name": "file", + "in": "formData", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/FileCreateResponseDoc" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + }, + "429": { + "description": "Too Many Requests", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/McpErrorDoc" + } + } + } + } + }, "/mcp/tools": { "get": { "security": [ @@ -22756,6 +22844,10 @@ "baseURL": { "type": "string" }, + "fileCreateEnabled": { + "description": "FileCreateEnabled authorizes automatic creation of one user file per persistent tool call.", + "type": "boolean" + }, "headersJSON": { "type": "string" }, @@ -23720,6 +23812,68 @@ } } }, + "FileCreateResponse": { + "type": "object", + "required": [ + "embed_status", + "extract_status", + "file_id", + "file_name", + "processing_ready", + "processing_status", + "replayed", + "reused", + "sha256", + "size_bytes" + ], + "properties": { + "embed_status": { + "type": "string" + }, + "extract_status": { + "type": "string" + }, + "file_id": { + "type": "string" + }, + "file_name": { + "type": "string" + }, + "processing_ready": { + "type": "boolean" + }, + "processing_status": { + "type": "string" + }, + "replayed": { + "type": "boolean" + }, + "reused": { + "type": "boolean" + }, + "sha256": { + "type": "string" + }, + "size_bytes": { + "type": "integer" + } + } + }, + "FileCreateResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/FileCreateResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, "FileEmbeddingSkipResponse": { "type": "object", "required": [ @@ -31055,6 +31209,7 @@ "activeToolCount", "baseURL", "createdAt", + "fileCreateEnabled", "headersJSON", "id", "lastError", @@ -31076,6 +31231,9 @@ "createdAt": { "type": "string" }, + "fileCreateEnabled": { + "type": "boolean" + }, "headersJSON": { "type": "string" }, diff --git a/backend/docs/swagger.yaml b/backend/docs/swagger.yaml index 57ce68b53..a79b193ce 100644 --- a/backend/docs/swagger.yaml +++ b/backend/docs/swagger.yaml @@ -3189,6 +3189,10 @@ definitions: type: string baseURL: type: string + fileCreateEnabled: + description: FileCreateEnabled authorizes automatic creation of one user file + per persistent tool call. + type: boolean headersJSON: type: string name: @@ -3865,6 +3869,50 @@ definitions: - data - errorMsg type: object + FileCreateResponse: + properties: + embed_status: + type: string + extract_status: + type: string + file_id: + type: string + file_name: + type: string + processing_ready: + type: boolean + processing_status: + type: string + replayed: + type: boolean + reused: + type: boolean + sha256: + type: string + size_bytes: + type: integer + required: + - embed_status + - extract_status + - file_id + - file_name + - processing_ready + - processing_status + - replayed + - reused + - sha256 + - size_bytes + type: object + FileCreateResponseDoc: + properties: + data: + $ref: '#/definitions/FileCreateResponse' + errorMsg: + type: string + required: + - data + - errorMsg + type: object FileEmbeddingSkipResponse: properties: fileID: @@ -9079,6 +9127,8 @@ definitions: type: string createdAt: type: string + fileCreateEnabled: + type: boolean headersJSON: type: string id: @@ -9105,6 +9155,7 @@ definitions: - activeToolCount - baseURL - createdAt + - fileCreateEnabled - headersJSON - id - lastError @@ -21483,6 +21534,66 @@ paths: summary: 读取模型展示图标 tags: - llm + /mcp/servers/{id}/files: + post: + consumes: + - multipart/form-data + description: 仅接受 DEEIX 在 tools/call 签发的一文件 capability，不接受用户 JWT 或 MCP 共享 HMAC + 上下文。重复相同文件返回同一 file_id；创建不代表提取或向量化已完成。 + parameters: + - description: 登记的 MCP 服务 ID + in: path + name: id + required: true + type: integer + - description: Bearer + in: header + name: Authorization + required: true + type: string + - description: 唯一文件，不接受其他字段 + in: formData + name: file + required: true + type: file + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/FileCreateResponseDoc' + "400": + description: Bad Request + schema: + $ref: '#/definitions/McpErrorDoc' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/McpErrorDoc' + "404": + description: Not Found + schema: + $ref: '#/definitions/McpErrorDoc' + "409": + description: Conflict + schema: + $ref: '#/definitions/McpErrorDoc' + "413": + description: Request Entity Too Large + schema: + $ref: '#/definitions/McpErrorDoc' + "429": + description: Too Many Requests + schema: + $ref: '#/definitions/McpErrorDoc' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/McpErrorDoc' + summary: 创建 MCP 工具产物文件 + tags: + - mcp /mcp/tools: get: description: 获取当前聊天侧可选择的 MCP 工具 diff --git a/backend/internal/app/app.go b/backend/internal/app/app.go index ac9d2d1cb..d58e58611 100644 --- a/backend/internal/app/app.go +++ b/backend/internal/app/app.go @@ -426,7 +426,7 @@ func NewAppWithOptions(opts Options) (*App, error) { runtimeCfg, conversationRepo, log, - appupload.Hooks{InitializeUploadedFile: processingService.InitializeUploadedFile}, + appupload.Hooks{InitializeUploadedFile: processingService.InitializeUploadedFile, EnsureFileCreateProcessing: processingService.EnsureFileCreateProcessing}, conversation.UploadErrorSet(), appprocessing.DefaultExtractorVersion, ) @@ -481,6 +481,9 @@ func NewAppWithOptions(opts Options) (*App, error) { mcpService := appmcp.NewServiceWithRuntime(runtimeCfg, mcpRepo, mcpClient) mcpService.SetBillingModeProvider(billingService) mcpHandler := mcphttp.NewHandler(mcpService) + fileCreateService := appmcp.NewFileCreateService(runtimeCfg, mcpRepo, conversationRepo, uploadService, auditService, log) + conversationService.SetMCPFileCreateService(fileCreateService) + mcpHandler.SetFileCreateService(fileCreateService) mcpModule := mcphttp.NewModule(mcpHandler) adminService := admin.NewService(userService, auditService) adminService.SetAuthSecurityService(authService) @@ -583,6 +586,7 @@ func NewAppWithOptions(opts Options) (*App, error) { } embeddingService.StartBackgroundWorkers(backgroundCtx) conversationService.StartBackgroundWorkers(backgroundCtx) + fileCreateService.StartMaintenance(backgroundCtx) contentModerationService.StartBackgroundWorkers(backgroundCtx) channelService.StartModelIconAssetCleanup(backgroundCtx) channelService.LoadModelCatalog(backgroundCtx) diff --git a/backend/internal/application/conversation/service.go b/backend/internal/application/conversation/service.go index 396cb8031..1678d1988 100644 --- a/backend/internal/application/conversation/service.go +++ b/backend/internal/application/conversation/service.go @@ -13,6 +13,7 @@ import ( appcm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/contentmoderation" appembedding "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/embedding" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/extraction" + appmcp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/mcp" appstorage "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/objectstorage" appprocessing "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/processing" apprag "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/rag" @@ -131,6 +132,7 @@ type Service struct { routeResolver routeResolver memoryRecorder memoryRecorder mcpRepo mcpToolResolver + mcpFileCreate *appmcp.FileCreateService llmClient llmGateway mediaDownloader generatedMediaDownloader // untrustedMediaDownloader 下载用户自带 Key 端点返回的制品：始终只允许公网目标，端点不放宽出站策略。 @@ -384,3 +386,7 @@ func (s *Service) SetObjectStoreProvider(provider appstorage.Provider) { func (s *Service) SetMCPRepository(repo mcpToolResolver) { s.mcpRepo = repo } + +func (s *Service) SetMCPFileCreateService(service *appmcp.FileCreateService) { + s.mcpFileCreate = service +} diff --git a/backend/internal/application/conversation/service_mcp_tools.go b/backend/internal/application/conversation/service_mcp_tools.go index 6db5e826a..1ba45fc5d 100644 --- a/backend/internal/application/conversation/service_mcp_tools.go +++ b/backend/internal/application/conversation/service_mcp_tools.go @@ -28,6 +28,7 @@ type selectedToolRuntime struct { type mcpToolCallBinding struct { Config mcp.CallConfig ServerID uint + ToolID uint ServerName string ToolName string PriceNanousd int64 @@ -150,12 +151,14 @@ func (s *Service) resolveSelectedToolRuntime(ctx context.Context, toolIDs []uint result.schemas[modelName] = schema result.mcpBindings[modelName] = mcpToolCallBinding{ Config: mcp.CallConfig{ - BaseURL: server.BaseURL, - AuthToken: token, - TimeoutMS: cfg.MCPToolTimeoutSeconds * 1000, - Headers: headers, + BaseURL: server.BaseURL, + AuthToken: token, + TimeoutMS: cfg.MCPToolTimeoutSeconds * 1000, + Headers: headers, + FileCreateEpoch: server.FileCreateEpoch, }, ServerID: server.ID, + ToolID: tool.ID, ServerName: server.Name, ToolName: tool.Name, PriceNanousd: tool.PriceNanousd, diff --git a/backend/internal/application/conversation/service_tool.go b/backend/internal/application/conversation/service_tool.go index ca8fd4631..52483ffe5 100644 --- a/backend/internal/application/conversation/service_tool.go +++ b/backend/internal/application/conversation/service_tool.go @@ -2,15 +2,18 @@ package conversation import ( "context" + "errors" "fmt" "strings" "time" "github.com/google/uuid" + appmcp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/mcp" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/config" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/pkg/mcpauth" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/ports/mcp" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/shared/toolresult" ) // ExecuteToolInput 定义工具执行入参。 @@ -21,6 +24,10 @@ type ExecuteToolInput struct { ToolName string ArgumentsJSON string MCPConfig *mcp.CallConfig + ServerID uint + ToolID uint + Ephemeral bool + CallID string } func (s *Service) executeToolCall(ctx context.Context, input ExecuteToolInput) (string, error) { @@ -35,24 +42,43 @@ func (s *Service) executeToolCall(ctx context.Context, input ExecuteToolInput) ( return "", fmt.Errorf("mcp client is not configured") } cfg := s.cfg.Snapshot() - mcpCfg, err := applySignedUserContext(cfg, *input.MCPConfig, input) - if err != nil { - return "", err - } - limit := cfg.MCPMaxConcurrentCalls if limit <= 0 { limit = 8 } return s.executeWithToolLimiter(ctx, limit, func() (string, error) { - return s.callMCPWithRetry(ctx, mcpCfg, mcp.CallInput{ + input.CallID = uuid.NewString() + mcpCfg, err := applySignedUserContext(cfg, *input.MCPConfig, input) + if err != nil { + return "", err + } + // No grants for temporary chats or attachment preprocessors. + mcpCfg.FileCreateToken = "" + if !input.Ephemeral && input.ServerID != 0 && s.mcpFileCreate != nil { + mcpCfg.FileCreateToken, err = s.mcpFileCreate.IssueFileCreateGrant(ctx, appmcp.FileCreateCall{ + ServerID: input.ServerID, ToolID: input.ToolID, UserID: input.UserID, + BaseURL: mcpCfg.BaseURL, CallID: input.CallID, RequestID: input.RequestID, + Epoch: mcpCfg.FileCreateEpoch, + }) + if err != nil { + return "", fmt.Errorf("MCP file capability issuance failed") + } + } + output, callErr := s.callMCPWithRetry(ctx, mcpCfg, mcp.CallInput{ ToolName: toolName, ArgumentsJSON: strings.TrimSpace(input.ArgumentsJSON), UserID: input.UserID, ConversationID: input.ConversationID, RequestID: strings.TrimSpace(input.RequestID), }, cfg.MCPToolRetryCount) + if token := mcpCfg.FileCreateToken; token != "" { + output = toolresult.RedactSecret(output, token) + if callErr != nil { + callErr = errors.New(toolresult.RedactSecret(callErr.Error(), token)) + } + } + return output, callErr }) } @@ -81,7 +107,7 @@ func applySignedUserContext(cfg config.Config, base mcp.CallConfig, input Execut ConversationID: input.ConversationID, RequestID: strings.TrimSpace(input.RequestID), Audience: base.BaseURL, - JTI: uuid.NewString(), + JTI: toolCallJTI(input.CallID), ExpiresAt: time.Now().Add(mcpauth.DefaultTTL).Unix(), }) if err != nil || token == "" { @@ -99,6 +125,13 @@ func applySignedUserContext(cfg config.Config, base mcp.CallConfig, input Execut return base, nil } +func toolCallJTI(callID string) string { + if callID != "" { + return callID + } + return uuid.NewString() +} + func (s *Service) resolveMaxToolCallsPerRun() int { maxCalls := s.cfg.Snapshot().MCPMaxToolCallsPerRun if maxCalls <= 0 { diff --git a/backend/internal/application/conversation/service_tool_execution.go b/backend/internal/application/conversation/service_tool_execution.go index 1f6549d56..2cdea71ae 100644 --- a/backend/internal/application/conversation/service_tool_execution.go +++ b/backend/internal/application/conversation/service_tool_execution.go @@ -152,6 +152,9 @@ func (s *Service) executeAssistantToolCalls(ctx context.Context, input executeAs ToolName: row.ToolName, ArgumentsJSON: row.InputJSON, MCPConfig: &binding.Config, + ServerID: binding.ServerID, + ToolID: binding.ToolID, + Ephemeral: input.Ephemeral, }) row.LatencyMS = time.Since(toolStartedAt).Milliseconds() if row.LatencyMS < 0 { diff --git a/backend/internal/application/conversation/service_tool_test.go b/backend/internal/application/conversation/service_tool_test.go index 98e1efaf7..55e7d5b9d 100644 --- a/backend/internal/application/conversation/service_tool_test.go +++ b/backend/internal/application/conversation/service_tool_test.go @@ -12,11 +12,15 @@ import ( "testing" "time" + appmcp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/mcp" model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" + domainmcp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/mcp" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/config" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/pkg/mcpauth" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/ports/llm" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/ports/mcp" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" + "go.uber.org/zap" ) type capturingMCPClient struct { @@ -27,7 +31,8 @@ type capturingMCPClient struct { // failures 是返回成功前先失败的次数，用于覆盖重试路径。 failures int // attempts 记录每次尝试收到的请求头。 - attempts []map[string]string + attempts []map[string]string + fileTokens []string } func (c *capturingMCPClient) CallTool(_ context.Context, cfg mcp.CallConfig, input mcp.CallInput) (string, error) { @@ -35,6 +40,7 @@ func (c *capturingMCPClient) CallTool(_ context.Context, cfg mcp.CallConfig, inp c.cfg = cfg c.input = input c.attempts = append(c.attempts, maps.Clone(cfg.Headers)) + c.fileTokens = append(c.fileTokens, cfg.FileCreateToken) if len(c.attempts) <= c.failures { return "", fmt.Errorf("attempt %d failed", len(c.attempts)) } @@ -246,6 +252,57 @@ func TestExecuteToolCallRejectsToolsNotEnabledForRun(t *testing.T) { } } +type fileCreateServerStub struct { + repository.MCPRepository + enabled bool +} + +func (s fileCreateServerStub) GetServer(context.Context, uint) (*domainmcp.Server, error) { + return &domainmcp.Server{ID: 1, BaseURL: "https://generator.example/mcp", Status: "active", FileCreateEnabled: s.enabled}, nil +} + +type fileCreateGrantStub struct { + repository.MCPFileCreateRepository + grants []domainmcp.FileCreateGrant +} + +func (r *fileCreateGrantStub) CreateFileCreateGrant(_ context.Context, grant *domainmcp.FileCreateGrant, _ string) error { + r.grants = append(r.grants, *grant) + return nil +} + +func TestExecuteToolCallFileCapabilityRetryAndScope(t *testing.T) { + for _, mode := range []string{"enabled", "disabled", "temporary"} { + t.Run(mode, func(t *testing.T) { + client := &capturingMCPClient{output: "ok", failures: 1} + cfg := config.NewRuntime(config.Config{MCPEnable: true, MCPToolRetryCount: 1, MCPUserContextSecret: "secret"}) + repo := &fileCreateGrantStub{} + svc := &Service{cfg: cfg, mcpClient: client} + svc.SetMCPFileCreateService(appmcp.NewFileCreateService(cfg, fileCreateServerStub{enabled: mode != "disabled"}, repo, nil, nil, zap.NewNop())) + _, err := svc.executeToolCall(context.Background(), ExecuteToolInput{ + UserID: 42, ServerID: 1, ToolID: 7, ToolName: "generate", ArgumentsJSON: `{}`, Ephemeral: mode == "temporary", + MCPConfig: &mcp.CallConfig{BaseURL: "https://generator.example/mcp", Headers: map[string]string{mcpauth.HeaderName: mcpauth.TemplateSignedUserContext}}, + }) + if err != nil { + t.Fatal(err) + } + if mode != "enabled" { + if len(repo.grants) != 0 || client.fileTokens[0] != "" { + t.Fatal("unexpected file capability") + } + return + } + if len(repo.grants) != 1 || client.fileTokens[0] == "" || client.fileTokens[0] != client.fileTokens[1] { + t.Fatal("retry minted a second grant") + } + payload, err := verifyUserContextForTest("secret", client.attempts[0][mcpauth.HeaderName]) + if err != nil || payload.JTI != repo.grants[0].CallID || repo.grants[0].UserID != 42 || repo.grants[0].ToolID != 7 { + t.Fatalf("call binding mismatch: %v", err) + } + }) + } +} + func TestExecuteAssistantToolCallsStopsWhenToolNotEnabledForRun(t *testing.T) { svc := &Service{} result := svc.executeAssistantToolCalls(context.Background(), executeAssistantToolCallsInput{ diff --git a/backend/internal/application/mcp/errs.go b/backend/internal/application/mcp/errs.go index 51fc19563..1bd49a772 100644 --- a/backend/internal/application/mcp/errs.go +++ b/backend/internal/application/mcp/errs.go @@ -16,5 +16,9 @@ var ( ErrInvalidToolPrice = apperr.New("request.invalid_mcp_tool_price", "invalid mcp tool price") ErrMCPClientUnavailable = apperr.New("mcp.client_unavailable", "mcp client unavailable") // ErrServerLimitExceeded MCP 服务数量超限。 - ErrServerLimitExceeded = apperr.New("mcp.server_limit_exceeded", "mcp server limit exceeded") + ErrServerLimitExceeded = apperr.New("mcp.server_limit_exceeded", "mcp server limit exceeded") + ErrFileCreateUnauthorized = apperr.New("mcp.file_create.unauthorized", "file create capability is invalid, expired or revoked") + ErrFileCreateConflict = apperr.New("mcp.file_create.conflict", "file create capability is bound to another file or existing content is unavailable") + ErrFileCreateResultGone = apperr.New("mcp.file_create.result_gone", "original file create result is no longer available") + ErrFileCreateInvalidInput = apperr.New("mcp.file_create.invalid_input", "exactly one bounded file is required") ) diff --git a/backend/internal/application/mcp/file_create.go b/backend/internal/application/mcp/file_create.go new file mode 100644 index 000000000..35458f328 --- /dev/null +++ b/backend/internal/application/mcp/file_create.go @@ -0,0 +1,240 @@ +package mcp + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "time" + + appaudit "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/audit" + appupload "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/upload" + domainconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" + domainmcp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/mcp" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/config" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/pkg/conv" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" + "github.com/google/uuid" + "go.uber.org/zap" +) + +const FileCreateTTL = 10 * time.Minute + +func fileCreateError(err error) error { + switch { + case errors.Is(err, repository.ErrFileCreateUnauthorized): + return ErrFileCreateUnauthorized + case errors.Is(err, repository.ErrFileCreateConflict): + return ErrFileCreateConflict + case errors.Is(err, repository.ErrFileNotFound), errors.Is(err, repository.ErrNotFound): + return ErrFileCreateResultGone + case errors.Is(err, repository.ErrInvalidInput): + return ErrFileCreateInvalidInput + default: + return err + } +} + +type fileCreateAuditWriter interface { + Write(context.Context, appaudit.WriteInput) +} + +type FileCreateService struct { + cfg *config.Runtime + servers repository.MCPRepository + repo repository.MCPFileCreateRepository + uploads *appupload.Service + audit fileCreateAuditWriter + logger *zap.Logger +} + +func NewFileCreateService(cfg *config.Runtime, servers repository.MCPRepository, repo repository.MCPFileCreateRepository, uploads *appupload.Service, audit fileCreateAuditWriter, logger *zap.Logger) *FileCreateService { + return &FileCreateService{cfg: cfg, servers: servers, repo: repo, uploads: uploads, audit: audit, logger: logger} +} + +type FileCreateCall struct { + ServerID uint + ToolID uint + UserID uint + BaseURL string + CallID string + RequestID string + Epoch uint +} + +// IssueFileCreateGrant is internal-only. No external identity/signature can mint it. +func (s *FileCreateService) IssueFileCreateGrant(ctx context.Context, call FileCreateCall) (string, error) { + if !s.cfg.Snapshot().MCPEnable { + return "", nil + } + server, err := s.servers.GetServer(ctx, call.ServerID) + if err != nil { + return "", err + } + if !server.FileCreateEnabled { + return "", nil + } + if server.Status != "active" || call.UserID == 0 || call.ToolID == 0 || call.CallID == "" || server.BaseURL != call.BaseURL || server.FileCreateEpoch != call.Epoch { + return "", repository.ErrFileCreateUnauthorized + } + raw := make([]byte, 32) + if _, err = rand.Read(raw); err != nil { + return "", err + } + token := "dxf1_" + base64.RawURLEncoding.EncodeToString(raw) + now := time.Now() + grant := &domainmcp.FileCreateGrant{ + TokenHash: fileCreateTokenHash(token), ServerID: call.ServerID, ToolID: call.ToolID, + UserID: call.UserID, Epoch: server.FileCreateEpoch, CallID: call.CallID, RequestID: call.RequestID, + CreatedAt: now, ExpiresAt: now.Add(FileCreateTTL), CandidateID: "file_" + conv.NormalizePublicID(uuid.NewString()), + } + if err = s.repo.CreateFileCreateGrant(ctx, grant, call.BaseURL); err != nil { + return "", err + } + return token, nil +} + +func fileCreateTokenHash(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +func (s *FileCreateService) Authorize(ctx context.Context, token string, serverID uint) (*domainmcp.FileCreateGrant, error) { + if !s.cfg.Snapshot().MCPEnable || len(token) != 48 || token[:5] != "dxf1_" { + return nil, ErrFileCreateUnauthorized + } + grant, err := s.repo.GetFileCreateGrant(ctx, fileCreateTokenHash(token), serverID) + return grant, fileCreateError(err) +} + +type FileCreateResult struct { + File domainconversation.FileObject + Reused bool + Replayed bool +} + +func (s *FileCreateService) Create(ctx context.Context, token string, serverID uint, input appupload.TemporaryFileInput) (result *FileCreateResult, err error) { + defer func() { err = fileCreateError(err) }() + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + grant, err := s.Authorize(ctx, token, serverID) + if err != nil { + return nil, err + } + prepared, err := s.uploads.PrepareTemporaryFile(ctx, input) + if err != nil { + return nil, err + } + defer prepared.Cleanup() + if len(prepared.FileName) > 255 || len(prepared.MimeType) > 128 { + return nil, repository.ErrInvalidInput + } + manifest, err := json.Marshal([]any{prepared.FileName, prepared.MimeType, prepared.SHA256, prepared.SizeBytes}) + if err != nil { + return nil, err + } + fingerprint := sha256.Sum256(manifest) + key, err := s.uploads.FileCreateStoragePath(ctx, grant, prepared) + if err != nil { + return nil, err + } + if err = s.repo.BindFileCreateGrant(ctx, grant.TokenHash, serverID, hex.EncodeToString(fingerprint[:]), key); err != nil { + return nil, err + } + result = &FileCreateResult{} + err = s.repo.WithFileCreateGrant(ctx, grant.TokenHash, serverID, func(locked *domainmcp.FileCreateGrant, uploads repository.UploadRepository) error { + if locked.FileID != "" { + file, readErr := uploads.GetActiveFileObjectByID(ctx, locked.UserID, locked.FileID) + if readErr != nil { + return readErr + } + result.File, result.Reused, result.Replayed = *file, locked.Reused, true + return nil + } + // ponytail: hold a user quota lock during bounded object I/O; use short + // leases only if measured per-user upload contention warrants the complexity. + if _, lockErr := uploads.GetOrInitUserStorageQuota(ctx, locked.UserID, s.cfg.Snapshot().UserStorageQuotaBytes); lockErr != nil { + return lockErr + } + created, createErr := s.uploads.CommitFileCreate(ctx, uploads, locked, prepared) + if createErr != nil { + return createErr + } + locked.FileID, locked.Reused = created.File.FileID, created.Reused + result.File, result.Reused = created.File, created.Reused + return nil + }) + if err != nil { + return nil, err + } + if file, finishErr := s.uploads.FinishFileCreateProcessing(ctx, grant.UserID, result.File.FileID); finishErr == nil { + result.File = *file + } + if s.audit != nil { + auditCtx, cancelAudit := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancelAudit() + s.audit.Write(auditCtx, appaudit.WriteInput{ + RequestID: grant.RequestID, ActorUserID: grant.UserID, Action: "mcp_create_file", Resource: "file", ResourceID: result.File.FileID, + Detail: map[string]any{"mcp_server_id": grant.ServerID, "mcp_tool_id": grant.ToolID, "call_id": grant.CallID, "target_user_id": grant.UserID, "size_bytes": result.File.SizeBytes, "reused": result.Reused, "replayed": result.Replayed}, + }) + } + return result, nil +} + +// Maintain resumes receipt-to-processing handoff and removes expired abandoned +// candidates. It uses existing processing workers, not an additional task queue. +func (s *FileCreateService) Maintain(ctx context.Context) { + now := time.Now() + grants, err := s.repo.ListFileCreateMaintenance(ctx, now) + if err != nil { + s.logger.Warn("mcp_file_create_maintenance_failed", zap.Error(err)) + return + } + for _, grant := range grants { + if grant.FileID != "" { + file, finishErr := s.uploads.FinishFileCreateProcessing(ctx, grant.UserID, grant.FileID) + if finishErr != nil && !errors.Is(finishErr, repository.ErrNotFound) && !errors.Is(finishErr, repository.ErrFileNotFound) { + continue // Keep durable handoff evidence on transient lookup failures. + } + if finishErr == nil && (file.ProcessingStatus == "uploaded" || file.ProcessingStatus == "queued") { + continue + } + } + if !grant.ExpiresAt.After(now) { + err = s.repo.ExpireFileCreateGrant(ctx, grant.TokenHash, now, func(key string) error { return s.uploads.RemoveFileCreateCandidate(ctx, key) }) + if err != nil && !errors.Is(err, repository.ErrNotFound) { + s.logger.Warn("mcp_file_create_cleanup_failed", zap.Error(err)) + } + } + } +} + +func (s *FileCreateService) MaxUploadBytes() int64 { + if n := s.cfg.Snapshot().MaxUploadFileBytes; n > 0 { + return n + } + return 20 * 1024 * 1024 +} + +func (s *FileCreateService) StartMaintenance(ctx context.Context) { + go func() { + startupCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + s.Maintain(startupCtx) + cancel() + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + maintenanceCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + s.Maintain(maintenanceCtx) + cancel() + } + } + }() +} diff --git a/backend/internal/application/mcp/service.go b/backend/internal/application/mcp/service.go index 127f05cae..b49dbece9 100644 --- a/backend/internal/application/mcp/service.go +++ b/backend/internal/application/mcp/service.go @@ -41,11 +41,12 @@ type ReorderServerInput struct { } type ServerInput struct { - Name string - BaseURL string - AuthToken string - HeadersJSON string - Status string + Name string + BaseURL string + AuthToken string + HeadersJSON string + Status string + FileCreateEnabled *bool } type ToolInput struct { @@ -91,11 +92,12 @@ func (s *Service) CreateServer(ctx context.Context, input ServerInput) (*domainm return nil, err } item, err := s.repo.CreateServer(ctx, repository.CreateMCPServerInput{ - Name: normalized.Name, - BaseURL: normalized.BaseURL, - AuthTokenEnc: tokenEnc, - HeadersJSON: normalized.HeadersJSON, - Status: normalized.Status, + Name: normalized.Name, + BaseURL: normalized.BaseURL, + AuthTokenEnc: tokenEnc, + HeadersJSON: normalized.HeadersJSON, + Status: normalized.Status, + FileCreateEnabled: normalized.FileCreateEnabled != nil && *normalized.FileCreateEnabled, }) if errors.Is(err, repository.ErrMCPServerLimitExceeded) { return nil, ErrServerLimitExceeded @@ -109,10 +111,11 @@ func (s *Service) UpdateServer(ctx context.Context, serverID uint, input ServerI return nil, err } update := repository.UpdateMCPServerInput{ - Name: &normalized.Name, - BaseURL: &normalized.BaseURL, - HeadersJSON: &normalized.HeadersJSON, - Status: &normalized.Status, + Name: &normalized.Name, + BaseURL: &normalized.BaseURL, + HeadersJSON: &normalized.HeadersJSON, + Status: &normalized.Status, + FileCreateEnabled: normalized.FileCreateEnabled, } if normalized.AuthToken != "" { tokenEnc, encryptErr := s.encryptToken(normalized.AuthToken) @@ -426,11 +429,12 @@ func (s *Service) normalizeServerInput(input ServerInput, requireToken bool) (Se input.AuthToken = strings.TrimSpace(input.AuthToken) } return ServerInput{ - Name: name, - BaseURL: baseURL, - AuthToken: strings.TrimSpace(input.AuthToken), - HeadersJSON: headersJSON, - Status: status, + Name: name, + BaseURL: baseURL, + AuthToken: strings.TrimSpace(input.AuthToken), + HeadersJSON: headersJSON, + Status: status, + FileCreateEnabled: input.FileCreateEnabled, }, nil } diff --git a/backend/internal/application/processing/mcp_file_create.go b/backend/internal/application/processing/mcp_file_create.go new file mode 100644 index 000000000..63a28c18c --- /dev/null +++ b/backend/internal/application/processing/mcp_file_create.go @@ -0,0 +1,65 @@ +package processing + +import ( + "context" + "errors" + "time" + + domainconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" +) + +// EnsureFileCreateProcessing resumes the existing pipeline after receipt commit. +// A conditional transition cannot reset a progressed file. Repeated queue +// delivery is safe because processing workers already claim with attempt IDs. +func (s *Service) EnsureFileCreateProcessing(ctx context.Context, file *domainconversation.FileObject) error { + return s.ensureUploadedFileProcessing(ctx, file, false) +} + +func (s *Service) ensureUploadedFileProcessing(ctx context.Context, file *domainconversation.FileObject, terminalQueueError bool) error { + if file == nil { + return nil + } + if file.ProcessingStatus != "uploaded" && file.ProcessingStatus != "queued" { + return nil + } + if file.ProcessingStatus == "uploaded" { + now := time.Now() + state := &domainconversation.FileObjectProcessing{ + FileObjectID: file.ID, UserID: file.UserID, DetectedMIME: file.DetectedMIME, + FileCategory: file.FileCategory, ProcessingStatus: "queued", ExtractStatus: "none", + ExtractorVersion: s.version(), StartedAt: &now, + } + if file.FileCategory == "video" || file.FileCategory == "audio" || (file.FileCategory == "image" && !s.snapshot().ExtractImageOCREnabled) { + state = s.readyWithoutExtractionState(file, file.FileCategory+"_not_applicable") + } else if !supportsExtraction(file.FileCategory) { + state.ProcessingStatus = "failed" + state.ErrorCode = "mime_blocked" + state.ErrorMessage = "unsupported file category" + } + state.ExpectedStatus = "uploaded" + if err := s.repo.UpdateFileObjectProcessingState(ctx, state); err != nil { + if errors.Is(err, repository.ErrConflict) { + return nil + } + return err + } + file.ProcessingStatus = state.ProcessingStatus + file.ProcessingReady = state.ProcessingReady + file.ExtractStatus = state.ExtractStatus + if state.ProcessingStatus != "queued" { + return nil + } + } + err := s.enqueueFileProcessing(ctx, file.UserID, file.FileID, 0, "") + if err != nil && terminalQueueError { + code := "queue_unavailable" + if errors.Is(err, repository.ErrFileProcessingQueueFull) { + code = "queue_full" + } + state := s.failedFileProcessingState(file, code, HumanizeFileProcessingError(file.FileCategory, code, "")) + state.ExpectedStatus = "queued" + _ = s.repo.UpdateFileObjectProcessingState(ctx, state) + } + return err +} diff --git a/backend/internal/application/processing/service.go b/backend/internal/application/processing/service.go index 26c0c1431..7f7ed7b73 100644 --- a/backend/internal/application/processing/service.go +++ b/backend/internal/application/processing/service.go @@ -307,56 +307,7 @@ func (s *Service) ResolveFileVectorizationCapabilities( // InitializeUploadedFile 初始化新上传文件的处理状态。 func (s *Service) InitializeUploadedFile(ctx context.Context, fileObj *domainconversation.FileObject) error { - if fileObj == nil { - return nil - } - if fileObj.FileCategory == "video" || fileObj.FileCategory == "audio" || (fileObj.FileCategory == "image" && !s.snapshot().ExtractImageOCREnabled) { - fileObj.ProcessingStatus = "ready" - fileObj.ProcessingReady = true - fileObj.ExtractStatus = "none" - ragReason := "image_not_applicable" - switch fileObj.FileCategory { - case "video": - ragReason = "video_not_applicable" - case "audio": - ragReason = "audio_not_applicable" - } - return s.repo.UpdateFileObjectProcessingState(ctx, s.readyWithoutExtractionState(fileObj, ragReason)) - } - - if !supportsExtraction(fileObj.FileCategory) { - return s.markFileProcessingFailed(ctx, fileObj, "mime_blocked", "unsupported file category") - } - - now := time.Now() - if err := s.repo.UpdateFileObjectProcessingState(ctx, &domainconversation.FileObjectProcessing{ - FileObjectID: fileObj.ID, - UserID: fileObj.UserID, - DetectedMIME: fileObj.DetectedMIME, - FileCategory: fileObj.FileCategory, - ProcessingStatus: "queued", - ProcessingReady: false, - ExtractStatus: "none", - ExtractorVersion: s.version(), - StartedAt: &now, - }); err != nil { - return err - } - if err := s.enqueueFileProcessing(ctx, fileObj.UserID, fileObj.FileID, 0, ""); err != nil { - code := "queue_unavailable" - if errors.Is(err, repository.ErrFileProcessingQueueFull) { - code = "queue_full" - } - if failErr := s.markFileProcessingFailed(ctx, fileObj, code, HumanizeFileProcessingError(fileObj.FileCategory, code, "")); failErr != nil && s.logger != nil { - s.logger.Warn("mark_file_failed_after_enqueue_error", - zap.Uint("user_id", fileObj.UserID), - zap.String("file_id", fileObj.FileID), - zap.Error(failErr), - ) - } - return err - } - return nil + return s.ensureUploadedFileProcessing(ctx, fileObj, true) } // ProcessFile 执行单个文件处理任务。 diff --git a/backend/internal/application/upload/mcp_file_create.go b/backend/internal/application/upload/mcp_file_create.go new file mode 100644 index 000000000..784aa90e2 --- /dev/null +++ b/backend/internal/application/upload/mcp_file_create.go @@ -0,0 +1,141 @@ +package upload + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + domainconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" + domainmcp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/mcp" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/ports/objectstorage" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" +) + +// FileCreateStoragePath derives a stable candidate key before object I/O. +func (s *Service) FileCreateStoragePath(ctx context.Context, grant *domainmcp.FileCreateGrant, prepared *PreparedTemporaryFile) (string, error) { + user, err := s.repo.GetUserByID(ctx, grant.UserID) + if err != nil { + return "", err + } + owner := user.PublicID + if owner == "" { + owner = fmt.Sprintf("uid_%d", grant.UserID) + } + return filepath.ToSlash(filepath.Join(owner, grant.CreatedAt.Format("2006"), grant.CreatedAt.Format("01"), grant.CandidateID+"_"+prepared.FileName)), nil +} + +// CommitFileCreate uses the caller's receipt transaction for file/quota settlement. +// The candidate key is already durably bound to the exact bytes. Never delete it +// on an uncertain storage/transaction error: replay reconciles that same key. +func (s *Service) CommitFileCreate(ctx context.Context, repo repository.UploadRepository, grant *domainmcp.FileCreateGrant, prepared *PreparedTemporaryFile) (*UploadFileResult, error) { + cfg := s.snapshot() + store, err := s.openObjectStore(ctx) + if err != nil { + return nil, err + } + existing, err := repo.GetLatestActiveFileObjectBySHA(ctx, grant.UserID, prepared.SHA256, prepared.SizeBytes) + if err != nil { + return nil, err + } + if existing != nil { + matches, matchErr := objectMatchesContent(ctx, store, existing.StoragePath, prepared.SHA256, prepared.SizeBytes) + if matchErr != nil { + return nil, matchErr + } + // A create-only capability must not repair/delete another file. + if !matches { + return nil, repository.ErrFileCreateConflict + } + quota, quotaErr := repo.GetOrInitUserStorageQuota(ctx, grant.UserID, cfg.UserStorageQuotaBytes) + if quotaErr != nil { + return nil, quotaErr + } + return &UploadFileResult{File: *existing, Quota: *quota, Reused: true}, nil + } + quota, err := repo.GetOrInitUserStorageQuota(ctx, grant.UserID, cfg.UserStorageQuotaBytes) + if err != nil { + return nil, err + } + if quota.QuotaBytes > 0 && quota.UsedBytes+quota.ReservedBytes+prepared.SizeBytes > quota.QuotaBytes { + return nil, s.errStorageQuotaExceeded() + } + matches, err := objectMatchesContent(ctx, store, grant.StoragePath, prepared.SHA256, prepared.SizeBytes) + if err != nil { + return nil, err + } + if !matches { + if err = putPreparedFile(ctx, store, grant.StoragePath, prepared); err != nil { + return nil, err + } + matches, err = objectMatchesContent(ctx, store, grant.StoragePath, prepared.SHA256, prepared.SizeBytes) + if err != nil { + return nil, err + } + if !matches { + return nil, errors.New("file create candidate content verification failed") + } + } + file := s.fileObjectFromPrepared(grant.CandidateID, grant.UserID, "mcp_output", grant.StoragePath, prepared) + quota, err = repo.CreateFileObjectAndConsumeQuota(ctx, file, cfg.UserStorageQuotaBytes) + if err != nil { + if errors.Is(err, repository.ErrStorageQuotaExceeded) { + return nil, s.errStorageQuotaExceeded() + } + return nil, err + } + return &UploadFileResult{File: *file, Quota: *quota}, nil +} + +func putPreparedFile(ctx context.Context, store objectstorage.Store, key string, prepared *PreparedTemporaryFile) error { + file, err := os.Open(prepared.AbsolutePath) + if err != nil { + return err + } + defer file.Close() //nolint:errcheck + _, err = store.Put(ctx, key, file, objectstorage.PutOptions{SizeBytes: prepared.SizeBytes, ContentType: prepared.DetectedMIME}) + return err +} + +func (s *Service) fileObjectFromPrepared(fileID string, userID uint, purpose string, key string, prepared *PreparedTemporaryFile) *domainconversation.FileObject { + cfg := s.snapshot() + category := prepared.FileCategory + return &domainconversation.FileObject{ + FileID: fileID, UserID: userID, Purpose: normalizePurpose(purpose), FileName: prepared.FileName, + MimeType: prepared.MimeType, DetectedMIME: prepared.DetectedMIME, FileCategory: category, + SizeBytes: prepared.SizeBytes, SHA256: prepared.SHA256, StoragePath: key, Status: "active", + ProcessingStatus: "uploaded", ProcessingReady: category == fileCategoryVideo || category == fileCategoryAudio || (category == fileCategoryImage && !cfg.ExtractImageOCREnabled), + ExtractStatus: "none", EmbedStatus: "none", ExtractorVersion: s.resolveExtractorVersion(), + } +} + +// RemoveFileCreateCandidate is only called under an expired receipt's lock. +func (s *Service) RemoveFileCreateCandidate(ctx context.Context, key string) error { + store, err := s.openObjectStore(ctx) + if err != nil { + return err + } + err = store.Delete(ctx, key) + if errors.Is(err, objectstorage.ErrNotFound) { + return nil + } + return err +} + +// FinishFileCreateProcessing reuses the existing processing hook then reloads +// persisted state; creation does not imply extraction/OCR/embedding completed. +func (s *Service) FinishFileCreateProcessing(ctx context.Context, userID uint, fileID string) (*domainconversation.FileObject, error) { + file, err := s.repo.GetActiveFileObjectByID(ctx, userID, fileID) + if err != nil { + return nil, err + } + if (file.ProcessingStatus == "uploaded" || file.ProcessingStatus == "queued") && s.hooks.EnsureFileCreateProcessing != nil { + initCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + _ = s.hooks.EnsureFileCreateProcessing(initCtx, file) + cancel() + return s.repo.GetActiveFileObjectByID(ctx, userID, fileID) + } + return file, nil +} diff --git a/backend/internal/application/upload/service.go b/backend/internal/application/upload/service.go index 72d21263a..41ec70ff7 100644 --- a/backend/internal/application/upload/service.go +++ b/backend/internal/application/upload/service.go @@ -32,7 +32,8 @@ var errLocalFileTooLarge = errors.New("local file too large") // Hooks 封装上传后续编排动作。 type Hooks struct { - InitializeUploadedFile func(ctx context.Context, file *domainconversation.FileObject) error + InitializeUploadedFile func(ctx context.Context, file *domainconversation.FileObject) error + EnsureFileCreateProcessing func(ctx context.Context, file *domainconversation.FileObject) error } // ErrorSet 允许上层注入统一错误语义。 @@ -297,25 +298,10 @@ func (s *Service) UploadFile(ctx context.Context, input UploadFileInput) (*Uploa return result, nil } - fileItem := &domainconversation.FileObject{ - FileID: fileID, - UserID: ownerUserID, - Purpose: normalizePurpose(input.Purpose), - FileName: normalizedName, - MimeType: normalizedMIME, - DetectedMIME: detectedMIME, - FileCategory: category, - SizeBytes: sizeBytes, - SHA256: shaValue, - StoragePath: relativePath, - Status: "active", - ProcessingStatus: "uploaded", - ProcessingReady: category == fileCategoryVideo || category == fileCategoryAudio || (category == fileCategoryImage && !cfg.ExtractImageOCREnabled), - ExtractStatus: "none", - EmbedStatus: "none", - ExtractorVersion: s.resolveExtractorVersion(), - ExpiresAt: nil, - } + fileItem := s.fileObjectFromPrepared(fileID, ownerUserID, input.Purpose, relativePath, &PreparedTemporaryFile{ + FileName: normalizedName, MimeType: normalizedMIME, DetectedMIME: detectedMIME, + FileCategory: category, SizeBytes: sizeBytes, SHA256: shaValue, + }) quota, err := s.repo.CreateFileObjectAndConsumeQuota(ctx, fileItem, quotaBytes) if err != nil && errors.Is(err, repository.ErrDuplicate) { @@ -985,14 +971,8 @@ func saveUploadedFile(ctx context.Context, input saveUploadedFileInput) (string, input.FileID+"_"+sanitizeFileName(input.FileName), ) relativePath = filepath.ToSlash(relativePath) - tmpFile, err := os.Open(staged.absolutePath) - if err != nil { - return "", "", "", 0, err - } - defer tmpFile.Close() //nolint:errcheck - if _, err = input.Store.Put(ctx, relativePath, tmpFile, objectstorage.PutOptions{ - SizeBytes: staged.sizeBytes, - ContentType: staged.detectedMIME, + if err = putPreparedFile(ctx, input.Store, relativePath, &PreparedTemporaryFile{ + AbsolutePath: staged.absolutePath, SizeBytes: staged.sizeBytes, DetectedMIME: staged.detectedMIME, }); err != nil { return "", "", "", 0, err } diff --git a/backend/internal/domain/conversation/types.go b/backend/internal/domain/conversation/types.go index 7be46a1ba..41e3cf0c0 100644 --- a/backend/internal/domain/conversation/types.go +++ b/backend/internal/domain/conversation/types.go @@ -345,6 +345,8 @@ func IsFileProcessing(file FileObject) bool { // FileObjectProcessing 表示 file_objects 中的服务端处理状态。 type FileObjectProcessing struct { + // ExpectedStatus fences initialization/failure writes against a progressed worker. + ExpectedStatus string ID uint FileObjectID uint UserID uint diff --git a/backend/internal/domain/mcp/types.go b/backend/internal/domain/mcp/types.go index e9fb7efef..fa87d81aa 100644 --- a/backend/internal/domain/mcp/types.go +++ b/backend/internal/domain/mcp/types.go @@ -18,6 +18,8 @@ type Server struct { AuthTokenEnc string HeadersJSON string Status string + FileCreateEnabled bool + FileCreateEpoch uint SortOrder int ToolCount int ActiveToolCount int @@ -33,6 +35,25 @@ type ServerWithTools struct { Tools []Tool } +// FileCreateGrant is a DEEIX-issued capability for one file from one tool call. +// TokenHash is never a credential; the random bearer is only sent to that MCP. +type FileCreateGrant struct { + TokenHash string + ServerID uint + ToolID uint + UserID uint + Epoch uint + CallID string + RequestID string + ExpiresAt time.Time + CreatedAt time.Time + Fingerprint string + CandidateID string + StoragePath string + FileID string + Reused bool +} + // Tool 表示从 MCP 服务发现并由管理员控制可用性的工具。 type Tool struct { ID uint diff --git a/backend/internal/infra/cache/memory/file_queue.go b/backend/internal/infra/cache/memory/file_queue.go index 0e84fbfd4..e415c30fb 100644 --- a/backend/internal/infra/cache/memory/file_queue.go +++ b/backend/internal/infra/cache/memory/file_queue.go @@ -2,6 +2,7 @@ package memory import ( "context" + "fmt" "strconv" "strings" "time" @@ -25,16 +26,18 @@ type fileProcessingLease struct { } type fileQueueState struct { - queue []repository.FileProcessingMessage - inflight map[string]fileProcessingLease - dlq []repository.FileProcessingMessage - notify chan struct{} + queue []repository.FileProcessingMessage + inflight map[string]fileProcessingLease + dlq []repository.FileProcessingMessage + notify chan struct{} + outstanding map[string]bool } func newFileQueueState() fileQueueState { return fileQueueState{ - inflight: map[string]fileProcessingLease{}, - notify: make(chan struct{}), + inflight: map[string]fileProcessingLease{}, + outstanding: map[string]bool{}, + notify: make(chan struct{}), } } @@ -88,6 +91,11 @@ func (c *Cache) enqueueFileMessage(ctx context.Context, message repository.FileP c.mu.Lock() now := time.Now() state := c.fileQueueState(message.Queue) + key := processingMessageKey(message) + if key != "" && state.outstanding[key] { + c.mu.Unlock() + return nil + } if len(state.queue) >= maxFileQueueLength { c.maybeSweepLocked(now) c.mu.Unlock() @@ -99,6 +107,9 @@ func (c *Cache) enqueueFileMessage(ctx context.Context, message repository.FileP message.LastError = truncateFileQueueError(message.LastError) msg := message state.queue = append(state.queue, msg) + if key != "" { + state.outstanding[key] = true + } notifyFileQueueLocked(state) c.maybeSweepLocked(now) c.mu.Unlock() @@ -233,6 +244,7 @@ func (c *Cache) SettleFileProcessingMessage(ctx context.Context, consumerName st return false, nil } delete(state.inflight, messageID) + delete(state.outstanding, processingMessageKey(message)) c.maybeSweepLocked(time.Now()) return true, nil } @@ -314,6 +326,7 @@ func (c *Cache) DeadLetterFileProcessingMessage( if len(state.dlq) > 10_000 { state.dlq = append([]repository.FileProcessingMessage(nil), state.dlq[len(state.dlq)-10_000:]...) } + delete(state.outstanding, processingMessageKey(message)) delete(state.inflight, messageID) c.maybeSweepLocked(time.Now()) return true, nil @@ -336,6 +349,13 @@ func queueForMessage(message repository.FileProcessingMessage) repository.FilePr return repository.FileProcessingQueueDefault } +func processingMessageKey(message repository.FileProcessingMessage) string { + if message.Kind != "" || queueForMessage(message) != repository.FileProcessingQueueDefault { + return "" + } + return fmt.Sprintf("%d:%s", message.UserID, strings.TrimSpace(message.FileID)) +} + func notifyFileQueueLocked(state *fileQueueState) { close(state.notify) state.notify = make(chan struct{}) diff --git a/backend/internal/infra/cache/memory/file_queue_test.go b/backend/internal/infra/cache/memory/file_queue_test.go index 7c87d524f..c14141861 100644 --- a/backend/internal/infra/cache/memory/file_queue_test.go +++ b/backend/internal/infra/cache/memory/file_queue_test.go @@ -134,3 +134,45 @@ func TestFileEmbeddingQueueIsIsolatedFromExtractionQueue(t *testing.T) { t.Fatalf("embedding message queue = %q", embeddingMessages[0].Queue) } } + +func TestFileProcessingRecoveryEnqueueIsIdempotentUntilSettlement(t *testing.T) { + cache := New() + ctx := context.Background() + for i := 0; i < 10; i++ { + if err := cache.EnqueueFileProcessing(ctx, 1, "file", 0, ""); err != nil { + t.Fatal(err) + } + } + if len(cache.fileProcessingQueue.queue) != 1 { + t.Fatal("recovery duplicated queued work") + } + messages, err := cache.ReadFileProcessingMessages(ctx, "worker") + if err != nil || len(messages) != 1 { + t.Fatal(err) + } + if err = cache.EnqueueFileProcessing(ctx, 1, "file", 0, ""); err != nil { + t.Fatal(err) + } + if len(cache.fileProcessingQueue.queue) != 0 { + t.Fatal("recovery duplicated inflight work") + } + if ok, retryErr := cache.RequeueFileProcessingMessage(ctx, "worker", messages[0], 1, "retry"); retryErr != nil || !ok { + t.Fatal(retryErr) + } + if err = cache.EnqueueFileProcessing(ctx, 1, "file", 0, ""); err != nil { + t.Fatal(err) + } + if len(cache.fileProcessingQueue.queue) != 1 { + t.Fatal("recovery duplicated a retry") + } + messages, err = cache.ReadFileProcessingMessages(ctx, "worker") + if err != nil { + t.Fatal(err) + } + if ok, settleErr := cache.SettleFileProcessingMessage(ctx, "worker", messages[0]); settleErr != nil || !ok { + t.Fatal(settleErr) + } + if len(cache.fileProcessingQueue.outstanding) != 0 { + t.Fatal("settlement leaked outstanding key") + } +} diff --git a/backend/internal/infra/cache/redis/conversation_cache.go b/backend/internal/infra/cache/redis/conversation_cache.go index e46189966..c38bd0de8 100644 --- a/backend/internal/infra/cache/redis/conversation_cache.go +++ b/backend/internal/infra/cache/redis/conversation_cache.go @@ -198,6 +198,7 @@ if #pending == 0 or pending[1][2] ~= ARGV[2] then end redis.call("XACK", KEYS[1], ARGV[1], ARGV[3]) redis.call("XDEL", KEYS[1], ARGV[3]) +if KEYS[2] ~= "" and redis.call("GET", KEYS[2]) == ARGV[3] then redis.call("DEL", KEYS[2]) end return 1 `) @@ -206,7 +207,7 @@ local pending = redis.call("XPENDING", KEYS[1], ARGV[1], ARGV[3], ARGV[3], 1) if #pending == 0 or pending[1][2] ~= ARGV[2] then return 0 end -redis.call( +local next_id = redis.call( "XADD", KEYS[1], "*", "user_id", ARGV[4], "file_id", ARGV[5], @@ -218,6 +219,7 @@ redis.call( ) redis.call("XACK", KEYS[1], ARGV[1], ARGV[3]) redis.call("XDEL", KEYS[1], ARGV[3]) +if KEYS[2] ~= "" and redis.call("GET", KEYS[2]) == ARGV[3] then redis.call("SET", KEYS[2], next_id) end return 1 `) @@ -238,9 +240,20 @@ redis.call( ) redis.call("XACK", KEYS[1], ARGV[1], ARGV[3]) redis.call("XDEL", KEYS[1], ARGV[3]) +if KEYS[3] and KEYS[3] ~= "" and redis.call("GET", KEYS[3]) == ARGV[3] then redis.call("DEL", KEYS[3]) end return 1 `) +// An outstanding stream entry, not an expiring key, is the enqueue authority. +// Recovery after an uncertain enqueue reuses it; deleted streams can be rebuilt. +var enqueueFileProcessingOnceScript = goredis.NewScript(` +local current = redis.call("GET", KEYS[2]) +if current and #redis.call("XRANGE", KEYS[1], current, current, "COUNT", 1) > 0 then return current end +local id = redis.call("XADD", KEYS[1], "*", "user_id", ARGV[1], "file_id", ARGV[2], "retry", ARGV[3], "last_error", ARGV[4]) +redis.call("SET", KEYS[2], id) +return id +`) + var renewGenerationStreamLeaseScript = goredis.NewScript(` if redis.call("GET", KEYS[1]) ~= ARGV[1] or redis.call("GET", KEYS[2]) ~= ARGV[2] @@ -331,18 +344,12 @@ func (c *conversationCache) EnqueueFileProcessing(ctx context.Context, userID ui if c.client == nil { return nil } - values := map[string]any{ - "user_id": userID, - "file_id": fileID, - "retry": retry, - } - if strings.TrimSpace(lastError) != "" { - values["last_error"] = truncateStr(lastError, 255) + fileID = strings.TrimSpace(fileID) + if fileID == "" { + return repository.ErrInvalidInput } - _, err := c.client.XAdd(ctx, &goredis.XAddArgs{ - Stream: fileProcessingStreamName, - Values: values, - }).Result() + _, err := enqueueFileProcessingOnceScript.Run(ctx, c.client, + []string{fileProcessingStreamName, processingOutstandingKey(userID, fileID)}, userID, fileID, retry, truncateStr(lastError, 255)).Result() return err } @@ -556,11 +563,16 @@ func (c *conversationCache) deadLetterInvalidFileProcessingMessage( queue fileQueueConfig, ) (bool, error) { lastError := "invalid queue message" + userID, userIDErr := strconv.ParseUint(getStringVal(message.Values["user_id"]), 10, strconv.IntSize) + marker := "" + if userIDErr == nil && queue.queue == repository.FileProcessingQueueDefault && getOptionalStringVal(message.Values, "kind") == "" { + marker = processingOutstandingKey(uint(userID), getStringVal(message.Values["file_id"])) + } return fileProcessingScriptResult(deadLetterFileProcessingMessageScript.Run( ctx, c.client, - []string{queue.stream, queue.dlq}, + []string{queue.stream, queue.dlq, marker}, queue.group, consumerName, message.ID, @@ -599,7 +611,7 @@ func (c *conversationCache) SettleFileProcessingMessage(ctx context.Context, con return fileProcessingScriptResult(settleFileProcessingMessageScript.Run( ctx, c.client, - []string{queue.stream}, + []string{queue.stream, processingOutstandingMessageKey(message)}, queue.group, consumerName, message.ID, @@ -620,7 +632,7 @@ func (c *conversationCache) RequeueFileProcessingMessage( return fileProcessingScriptResult(requeueFileProcessingMessageScript.Run( ctx, c.client, - []string{queue.stream}, + []string{queue.stream, processingOutstandingMessageKey(message)}, queue.group, consumerName, message.ID, @@ -647,7 +659,7 @@ func (c *conversationCache) DeadLetterFileProcessingMessage( return fileProcessingScriptResult(deadLetterFileProcessingMessageScript.Run( ctx, c.client, - []string{queue.stream, queue.dlq}, + []string{queue.stream, queue.dlq, processingOutstandingMessageKey(message)}, queue.group, consumerName, message.ID, @@ -662,6 +674,17 @@ func (c *conversationCache) DeadLetterFileProcessingMessage( ).Result()) } +func processingOutstandingKey(userID uint, fileID string) string { + return fmt.Sprintf("file_processing:outstanding:%d:%s", userID, strings.TrimSpace(fileID)) +} + +func processingOutstandingMessageKey(message repository.FileProcessingMessage) string { + if redisQueueForMessage(message).queue != repository.FileProcessingQueueDefault || message.Kind != "" { + return "" + } + return processingOutstandingKey(message.UserID, message.FileID) +} + func processingQueueConfig() fileQueueConfig { return fileQueueConfig{ stream: fileProcessingStreamName, diff --git a/backend/internal/infra/cache/redis/file_queue_test.go b/backend/internal/infra/cache/redis/file_queue_test.go index ae717b983..a2f4ae001 100644 --- a/backend/internal/infra/cache/redis/file_queue_test.go +++ b/backend/internal/infra/cache/redis/file_queue_test.go @@ -1,6 +1,8 @@ package redis import ( + "context" + "os" "testing" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" @@ -30,6 +32,128 @@ func TestParseFileEmbeddingMessagePreservesMetadata(t *testing.T) { } } +func TestFileProcessingRedisRecoveryEnqueueIsIdempotent(t *testing.T) { + addr := os.Getenv("DEEIX_TEST_REDIS_ADDR") + if addr == "" { + t.Skip("set DEEIX_TEST_REDIS_ADDR to a dedicated Redis test instance") + } + client := goredis.NewClient(&goredis.Options{Addr: addr, DB: 15}) + t.Cleanup(func() { _ = client.Close() }) + ctx := context.Background() + if err := client.FlushDB(ctx).Err(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = client.FlushDB(ctx).Err() }) + cache := &conversationCache{client: client} + if err := cache.InitFileProcessingStream(ctx); err != nil { + t.Fatal(err) + } + for i := 0; i < 10; i++ { + if err := cache.EnqueueFileProcessing(ctx, 1, "file", 0, ""); err != nil { + t.Fatal(err) + } + } + if count, err := client.XLen(ctx, fileProcessingStreamName).Result(); err != nil || count != 1 { + t.Fatalf("stream length %d: %v", count, err) + } + messages, err := cache.ReadFileProcessingMessages(ctx, "worker") + if err != nil || len(messages) != 1 { + t.Fatalf("read: %#v / %v", messages, err) + } + if ok, retryErr := cache.RequeueFileProcessingMessage(ctx, "worker", messages[0], 1, "retry"); retryErr != nil || !ok { + t.Fatal(retryErr) + } + if err = cache.EnqueueFileProcessing(ctx, 1, "file", 0, ""); err != nil { + t.Fatal(err) + } + if count, err := client.XLen(ctx, fileProcessingStreamName).Result(); err != nil || count != 1 { + t.Fatalf("retry duplicated stream: %d / %v", count, err) + } + messages, err = cache.ReadFileProcessingMessages(ctx, "worker") + if err != nil || len(messages) != 1 { + t.Fatal(err) + } + if ok, settleErr := cache.SettleFileProcessingMessage(ctx, "worker", messages[0]); settleErr != nil || !ok { + t.Fatal(settleErr) + } + if client.Exists(ctx, processingOutstandingKey(1, "file")).Val() != 0 { + t.Fatal("settlement leaked key") + } + if err = cache.EnqueueFileProcessing(ctx, 1, "file", 0, ""); err != nil { + t.Fatal(err) + } + messages, err = cache.ReadFileProcessingMessages(ctx, "worker") + if err != nil || len(messages) != 1 { + t.Fatal(err) + } + if ok, dlqErr := cache.DeadLetterFileProcessingMessage(ctx, "worker", messages[0], "terminal"); dlqErr != nil || !ok { + t.Fatal(dlqErr) + } + if client.Exists(ctx, processingOutstandingKey(1, "file")).Val() != 0 { + t.Fatal("dead letter leaked key") + } + // Stream IDs are only unique within a stream. An embedding settlement + // must not touch an extraction marker even if the IDs happen to match. + if err = cache.EnqueueFileEmbedding(ctx, 1, "file", "sig", "https://embedding.example"); err != nil { + t.Fatal(err) + } + embed, err := cache.ReadFileEmbeddingMessages(ctx, "worker") + if err != nil || len(embed) != 1 { + t.Fatal(err) + } + if err = client.Set(ctx, processingOutstandingKey(1, "file"), embed[0].ID, 0).Err(); err != nil { + t.Fatal(err) + } + if ok, settleErr := cache.SettleFileProcessingMessage(ctx, "worker", embed[0]); settleErr != nil || !ok { + t.Fatal(settleErr) + } + if client.Exists(ctx, processingOutstandingKey(1, "file")).Val() != 1 { + t.Fatal("embedding settled extraction marker") + } + // The legacy parser rejects zero-owner extraction. Invalid-message DLQ + // cleanup must still remove the outstanding marker atomically. + if err = cache.EnqueueFileProcessing(ctx, 0, "invalid", 0, ""); err != nil { + t.Fatal(err) + } + _, err = cache.ReadFileProcessingMessages(ctx, "worker") + if err != nil { + t.Fatal(err) + } + if client.Exists(ctx, processingOutstandingKey(0, "invalid")).Val() != 0 { + t.Fatal("invalid-message DLQ leaked marker") + } + // ParseUint returns a saturated value on overflow and zero on syntax errors. + // Neither value is an authenticated identity for another file's marker. + for _, test := range []struct { + name string + rawUserID string + aliasUserID uint + }{ + {name: "overflow", rawUserID: "18446744073709551616", aliasUserID: ^uint(0)}, + {name: "invalid", rawUserID: "not-a-user", aliasUserID: 0}, + } { + t.Run(test.name, func(t *testing.T) { + fileID := "invalid-" + test.name + id, err := client.XAdd(ctx, &goredis.XAddArgs{Stream: fileProcessingStreamName, Values: map[string]any{ + "user_id": test.rawUserID, "file_id": fileID, "retry": 0, + }}).Result() + if err != nil { + t.Fatal(err) + } + key := processingOutstandingKey(test.aliasUserID, fileID) + if err = client.Set(ctx, key, id, 0).Err(); err != nil { + t.Fatal(err) + } + if _, err = cache.ReadFileProcessingMessages(ctx, "worker"); err != nil { + t.Fatal(err) + } + if value, err := client.Get(ctx, key).Result(); err != nil || value != id { + t.Fatalf("invalid user %q altered marker for %d: %q / %v", test.rawUserID, test.aliasUserID, value, err) + } + }) + } +} + func TestRedisQueueForMessagePreservesLegacySourceQueue(t *testing.T) { legacy := repository.FileProcessingMessage{ Kind: repository.FileProcessingKindEmbedding, diff --git a/backend/internal/infra/mcp/client.go b/backend/internal/infra/mcp/client.go index 2e23412c5..f0ff1e582 100644 --- a/backend/internal/infra/mcp/client.go +++ b/backend/internal/infra/mcp/client.go @@ -18,6 +18,7 @@ import ( "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/outboundhttp" portmcp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/ports/mcp" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/shared/security" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/shared/toolresult" ) const ( @@ -52,7 +53,16 @@ func newMCPHTTPClient(policy security.OutboundPolicy, redirectPolicy security.Ou transport := security.NewOutboundHTTPTransport(policy, defaultConnectTimeout) client := &http.Client{Transport: platformtracing.NewHTTPTransport(transport)} if trustedOrigin != "" { - client.CheckRedirect = outboundhttp.NewRedirectPolicy(redirectPolicy, trustedOrigin, "MCP request") + baseRedirect := outboundhttp.NewRedirectPolicy(redirectPolicy, trustedOrigin, "MCP request") + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) > 0 && via[0].Header.Get(portmcp.FileCreateHeader) != "" { + origin, err := security.HTTPOrigin(req.URL.String()) + if err != nil || origin != trustedOrigin { + return fmt.Errorf("MCP file capability cannot cross origins") + } + } + return baseRedirect(req, via) + } } return outboundhttp.ManagedClient{Client: client, CloseIdleConnections: transport.CloseIdleConnections}, nil } @@ -175,11 +185,14 @@ func (c *Client) rpcWithSession( } for key, value := range cfg.Headers { headerKey := strings.TrimSpace(key) - if headerKey == "" { + if headerKey == "" || strings.EqualFold(headerKey, portmcp.FileCreateHeader) { continue } req.Header.Set(headerKey, strings.TrimSpace(value)) } + if method == "tools/call" && cfg.FileCreateToken != "" { + req.Header.Set(portmcp.FileCreateHeader, cfg.FileCreateToken) + } resp, err := c.httpClients.Do(req, cfg.BaseURL, "") if err != nil { @@ -200,6 +213,7 @@ func (c *Client) rpcWithSession( if notification { return nil, sessionID, nil } + body = []byte(toolresult.RedactSecret(string(body), cfg.FileCreateToken)) result, err := parseRPCResponse(resp.Header.Get("Content-Type"), body) if err != nil { return nil, sessionID, err diff --git a/backend/internal/infra/mcp/client_test.go b/backend/internal/infra/mcp/client_test.go index 57bd01639..b3ce364d1 100644 --- a/backend/internal/infra/mcp/client_test.go +++ b/backend/internal/infra/mcp/client_test.go @@ -3,11 +3,13 @@ package mcp import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" "testing" + portmcp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/ports/mcp" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/shared/security" ) @@ -80,6 +82,54 @@ func TestClientCallToolUsesStreamableHTTPJSONRPC(t *testing.T) { } } +func TestFileCreateCapabilityIsCallOnlyAndCannotRedirect(t *testing.T) { + for _, redirect := range []bool{false, true} { + t.Run(fmt.Sprintf("redirect_%v", redirect), func(t *testing.T) { + targetCalled := false + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { targetCalled = true; w.WriteHeader(200) })) + defer target.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + ID any `json:"id"` + Method string `json:"method"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + want := "" + if req.Method == "tools/call" { + want = "issued-capability" + } + if r.Header.Get(portmcp.FileCreateHeader) != want { + t.Errorf("%s leaked or replaced capability", req.Method) + } + if redirect && req.Method == "tools/call" { + http.Redirect(w, r, target.URL, http.StatusTemporaryRedirect) + return + } + w.Header().Set("Content-Type", "application/json") + if req.Method == "notifications/initialized" { + w.WriteHeader(202) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": map[string]any{}}) + })) + defer server.Close() + client := NewClient(security.NewStrictOutboundPolicy(true)) + cfg := CallConfig{BaseURL: server.URL, FileCreateToken: "issued-capability", Headers: map[string]string{strings.ToLower(portmcp.FileCreateHeader): "static-forgery"}} + _, err := client.CallTool(context.Background(), cfg, CallInput{ToolName: "generate", ArgumentsJSON: `{}`}) + if redirect { + if err == nil || targetCalled { + t.Fatal("capability crossed origins") + } + } else if err != nil { + t.Fatal(err) + } + }) + } +} + func TestClientCallToolRejectsInvalidArgumentsJSON(t *testing.T) { client := NewClient(security.OutboundPolicy{}) _, err := client.CallTool(context.Background(), CallConfig{BaseURL: "http://127.0.0.1/mcp"}, CallInput{ diff --git a/backend/internal/infra/persistence/models/mcp.go b/backend/internal/infra/persistence/models/mcp.go index 89254f8ad..589846879 100644 --- a/backend/internal/infra/persistence/models/mcp.go +++ b/backend/internal/infra/persistence/models/mcp.go @@ -5,15 +5,17 @@ import "time" // MCPServer 存储管理员配置的 MCP 服务。 type MCPServer struct { ControlPlaneModel - Name string `gorm:"size:128;not null;default:'';comment:MCP服务名称"` - BaseURL string `gorm:"size:512;not null;default:'';comment:MCP服务地址"` - AuthTokenEnc string `gorm:"type:text;not null;default:'';comment:加密后的鉴权Token"` - HeadersJSON string `gorm:"type:text;not null;default:'{}';comment:附加请求头JSON"` - Status string `gorm:"size:32;not null;default:'active';index:idx_mcp_servers_status;comment:服务状态(active/inactive)"` - SortOrder int `gorm:"not null;default:0;index:idx_mcp_servers_sort_order;comment:展示顺序"` - ToolCount int `gorm:"not null;default:0;comment:最近发现工具数量"` - LastSyncedAt *time.Time `gorm:"comment:最近同步工具时间"` - LastError string `gorm:"type:text;not null;default:'';comment:最近同步或调用错误"` + Name string `gorm:"size:128;not null;default:'';comment:MCP服务名称"` + BaseURL string `gorm:"size:512;not null;default:'';comment:MCP服务地址"` + AuthTokenEnc string `gorm:"type:text;not null;default:'';comment:加密后的鉴权Token"` + HeadersJSON string `gorm:"type:text;not null;default:'{}';comment:附加请求头JSON"` + Status string `gorm:"size:32;not null;default:'active';index:idx_mcp_servers_status;comment:服务状态(active/inactive)"` + FileCreateEnabled bool `gorm:"not null;default:false;comment:允许工具产物创建用户文件"` + FileCreateEpoch uint `gorm:"not null;default:0;comment:文件创建授权撤销代数"` + SortOrder int `gorm:"not null;default:0;index:idx_mcp_servers_sort_order;comment:展示顺序"` + ToolCount int `gorm:"not null;default:0;comment:最近发现工具数量"` + LastSyncedAt *time.Time `gorm:"comment:最近同步工具时间"` + LastError string `gorm:"type:text;not null;default:'';comment:最近同步或调用错误"` } func (MCPServer) TableName() string { @@ -41,3 +43,25 @@ type MCPTool struct { func (MCPTool) TableName() string { return "mcp_tools" } + +// MCPFileCreateGrant stores only the hash of a short-lived, one-file capability. +// FileID is committed in the same transaction as file metadata and quota. +type MCPFileCreateGrant struct { + TokenHash string `gorm:"primaryKey;size:64"` + ServerID uint `gorm:"not null;index"` + ToolID uint `gorm:"not null"` + UserID uint `gorm:"not null;index"` + Epoch uint `gorm:"not null"` + CallID string `gorm:"size:36;not null"` + RequestID string `gorm:"type:text;not null;default:''"` + ExpiresAt time.Time `gorm:"not null;index"` + CreatedAt time.Time `gorm:"not null"` + MaintainedAt time.Time `gorm:"not null"` + Fingerprint string `gorm:"size:64;not null;default:''"` + CandidateID string `gorm:"size:64;not null"` + StoragePath string `gorm:"size:1024;not null;default:''"` + FileID string `gorm:"size:64;not null;default:''"` + Reused bool `gorm:"not null;default:false"` +} + +func (MCPFileCreateGrant) TableName() string { return "mcp_file_create_grants" } diff --git a/backend/internal/infra/persistence/postgres/conversation/repository.go b/backend/internal/infra/persistence/postgres/conversation/repository.go index ba477144f..9469c47bb 100644 --- a/backend/internal/infra/persistence/postgres/conversation/repository.go +++ b/backend/internal/infra/persistence/postgres/conversation/repository.go @@ -2941,7 +2941,12 @@ func getOrInitQuotaForUpdate(tx *gorm.DB, userID uint, defaultQuotaBytes int64) UsedBytes: 0, ReservedBytes: 0, } - if err := tx.Select("UserID", "QuotaBytes", "UsedBytes", "ReservedBytes").Create("a).Error; err != nil { + if err := tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "user_id"}}, DoNothing: true}). + Select("UserID", "QuotaBytes", "UsedBytes", "ReservedBytes").Create("a).Error; err != nil { + return nil, dberror.Translate(err) + } + // A concurrent first upload may have initialized this account while we waited. + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("user_id = ?", userID).First("a).Error; err != nil { return nil, dberror.Translate(err) } } else { diff --git a/backend/internal/infra/persistence/postgres/conversation/repository_mcp_file_create.go b/backend/internal/infra/persistence/postgres/conversation/repository_mcp_file_create.go new file mode 100644 index 000000000..7028a73f7 --- /dev/null +++ b/backend/internal/infra/persistence/postgres/conversation/repository_mcp_file_create.go @@ -0,0 +1,208 @@ +package conversation + +import ( + "context" + "errors" + "strconv" + "time" + + domainmcp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/mcp" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/models" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +func (r *Repo) CreateFileCreateGrant(ctx context.Context, grant *domainmcp.FileCreateGrant, baseURL string) error { + return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + row := fileCreateGrantModel(grant) + row.MaintainedAt = grant.CreatedAt + if err := validateFileCreateAuthority(tx, &row); err != nil { + return err + } + var server models.MCPServer + if err := tx.First(&server, row.ServerID).Error; err != nil || server.BaseURL != baseURL { + return repository.ErrFileCreateUnauthorized + } + return tx.Create(&row).Error + }) +} + +func (r *Repo) GetFileCreateGrant(ctx context.Context, tokenHash string, serverID uint) (*domainmcp.FileCreateGrant, error) { + var grant domainmcp.FileCreateGrant + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + row, err := loadFileCreateGrant(tx, tokenHash, serverID) + if err != nil { + return err + } + grant = fileCreateGrantDomain(row) + return nil + }) + return &grant, err +} + +func (r *Repo) BindFileCreateGrant(ctx context.Context, tokenHash string, serverID uint, fingerprint string, storagePath string) error { + return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + row, err := lockFileCreateGrant(tx, tokenHash, serverID) + if err != nil { + return err + } + if row.Fingerprint != "" { + if row.Fingerprint != fingerprint { + return repository.ErrFileCreateConflict + } + return nil + } + return tx.Model(&row).Updates(map[string]any{"fingerprint": fingerprint, "storage_path": storagePath}).Error + }) +} + +func (r *Repo) WithFileCreateGrant(ctx context.Context, tokenHash string, serverID uint, fn func(*domainmcp.FileCreateGrant, repository.UploadRepository) error) error { + return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + row, err := lockFileCreateGrant(tx, tokenHash, serverID) + if err != nil { + return err + } + grant := fileCreateGrantDomain(row) + if err = fn(&grant, NewRepo(tx)); err != nil { + return err + } + return tx.Model(&row).Updates(map[string]any{"file_id": grant.FileID, "reused": grant.Reused}).Error + }) +} + +func lockFileCreateGrant(tx *gorm.DB, tokenHash string, serverID uint) (models.MCPFileCreateGrant, error) { + // SQLite has one writer, not row locks. Acquire its write lock before reading. + if tx.Dialector.Name() == "sqlite" { + if err := tx.Model(&models.MCPFileCreateGrant{}).Where("token_hash = ? AND server_id = ?", tokenHash, serverID). + Update("token_hash", tokenHash).Error; err != nil { + return models.MCPFileCreateGrant{}, err + } + } + return loadFileCreateGrant(tx, tokenHash, serverID) +} + +func loadFileCreateGrant(tx *gorm.DB, tokenHash string, serverID uint) (models.MCPFileCreateGrant, error) { + var row models.MCPFileCreateGrant + if err := tx.First(&row, "token_hash = ? AND server_id = ?", tokenHash, serverID).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return row, repository.ErrFileCreateUnauthorized + } + return row, err + } + if !row.ExpiresAt.After(time.Now()) { + return row, repository.ErrFileCreateUnauthorized + } + // Lock authority before the receipt, matching disable/update transaction order. + if err := validateFileCreateAuthority(tx, &row); err != nil { + return row, err + } + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&row, "token_hash = ? AND server_id = ?", tokenHash, serverID).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return row, repository.ErrFileCreateUnauthorized + } + return row, err + } + if !row.ExpiresAt.After(time.Now()) { + return row, repository.ErrFileCreateUnauthorized + } + return row, nil +} + +func validateFileCreateAuthority(tx *gorm.DB, row *models.MCPFileCreateGrant) error { + // Read the persisted global switch too; another instance may have a stale runtime snapshot. + var setting models.SystemSetting + err := tx.Clauses(clause.Locking{Strength: "SHARE"}).Where("namespace = ? AND key = ?", "mcp", "mcp_enable").First(&setting).Error + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + if err == nil { + enabled, parseErr := strconv.ParseBool(setting.Value) + if parseErr != nil || !enabled { + return repository.ErrFileCreateUnauthorized + } + } + var server models.MCPServer + if err = tx.Clauses(clause.Locking{Strength: "SHARE"}).First(&server, row.ServerID).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return repository.ErrFileCreateUnauthorized + } + return err + } + if server.Status != "active" || !server.FileCreateEnabled || server.FileCreateEpoch != row.Epoch { + return repository.ErrFileCreateUnauthorized + } + var tool models.MCPTool + if err = tx.Clauses(clause.Locking{Strength: "SHARE"}).First(&tool, "id = ? AND server_id = ?", row.ToolID, row.ServerID).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return repository.ErrFileCreateUnauthorized + } + return err + } + var user models.User + if err = tx.Clauses(clause.Locking{Strength: "SHARE"}).First(&user, row.UserID).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return repository.ErrFileCreateUnauthorized + } + return err + } + if row.UserID == 0 || user.Status != "active" || tool.Status != "active" { + return repository.ErrFileCreateUnauthorized + } + return nil +} + +func (r *Repo) ListFileCreateMaintenance(ctx context.Context, now time.Time) ([]domainmcp.FileCreateGrant, error) { + var rows []models.MCPFileCreateGrant + // Rotate visited receipts so a stuck storage/queue cannot starve cleanup. + err := r.db.WithContext(ctx).Where("expires_at <= ? OR file_id <> ''", now).Order("maintained_at").Limit(100).Find(&rows).Error + if err == nil && len(rows) > 0 { + hashes := make([]string, 0, len(rows)) + for _, row := range rows { + hashes = append(hashes, row.TokenHash) + } + err = r.db.WithContext(ctx).Model(&models.MCPFileCreateGrant{}).Where("token_hash IN ?", hashes).Update("maintained_at", now).Error + } + result := make([]domainmcp.FileCreateGrant, 0, len(rows)) + for _, row := range rows { + result = append(result, fileCreateGrantDomain(row)) + } + return result, err +} + +func (r *Repo) ExpireFileCreateGrant(ctx context.Context, hash string, now time.Time, remove func(string) error) error { + return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if tx.Dialector.Name() == "sqlite" { + if err := tx.Model(&models.MCPFileCreateGrant{}).Where("token_hash = ?", hash).Update("token_hash", hash).Error; err != nil { + return err + } + } + var row models.MCPFileCreateGrant + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&row, "token_hash = ?", hash).Error; err != nil { + return err + } + if row.ExpiresAt.After(now) { + return nil + } + if row.StoragePath != "" { + var references int64 + if err := tx.Model(&models.FileObject{}).Where("storage_path = ? AND status = ?", row.StoragePath, "active").Count(&references).Error; err != nil { + return err + } + if references == 0 { + if err := remove(row.StoragePath); err != nil { + return err + } + } + } + return tx.Delete(&row).Error + }) +} + +func fileCreateGrantModel(g *domainmcp.FileCreateGrant) models.MCPFileCreateGrant { + return models.MCPFileCreateGrant{TokenHash: g.TokenHash, ServerID: g.ServerID, ToolID: g.ToolID, UserID: g.UserID, Epoch: g.Epoch, CallID: g.CallID, RequestID: g.RequestID, ExpiresAt: g.ExpiresAt, CreatedAt: g.CreatedAt, Fingerprint: g.Fingerprint, CandidateID: g.CandidateID, StoragePath: g.StoragePath, FileID: g.FileID, Reused: g.Reused} +} + +func fileCreateGrantDomain(g models.MCPFileCreateGrant) domainmcp.FileCreateGrant { + return domainmcp.FileCreateGrant{TokenHash: g.TokenHash, ServerID: g.ServerID, ToolID: g.ToolID, UserID: g.UserID, Epoch: g.Epoch, CallID: g.CallID, RequestID: g.RequestID, ExpiresAt: g.ExpiresAt, CreatedAt: g.CreatedAt, Fingerprint: g.Fingerprint, CandidateID: g.CandidateID, StoragePath: g.StoragePath, FileID: g.FileID, Reused: g.Reused} +} diff --git a/backend/internal/infra/persistence/postgres/conversation/repository_processing.go b/backend/internal/infra/persistence/postgres/conversation/repository_processing.go index a8972b0a1..422bf03bb 100644 --- a/backend/internal/infra/persistence/postgres/conversation/repository_processing.go +++ b/backend/internal/infra/persistence/postgres/conversation/repository_processing.go @@ -15,14 +15,20 @@ func (r *Repo) UpdateFileObjectProcessingState(ctx context.Context, item *domain if item == nil { return nil } - result := r.db.WithContext(ctx). + query := r.db.WithContext(ctx). Model(&models.FileObject{}). - Where("id = ? AND user_id = ?", item.FileObjectID, item.UserID). - Updates(fileObjectProcessingStateUpdates(item)) + Where("id = ? AND user_id = ?", item.FileObjectID, item.UserID) + if item.ExpectedStatus != "" { + query = query.Where("status = ? AND processing_status = ?", "active", item.ExpectedStatus) + } + result := query.Updates(fileObjectProcessingStateUpdates(item)) if result.Error != nil { return dberror.Translate(result.Error) } if result.RowsAffected == 0 { + if item.ExpectedStatus != "" { + return repository.ErrConflict + } return repository.ErrNotFound } return nil diff --git a/backend/internal/infra/persistence/postgres/mcp/repository.go b/backend/internal/infra/persistence/postgres/mcp/repository.go index 8c8909995..92b512fa7 100644 --- a/backend/internal/infra/persistence/postgres/mcp/repository.go +++ b/backend/internal/infra/persistence/postgres/mcp/repository.go @@ -40,12 +40,13 @@ func (r *Repo) CreateServer(ctx context.Context, input repository.CreateMCPServe return err } item := models.MCPServer{ - Name: input.Name, - BaseURL: input.BaseURL, - AuthTokenEnc: input.AuthTokenEnc, - HeadersJSON: input.HeadersJSON, - Status: input.Status, - SortOrder: maxSortOrder + 100, + Name: input.Name, + BaseURL: input.BaseURL, + AuthTokenEnc: input.AuthTokenEnc, + HeadersJSON: input.HeadersJSON, + Status: input.Status, + FileCreateEnabled: input.FileCreateEnabled, + SortOrder: maxSortOrder + 100, } if err := tx.Create(&item).Error; err != nil { return err @@ -78,6 +79,13 @@ func (r *Repo) UpdateServer(ctx context.Context, serverID uint, input repository if input.LastError != nil { updates["last_error"] = *input.LastError } + if input.FileCreateEnabled != nil { + updates["file_create_enabled"] = *input.FileCreateEnabled + } + // Re-enabling must not revive an old capability. Connection changes revoke too. + if input.Status != nil || input.BaseURL != nil || input.AuthTokenEnc != nil || input.HeadersJSON != nil || input.FileCreateEnabled != nil { + updates["file_create_epoch"] = gorm.Expr("file_create_epoch + 1") + } if len(updates) > 0 { if err := r.db.WithContext(ctx).Model(&models.MCPServer{}).Where("id = ?", serverID).Updates(updates).Error; err != nil { return nil, err @@ -166,6 +174,10 @@ func (r *Repo) GetServer(ctx context.Context, serverID uint) (*domainmcp.Server, func (r *Repo) DeleteServer(ctx context.Context, serverID uint) error { return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var server models.MCPServer + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&server, serverID).Error; err != nil { + return err + } toolIDs := make([]uint, 0) if err := tx.Model(&models.MCPTool{}).Where("server_id = ?", serverID).Pluck("id", &toolIDs).Error; err != nil { return err @@ -183,6 +195,10 @@ func (r *Repo) DeleteServer(ctx context.Context, serverID uint) error { func (r *Repo) ReplaceServerTools(ctx context.Context, serverID uint, tools []domainmcp.Tool, overwriteCustomizedMetadata bool) error { now := time.Now() return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var server models.MCPServer + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&server, serverID).Error; err != nil { + return err + } var maxSortOrder int if err := tx.Model(&models.MCPTool{}). Where("server_id = ?", serverID). @@ -369,6 +385,11 @@ func (r *Repo) UpdateTool(ctx context.Context, toolID uint, input repository.Upd return err } } + if input.Status != nil && *input.Status != "active" { + if err := tx.Model(&models.MCPFileCreateGrant{}).Where("tool_id = ?", toolID).Update("expires_at", time.Unix(0, 0)).Error; err != nil { + return err + } + } result = toDomainTool(row) return nil }) @@ -379,10 +400,15 @@ func (r *Repo) UpdateTool(ctx context.Context, toolID uint, input repository.Upd } func (r *Repo) UpdateServerToolsStatus(ctx context.Context, serverID uint, toolIDs []uint, status string) ([]domainmcp.Tool, error) { - if err := r.db.WithContext(ctx). - Model(&models.MCPTool{}). - Where("server_id = ? AND id IN ?", serverID, toolIDs). - Update("status", status).Error; err != nil { + if err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Model(&models.MCPTool{}).Where("server_id = ? AND id IN ?", serverID, toolIDs).Update("status", status).Error; err != nil { + return err + } + if status != "active" { + return tx.Model(&models.MCPFileCreateGrant{}).Where("server_id = ? AND tool_id IN ?", serverID, toolIDs).Update("expires_at", time.Unix(0, 0)).Error + } + return nil + }); err != nil { return nil, err } return r.ListTools(ctx, serverID, false) @@ -485,19 +511,21 @@ func (r *Repo) ReorderServersWithTools(ctx context.Context, order []repository.R func toDomainServer(row models.MCPServer) domainmcp.Server { return domainmcp.Server{ - ID: row.ID, - Name: row.Name, - BaseURL: row.BaseURL, - AuthTokenEnc: row.AuthTokenEnc, - HeadersJSON: row.HeadersJSON, - Status: row.Status, - SortOrder: row.SortOrder, - ToolCount: row.ToolCount, - ActiveToolCount: 0, - LastSyncedAt: row.LastSyncedAt, - LastError: row.LastError, - CreatedAt: row.CreatedAt, - UpdatedAt: row.UpdatedAt, + ID: row.ID, + Name: row.Name, + BaseURL: row.BaseURL, + AuthTokenEnc: row.AuthTokenEnc, + HeadersJSON: row.HeadersJSON, + Status: row.Status, + FileCreateEnabled: row.FileCreateEnabled, + FileCreateEpoch: row.FileCreateEpoch, + SortOrder: row.SortOrder, + ToolCount: row.ToolCount, + ActiveToolCount: 0, + LastSyncedAt: row.LastSyncedAt, + LastError: row.LastError, + CreatedAt: row.CreatedAt, + UpdatedAt: row.UpdatedAt, } } diff --git a/backend/internal/infra/persistence/postgres/mcp/repository_sqlite_test.go b/backend/internal/infra/persistence/postgres/mcp/repository_sqlite_test.go index 0949e1c6a..fe4a2e033 100644 --- a/backend/internal/infra/persistence/postgres/mcp/repository_sqlite_test.go +++ b/backend/internal/infra/persistence/postgres/mcp/repository_sqlite_test.go @@ -505,7 +505,7 @@ func openMCPSQLiteTestDB(t *testing.T) *gorm.DB { if err != nil { t.Fatalf("open sqlite: %v", err) } - if err := db.AutoMigrate(&models.MCPServer{}, &models.MCPTool{}, &models.ConversationProjectMCPTool{}); err != nil { + if err := db.AutoMigrate(&models.MCPServer{}, &models.MCPTool{}, &models.MCPFileCreateGrant{}, &models.ConversationProjectMCPTool{}); err != nil { t.Fatalf("migrate sqlite: %v", err) } return db diff --git a/backend/internal/infra/persistence/postgres/settings/repository.go b/backend/internal/infra/persistence/postgres/settings/repository.go index 9d6af5be5..43c2c9f8d 100644 --- a/backend/internal/infra/persistence/postgres/settings/repository.go +++ b/backend/internal/infra/persistence/postgres/settings/repository.go @@ -2,6 +2,8 @@ package settings import ( "context" + "strconv" + "time" domainsettings "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/settings" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/dberror" @@ -44,12 +46,23 @@ func (r *Repo) Upsert(ctx context.Context, items []domainsettings.SystemSetting) return nil } dbItems := toModelSystemSettings(items) - return dberror.Translate(r.db.WithContext(ctx). - Clauses(clause.OnConflict{ + return dberror.Translate(r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "namespace"}, {Name: "key"}}, DoUpdates: clause.AssignmentColumns([]string{"value", "updated_at"}), - }). - Create(&dbItems).Error) + }).Create(&dbItems).Error; err != nil { + return err + } + for _, item := range items { + if item.Namespace == "mcp" && item.Key == "mcp_enable" { + enabled, err := strconv.ParseBool(item.Value) + if err != nil || !enabled { + return tx.Model(&models.MCPFileCreateGrant{}).Where("expires_at > ?", time.Unix(0, 0)).Update("expires_at", time.Unix(0, 0)).Error + } + } + } + return nil + })) } // UpsertWithDescription 批量写入含描述的配置（种子用）。 diff --git a/backend/internal/infra/persistence/postgres/user/repository.go b/backend/internal/infra/persistence/postgres/user/repository.go index e447137b1..fd50ec507 100644 --- a/backend/internal/infra/persistence/postgres/user/repository.go +++ b/backend/internal/infra/persistence/postgres/user/repository.go @@ -749,11 +749,15 @@ func (r *Repo) ResetLoginFailure(ctx context.Context, userID uint) error { // UpdateUserStatus 更新用户状态。 func (r *Repo) UpdateUserStatus(ctx context.Context, userID uint, status string) error { - return translateError(r.db.WithContext(ctx). - Model(&models.User{}). - Where("id = ?", userID). - Update("status", status). - Error) + return translateError(r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Model(&models.User{}).Where("id = ?", userID).Update("status", status).Error; err != nil { + return err + } + if status != models.UserStatusActive { + return tx.Model(&models.MCPFileCreateGrant{}).Where("user_id = ?", userID).Update("expires_at", time.Unix(0, 0)).Error + } + return nil + })) } // ResetPasswordByAdmin 重置用户密码并更新凭据元信息。 @@ -853,6 +857,10 @@ func (r *Repo) ListLatestSessionActivityByUserIDs(ctx context.Context, userIDs [ // DeleteAccountHard 删除用户主记录及主要用户域数据。 func (r *Repo) DeleteAccountHard(ctx context.Context, userID uint) error { return translateError(r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var owner models.User + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&owner, userID).Error; err != nil { + return err + } var builtinFileReferences int64 if err := tx.Table("knowledge_base_files AS kbf"). Joins("JOIN knowledge_bases AS kb ON kb.id = kbf.knowledge_base_id"). @@ -1034,6 +1042,10 @@ func (r *Repo) DeleteAccountHard(ctx context.Context, userID uint) error { { label: "file_objects", run: func(db *gorm.DB) error { + // Revoke immediately; maintenance retains candidate keys for safe cleanup. + if err := db.Model(&models.MCPFileCreateGrant{}).Where("user_id = ?", userID).Update("expires_at", time.Unix(0, 0)).Error; err != nil { + return err + } return db.Unscoped().Where("user_id = ?", userID).Delete(&models.FileObject{}).Error }, }, diff --git a/backend/internal/infra/persistence/schema/schema.go b/backend/internal/infra/persistence/schema/schema.go index a1e3d0917..86b198c54 100644 --- a/backend/internal/infra/persistence/schema/schema.go +++ b/backend/internal/infra/persistence/schema/schema.go @@ -32,6 +32,7 @@ func Models() []any { &models.LLMUserProvider{}, &models.MCPServer{}, &models.MCPTool{}, + &models.MCPFileCreateGrant{}, &models.Conversation{}, &models.ConversationProject{}, &models.ConversationShare{}, diff --git a/backend/internal/ports/mcp/contract.go b/backend/internal/ports/mcp/contract.go index dc69c8f71..32126ca52 100644 --- a/backend/internal/ports/mcp/contract.go +++ b/backend/internal/ports/mcp/contract.go @@ -3,12 +3,17 @@ package mcp import "encoding/json" +const FileCreateHeader = "X-Deeix-File-Create-Token" + // CallConfig 定义 MCP 调用配置。 type CallConfig struct { BaseURL string AuthToken string TimeoutMS int Headers map[string]string + // FileCreateToken is issued by DEEIX, only sent on tools/call, never discovery. + FileCreateToken string + FileCreateEpoch uint } // CallInput 定义 MCP 工具调用入参。 diff --git a/backend/internal/repository/mcp.go b/backend/internal/repository/mcp.go index 1fc240a7f..b1d3f0c77 100644 --- a/backend/internal/repository/mcp.go +++ b/backend/internal/repository/mcp.go @@ -8,21 +8,23 @@ import ( // CreateMCPServerInput 定义创建 MCP 服务字段。 type CreateMCPServerInput struct { - Name string - BaseURL string - AuthTokenEnc string - HeadersJSON string - Status string + Name string + BaseURL string + AuthTokenEnc string + HeadersJSON string + Status string + FileCreateEnabled bool } // UpdateMCPServerInput 定义更新 MCP 服务字段。 type UpdateMCPServerInput struct { - Name *string - BaseURL *string - AuthTokenEnc *string - HeadersJSON *string - Status *string - LastError *string + Name *string + BaseURL *string + AuthTokenEnc *string + HeadersJSON *string + Status *string + LastError *string + FileCreateEnabled *bool } // UpdateMCPToolInput 定义更新 MCP 工具字段。 diff --git a/backend/internal/repository/mcp_file_create.go b/backend/internal/repository/mcp_file_create.go new file mode 100644 index 000000000..11e996e1d --- /dev/null +++ b/backend/internal/repository/mcp_file_create.go @@ -0,0 +1,24 @@ +package repository + +import ( + "context" + "errors" + "time" + + domainmcp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/mcp" +) + +var ( + ErrFileCreateUnauthorized = errors.New("file create capability is invalid, expired or revoked") + ErrFileCreateConflict = errors.New("file create capability is already bound to another file") +) + +// MCPFileCreateRepository is the one-file receipt boundary, not a general permission store. +type MCPFileCreateRepository interface { + CreateFileCreateGrant(context.Context, *domainmcp.FileCreateGrant, string) error + GetFileCreateGrant(context.Context, string, uint) (*domainmcp.FileCreateGrant, error) + BindFileCreateGrant(context.Context, string, uint, string, string) error + WithFileCreateGrant(context.Context, string, uint, func(*domainmcp.FileCreateGrant, UploadRepository) error) error + ListFileCreateMaintenance(context.Context, time.Time) ([]domainmcp.FileCreateGrant, error) + ExpireFileCreateGrant(context.Context, string, time.Time, func(string) error) error +} diff --git a/backend/internal/shared/toolresult/redact.go b/backend/internal/shared/toolresult/redact.go new file mode 100644 index 000000000..11556e37c --- /dev/null +++ b/backend/internal/shared/toolresult/redact.go @@ -0,0 +1,80 @@ +package toolresult + +import ( + "encoding/json" + "io" + "strings" +) + +// RedactSecret strips an issued bearer from JSON values/keys (including JSON +// text blocks) after decoding escapes, before tool results are saved or shown. +func RedactSecret(raw string, secret string) string { + if secret == "" { + return raw + } + text := strings.ReplaceAll(raw, secret, "[redacted]") + decoder := json.NewDecoder(strings.NewReader(text)) + decoder.UseNumber() + var value any + if decoder.Decode(&value) != nil || decoder.Decode(&struct{}{}) != io.EOF { + return redactEmbeddedSecret(text, secret) + } + encoded, err := json.Marshal(redactSecretValue(value, secret)) + if err != nil { + return text + } + return string(encoded) +} + +func redactEmbeddedSecret(text string, secret string) string { + var result strings.Builder + for offset := 0; offset < len(text); { + next := strings.IndexAny(text[offset:], "{[") + if next < 0 { + result.WriteString(text[offset:]) + break + } + next += offset + result.WriteString(text[offset:next]) + decoder := json.NewDecoder(strings.NewReader(text[next:])) + decoder.UseNumber() + var value any + if decoder.Decode(&value) == nil { + encoded, err := json.Marshal(redactSecretValue(value, secret)) + if err == nil { + result.Write(encoded) + offset = next + int(decoder.InputOffset()) + continue + } + } + // Presentation only decodes complete JSON blocks. Skip malformed lines + // once, bounding work on arbitrary tool prose with many opening brackets. + lineEnd := strings.IndexByte(text[next:], '\n') + if lineEnd < 0 { + result.WriteString(text[next:]) + break + } + lineEnd += next + 1 + result.WriteString(text[next:lineEnd]) + offset = lineEnd + } + return result.String() +} + +func redactSecretValue(value any, secret string) any { + switch item := value.(type) { + case string: + return RedactSecret(item, secret) + case []any: + for i, child := range item { + item[i] = redactSecretValue(child, secret) + } + case map[string]any: + result := make(map[string]any, len(item)) + for key, child := range item { + result[strings.ReplaceAll(key, secret, "[redacted]")] = redactSecretValue(child, secret) + } + return result + } + return value +} diff --git a/backend/internal/shared/toolresult/redact_test.go b/backend/internal/shared/toolresult/redact_test.go new file mode 100644 index 000000000..4d6582bcb --- /dev/null +++ b/backend/internal/shared/toolresult/redact_test.go @@ -0,0 +1,24 @@ +package toolresult + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestRedactSecretDecodesToolJSONAndTextBlocks(t *testing.T) { + const secret = "dxf1_sensitive" + raw := `{"dxf\u0031_sensitive":"dxf\u0031_sensitive","content":[{"type":"text","text":"{\"credential\":\"dxf\\u0031_sensitive\"}"}],"number":9007199254740993}` + output := RedactSecret(raw, secret) + if strings.Contains(output, "sensitive") || !strings.Contains(output, "9007199254740993") || !json.Valid([]byte(output)) { + t.Fatalf("unsafe output: %s", output) + } + if RedactSecret("failed: "+secret, secret) != "failed: [redacted]" { + t.Fatal("plain error leaked bearer") + } + for _, raw := range []string{`mcp tool error: {"credential":"dxf\u0031_sensitive"}`, "Sources:\n" + `{"credential":"dxf\u0031_sensitive"}`} { + if output := RedactSecret(raw, secret); strings.Contains(output, "sensitive") { + t.Fatalf("embedded JSON leaked: %s", output) + } + } +} diff --git a/backend/internal/transport/http/mcp/dto.go b/backend/internal/transport/http/mcp/dto.go index 3e29916cf..8efb4da7c 100644 --- a/backend/internal/transport/http/mcp/dto.go +++ b/backend/internal/transport/http/mcp/dto.go @@ -8,6 +8,7 @@ type ServerResponse struct { BaseURL string `json:"baseURL"` HeadersJSON string `json:"headersJSON"` Status string `json:"status"` + FileCreateEnabled bool `json:"fileCreateEnabled"` SortOrder int `json:"sortOrder"` ToolCount int `json:"toolCount"` ActiveToolCount int `json:"activeToolCount"` @@ -43,6 +44,8 @@ type CreateServerRequest struct { AuthToken string `json:"authToken,omitempty"` HeadersJSON string `json:"headersJSON,omitempty"` Status string `json:"status,omitempty"` + // FileCreateEnabled authorizes automatic creation of one user file per persistent tool call. + FileCreateEnabled *bool `json:"fileCreateEnabled,omitempty"` } type UpdateToolRequest struct { @@ -101,6 +104,24 @@ type ErrorDoc struct { ErrorMsg string `json:"errorMsg"` } +type FileCreateResponse struct { + FileID string `json:"file_id"` + FileName string `json:"file_name"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Reused bool `json:"reused"` + Replayed bool `json:"replayed"` + ProcessingStatus string `json:"processing_status"` + ExtractStatus string `json:"extract_status"` + EmbedStatus string `json:"embed_status"` + ProcessingReady bool `json:"processing_ready"` +} + +type FileCreateResponseDoc struct { + ErrorMsg string `json:"errorMsg"` + Data FileCreateResponse `json:"data"` +} + // ServerListResponseDoc 包裹 MCP 服务列表响应。 type ServerListResponseDoc struct { ErrorMsg string `json:"errorMsg"` diff --git a/backend/internal/transport/http/mcp/handler.go b/backend/internal/transport/http/mcp/handler.go index 5033fb7ac..e772a5097 100644 --- a/backend/internal/transport/http/mcp/handler.go +++ b/backend/internal/transport/http/mcp/handler.go @@ -14,9 +14,12 @@ import ( ) type Handler struct { - service *appmcp.Service + service *appmcp.Service + fileCreate *appmcp.FileCreateService } +func (h *Handler) SetFileCreateService(service *appmcp.FileCreateService) { h.fileCreate = service } + func NewHandler(service *appmcp.Service) *Handler { return &Handler{service: service} } @@ -84,11 +87,12 @@ func (h *Handler) CreateServer(c *gin.Context) { return } item, err := h.service.CreateServer(c.Request.Context(), appmcp.ServerInput{ - Name: req.Name, - BaseURL: req.BaseURL, - AuthToken: req.AuthToken, - HeadersJSON: req.HeadersJSON, - Status: req.Status, + Name: req.Name, + BaseURL: req.BaseURL, + AuthToken: req.AuthToken, + HeadersJSON: req.HeadersJSON, + Status: req.Status, + FileCreateEnabled: req.FileCreateEnabled, }) if err != nil { writeServiceError(c, err) @@ -121,11 +125,12 @@ func (h *Handler) UpdateServer(c *gin.Context) { return } item, err := h.service.UpdateServer(c.Request.Context(), serverID, appmcp.ServerInput{ - Name: req.Name, - BaseURL: req.BaseURL, - AuthToken: req.AuthToken, - HeadersJSON: req.HeadersJSON, - Status: req.Status, + Name: req.Name, + BaseURL: req.BaseURL, + AuthToken: req.AuthToken, + HeadersJSON: req.HeadersJSON, + Status: req.Status, + FileCreateEnabled: req.FileCreateEnabled, }) if err != nil { writeServiceError(c, err) @@ -379,6 +384,7 @@ func writeServiceError(c *gin.Context, err error) { func toServerResponse(item domainmcp.Server) ServerResponse { return ServerResponse{ + FileCreateEnabled: item.FileCreateEnabled, ID: item.ID, Name: item.Name, BaseURL: item.BaseURL, diff --git a/backend/internal/transport/http/mcp/handler_file_create.go b/backend/internal/transport/http/mcp/handler_file_create.go new file mode 100644 index 000000000..07253fa76 --- /dev/null +++ b/backend/internal/transport/http/mcp/handler_file_create.go @@ -0,0 +1,121 @@ +package mcp + +import ( + "errors" + "net/http" + "strings" + + appconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/conversation" + appmcp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/mcp" + appupload "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/upload" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/shared/response" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/middleware" + "github.com/gin-gonic/gin" +) + +func (h *Handler) authorizeFileCreate(c *gin.Context) { + serverID, ok := parseIDParam(c, "id", errInvalidMCPServerID) + if !ok { + c.Abort() + return + } + authorization := c.GetHeader("Authorization") + token := strings.TrimPrefix(authorization, "Bearer ") + if !strings.HasPrefix(authorization, "Bearer ") { + token = "" + } + if h.fileCreate == nil { + response.ErrorFrom(c, http.StatusUnauthorized, appmcp.ErrFileCreateUnauthorized) + c.Abort() + return + } + grant, err := h.fileCreate.Authorize(c.Request.Context(), token, serverID) + if err != nil { + writeFileCreateError(c, err) + c.Abort() + return + } + c.Set(middleware.ContextKeyUserID, grant.UserID) + c.Next() +} + +// CreateFile godoc +// @Summary 创建 MCP 工具产物文件 +// @Description 仅接受 DEEIX 在 tools/call 签发的一文件 capability，不接受用户 JWT 或 MCP 共享 HMAC 上下文。重复相同文件返回同一 file_id；创建不代表提取或向量化已完成。 +// @Tags mcp +// @Accept multipart/form-data +// @Produce json +// @Param id path int true "登记的 MCP 服务 ID" +// @Param Authorization header string true "Bearer " +// @Param file formData file true "唯一文件，不接受其他字段" +// @Success 200 {object} FileCreateResponseDoc +// @Failure 400 {object} ErrorDoc +// @Failure 401 {object} ErrorDoc +// @Failure 404 {object} ErrorDoc +// @Failure 409 {object} ErrorDoc +// @Failure 413 {object} ErrorDoc +// @Failure 429 {object} ErrorDoc +// @Failure 500 {object} ErrorDoc +// @Router /mcp/servers/{id}/files [post] +func (h *Handler) CreateFile(c *gin.Context) { + serverID, ok := parseIDParam(c, "id", errInvalidMCPServerID) + if !ok { + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.fileCreate.MaxUploadBytes()+(1<<20)) + err := c.Request.ParseMultipartForm(8 << 20) + if c.Request.MultipartForm != nil { + defer c.Request.MultipartForm.RemoveAll() + } //nolint:errcheck + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + response.ErrorFrom(c, http.StatusRequestEntityTooLarge, appconversation.ErrFileTooLarge) + } else { + response.ErrorFrom(c, http.StatusBadRequest, appmcp.ErrFileCreateInvalidInput) + } + return + } + form := c.Request.MultipartForm + if form == nil || len(form.Value) != 0 || len(form.File) != 1 || len(form.File["file"]) != 1 || c.Request.URL.RawQuery != "" { + response.ErrorFrom(c, http.StatusBadRequest, appmcp.ErrFileCreateInvalidInput) + return + } + part := form.File["file"][0] + reader, err := part.Open() + if err != nil { + response.ErrorFrom(c, http.StatusBadRequest, appmcp.ErrFileCreateInvalidInput) + return + } + defer reader.Close() //nolint:errcheck + result, err := h.fileCreate.Create(c.Request.Context(), strings.TrimPrefix(c.GetHeader("Authorization"), "Bearer "), serverID, appupload.TemporaryFileInput{ + FileName: part.Filename, MimeType: part.Header.Get("Content-Type"), DeclaredSize: part.Size, Reader: reader, + }) + if err != nil { + writeFileCreateError(c, err) + return + } + file := result.File + response.Success(c, FileCreateResponse{ + FileID: file.FileID, FileName: file.FileName, SizeBytes: file.SizeBytes, SHA256: file.SHA256, + Reused: result.Reused, Replayed: result.Replayed, ProcessingStatus: file.ProcessingStatus, + ExtractStatus: file.ExtractStatus, EmbedStatus: file.EmbedStatus, ProcessingReady: file.ProcessingReady, + }) +} + +func writeFileCreateError(c *gin.Context, err error) { + status := http.StatusInternalServerError + switch { + case errors.Is(err, appmcp.ErrFileCreateUnauthorized): + status = http.StatusUnauthorized + case errors.Is(err, appmcp.ErrFileCreateConflict), errors.Is(err, appconversation.ErrStorageQuotaExceeded): + status = http.StatusConflict + case errors.Is(err, appmcp.ErrFileCreateResultGone): + status = http.StatusNotFound + case errors.Is(err, appconversation.ErrFileTooLarge): + status = http.StatusRequestEntityTooLarge + case errors.Is(err, appconversation.ErrMIMEBlocked), errors.Is(err, appconversation.ErrDangerousMIMEType), errors.Is(err, appconversation.ErrInvalidFileReference), errors.Is(err, appmcp.ErrFileCreateInvalidInput): + status = http.StatusBadRequest + } + response.ErrorFrom(c, status, err) +} diff --git a/backend/internal/transport/http/mcp/handler_file_create_test.go b/backend/internal/transport/http/mcp/handler_file_create_test.go new file mode 100644 index 000000000..d93385b25 --- /dev/null +++ b/backend/internal/transport/http/mcp/handler_file_create_test.go @@ -0,0 +1,667 @@ +package mcp + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + appaudit "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/audit" + appconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/conversation" + appmcp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/mcp" + appprocessing "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/processing" + appupload "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/upload" + domainconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" + domainmcp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/mcp" + domainsettings "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/settings" + memorycache "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/cache/memory" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/config" + storeinfra "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/objectstorage" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/models" + conversationrepo "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/postgres/conversation" + mcprepo "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/postgres/mcp" + settingsrepo "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/postgres/settings" + userrepo "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/postgres/user" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/pkg/mcpauth" + portstore "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/ports/objectstorage" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/middleware" + "github.com/gin-gonic/gin" + "go.uber.org/zap" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +type fileCreateFixture struct { + db *gorm.DB + runtime *config.Runtime + repo *conversationrepo.Repo + servers *mcprepo.Repo + service *appmcp.FileCreateService + uploads *appupload.Service + store portstore.Store + root string + dsn string + server models.MCPServer + tool models.MCPTool +} + +type fileCreateStoreProvider struct{ store portstore.Store } + +func (p fileCreateStoreProvider) Open(context.Context) (portstore.Store, error) { return p.store, nil } + +func openFileCreateDB(t *testing.T, dsn string) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + t.Fatal(err) + } + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + sqlDB.SetMaxOpenConns(1) + t.Cleanup(func() { _ = sqlDB.Close() }) + return db +} + +func newFileCreateFixture(t *testing.T) *fileCreateFixture { + t.Helper() + f := &fileCreateFixture{root: t.TempDir()} + f.dsn = filepath.ToSlash(filepath.Join(t.TempDir(), "files.sqlite")) + "?_busy_timeout=10000&_journal_mode=WAL" + f.db = openFileCreateDB(t, f.dsn) + return initializeFileCreateFixture(t, f) +} + +func initializeFileCreateFixture(t *testing.T, f *fileCreateFixture) *fileCreateFixture { + t.Helper() + if err := f.db.AutoMigrate(&models.MCPServer{}, &models.MCPTool{}, &models.MCPFileCreateGrant{}, &models.SystemSetting{}, &models.User{}, &models.FileObject{}, &models.UserStorageQuota{}); err != nil { + t.Fatal(err) + } + for id := uint(1); id <= 2; id++ { + user := models.User{BaseModel: models.BaseModel{ID: id}, Username: fmt.Sprintf("user%d", id), PublicID: fmt.Sprintf("usr%d", id), Status: "active"} + if err := f.db.Create(&user).Error; err != nil { + t.Fatal(err) + } + } + f.server = models.MCPServer{Name: "generator", BaseURL: "https://generator.example/mcp", Status: "active", FileCreateEnabled: true} + if err := f.db.Create(&f.server).Error; err != nil { + t.Fatal(err) + } + f.tool = models.MCPTool{ServerID: f.server.ID, Name: "generate", Status: "active"} + if err := f.db.Create(&f.tool).Error; err != nil { + t.Fatal(err) + } + f.runtime = config.NewRuntime(config.Config{MCPEnable: true, MaxUploadFileBytes: 1024, UserStorageQuotaBytes: 1024, FileAllowedMIMETypes: "text/plain", MCPUserContextSecret: "shared-secret"}) + f.repo = conversationrepo.NewRepo(f.db) + f.servers = mcprepo.NewRepo(f.db) + f.store = storeinfra.NewLocal(f.root) + f.uploads = appupload.NewServiceWithRuntime(f.runtime, f.repo, zap.NewNop(), appupload.Hooks{}, appconversation.UploadErrorSet(), "test") + f.uploads.SetObjectStoreProvider(fileCreateStoreProvider{f.store}) + f.service = appmcp.NewFileCreateService(f.runtime, f.servers, f.repo, f.uploads, nil, zap.NewNop()) + return f +} + +func (f *fileCreateFixture) token(t *testing.T, userID uint) string { + t.Helper() + server, err := f.servers.GetServer(context.Background(), f.server.ID) + if err != nil { + t.Fatal(err) + } + token, err := f.service.IssueFileCreateGrant(context.Background(), appmcp.FileCreateCall{ServerID: f.server.ID, ToolID: f.tool.ID, UserID: userID, BaseURL: f.server.BaseURL, Epoch: server.FileCreateEpoch, CallID: "00000000-0000-0000-0000-000000000001", RequestID: "request1"}) + if err != nil || token == "" { + t.Fatalf("issue: %v", err) + } + return token +} + +func fileCreateInput(name, text string) appupload.TemporaryFileInput { + return appupload.TemporaryFileInput{FileName: name, MimeType: "text/plain", Reader: strings.NewReader(text), DeclaredSize: int64(len(text))} +} + +func TestFileCreateOwnershipQuotaAndReplay(t *testing.T) { + f := newFileCreateFixture(t) + ctx := context.Background() + token := f.token(t, 2) + first, err := f.service.Create(ctx, token, f.server.ID, fileCreateInput("report.txt", "hello")) + if err != nil { + t.Fatal(err) + } + if first.File.UserID != 2 || first.File.Purpose != "mcp_output" || first.Replayed { + t.Fatalf("wrong creation: %#v", first) + } + // Simulate response loss/restart: a new service instance replays the DB receipt. + restarted := appmcp.NewFileCreateService(f.runtime, f.servers, conversationrepo.NewRepo(f.db), f.uploads, nil, zap.NewNop()) + second, err := restarted.Create(ctx, token, f.server.ID, fileCreateInput("report.txt", "hello")) + if err != nil || !second.Replayed || second.File.FileID != first.File.FileID { + t.Fatalf("replay: %#v / %v", second, err) + } + files, err := f.uploads.ListFiles(ctx, appupload.ListFilesInput{UserID: 2, Page: 1, PageSize: 20}) + if err != nil || files.Total != 1 || files.Quota.UsedBytes != 5 { + t.Fatalf("list/quota: %#v / %v", files, err) + } + other, err := f.uploads.ListFiles(ctx, appupload.ListFilesInput{UserID: 1, Page: 1, PageSize: 20}) + if err != nil || other.Total != 0 || other.Quota.UsedBytes != 0 { + t.Fatalf("other user: %#v / %v", other, err) + } + for _, input := range []appupload.TemporaryFileInput{fileCreateInput("changed.txt", "hello"), fileCreateInput("report.txt", "world")} { + if _, err = f.service.Create(ctx, token, f.server.ID, input); !errors.Is(err, appmcp.ErrFileCreateConflict) { + t.Fatalf("changed file accepted: %v", err) + } + } + // A different request with equal content reuses the file but is not a replay. + third, err := f.service.Create(ctx, f.token(t, 2), f.server.ID, fileCreateInput("copy.txt", "hello")) + if err != nil || !third.Reused || third.Replayed || third.File.FileID != first.File.FileID { + t.Fatalf("dedup: %#v / %v", third, err) + } + if err = f.db.Model(&models.FileObject{}).Where("file_id = ?", first.File.FileID).Update("status", "deleted").Error; err != nil { + t.Fatal(err) + } + if _, err = f.service.Create(ctx, token, f.server.ID, fileCreateInput("report.txt", "hello")); !errors.Is(err, appmcp.ErrFileCreateResultGone) { + t.Fatalf("deleted result resurrected: %v", err) + } +} + +func TestFileCreateTrustBoundary(t *testing.T) { + for _, test := range []string{"wrong service", "forged token", "shared HMAC", "expired", "server disabled", "permission revoked", "reenabled", "user disabled", "user reenabled", "tool disabled", "tool reenabled", "global disabled", "global reenabled"} { + t.Run(test, func(t *testing.T) { + f := newFileCreateFixture(t) + token := f.token(t, 1) + id := f.server.ID + ctx := context.Background() + grant, err := f.service.Authorize(ctx, token, id) + if err != nil { + t.Fatal(err) + } + switch test { + case "wrong service": + id++ + case "forged token": + token = "dxf1_" + strings.Repeat("x", 43) + case "shared HMAC": + token, err = mcpauth.Sign("shared-secret", mcpauth.Payload{UserID: 2, Audience: f.server.BaseURL, JTI: "fake", ExpiresAt: time.Now().Add(time.Hour).Unix()}) + case "expired": + err = f.db.Model(&models.MCPFileCreateGrant{}).Where("token_hash = ?", grant.TokenHash).Update("expires_at", time.Now().Add(-time.Second)).Error + case "server disabled": + status := "inactive" + _, err = f.servers.UpdateServer(ctx, id, repository.UpdateMCPServerInput{Status: &status}) + case "permission revoked": + enabled := false + _, err = f.servers.UpdateServer(ctx, id, repository.UpdateMCPServerInput{FileCreateEnabled: &enabled}) + case "reenabled": + enabled := false + _, err = f.servers.UpdateServer(ctx, id, repository.UpdateMCPServerInput{FileCreateEnabled: &enabled}) + enabled = true + if err == nil { + _, err = f.servers.UpdateServer(ctx, id, repository.UpdateMCPServerInput{FileCreateEnabled: &enabled}) + } + case "user disabled": + err = f.db.Model(&models.User{}).Where("id = ?", 1).Update("status", "suspended").Error + case "user reenabled": + err = userrepo.NewRepo(f.db).UpdateUserStatus(ctx, 1, "suspended") + if err == nil { + err = userrepo.NewRepo(f.db).UpdateUserStatus(ctx, 1, "active") + } + case "tool disabled": + err = f.db.Model(&models.MCPTool{}).Where("id = ?", f.tool.ID).Update("status", "inactive").Error + case "tool reenabled": + _, err = f.servers.UpdateServerToolsStatus(ctx, id, []uint{f.tool.ID}, "inactive") + if err == nil { + status := "active" + _, err = f.servers.UpdateTool(ctx, f.tool.ID, repository.UpdateMCPToolInput{Status: &status}) + } + case "global disabled": + err = f.db.Create(&models.SystemSetting{Namespace: "mcp", Key: "mcp_enable", Value: "false", ValueType: "bool"}).Error + case "global reenabled": + set := settingsrepo.NewRepo(f.db) + err = set.Upsert(ctx, []domainsettings.SystemSetting{{Namespace: "mcp", Key: "mcp_enable", Value: "false", ValueType: "bool"}}) + if err == nil { + err = set.Upsert(ctx, []domainsettings.SystemSetting{{Namespace: "mcp", Key: "mcp_enable", Value: "true", ValueType: "bool"}}) + } + } + if err != nil { + t.Fatal(err) + } + if _, err = f.service.Create(ctx, token, id, fileCreateInput("report.txt", "hello")); !errors.Is(err, appmcp.ErrFileCreateUnauthorized) { + t.Fatalf("expected authorization failure, got %v", err) + } + var count int64 + if err = f.db.Model(&models.FileObject{}).Count(&count).Error; err != nil || count != 0 { + t.Fatalf("unauthorized file count %d: %v", count, err) + } + }) + } +} + +func TestFileCreateIssuanceChecksConnectionSnapshotAndBoolAliases(t *testing.T) { + f := newFileCreateFixture(t) + ctx := context.Background() + for _, enabled := range []string{"TRUE", "1", "true"} { + if err := settingsrepo.NewRepo(f.db).Upsert(ctx, []domainsettings.SystemSetting{{Namespace: "mcp", Key: "mcp_enable", Value: enabled, ValueType: "bool"}}); err != nil { + t.Fatal(err) + } + _ = f.token(t, 1) + } + headers := `{"X-Tenant":"changed"}` + if _, err := f.servers.UpdateServer(ctx, f.server.ID, repository.UpdateMCPServerInput{HeadersJSON: &headers}); err != nil { + t.Fatal(err) + } + _, err := f.service.IssueFileCreateGrant(ctx, appmcp.FileCreateCall{ServerID: f.server.ID, ToolID: f.tool.ID, UserID: 1, BaseURL: f.server.BaseURL, Epoch: 0, CallID: "new-call"}) + if !errors.Is(err, repository.ErrFileCreateUnauthorized) { + t.Fatalf("stale route snapshot received fresh authorization: %v", err) + } +} + +type fileCreateAuditCapture struct{ records []appaudit.WriteInput } + +func (a *fileCreateAuditCapture) Write(_ context.Context, input appaudit.WriteInput) { + a.records = append(a.records, input) +} + +func TestFileCreateAuditIdentifiesServiceAndUserWithoutCredentials(t *testing.T) { + f := newFileCreateFixture(t) + audit := &fileCreateAuditCapture{} + svc := appmcp.NewFileCreateService(f.runtime, f.servers, f.repo, f.uploads, audit, zap.NewNop()) + token := f.token(t, 2) + if _, err := svc.Create(context.Background(), token, f.server.ID, fileCreateInput("report.txt", "private-file-content")); err != nil { + t.Fatal(err) + } + if len(audit.records) != 1 || audit.records[0].ActorUserID != 2 || audit.records[0].Action != "mcp_create_file" { + t.Fatal("missing provenance") + } + raw, _ := json.Marshal(audit.records) + if strings.Contains(string(raw), token) || strings.Contains(string(raw), "private-file-content") || !strings.Contains(string(raw), "mcp_server_id") { + t.Fatalf("unsafe audit: %s", raw) + } +} + +type flakyFileCreateUploads struct { + repository.UploadRepository + fail bool +} + +func (r *flakyFileCreateUploads) GetActiveFileObjectByID(ctx context.Context, userID uint, fileID string) (*domainconversation.FileObject, error) { + if r.fail { + return nil, errors.New("database temporarily unavailable") + } + return r.UploadRepository.GetActiveFileObjectByID(ctx, userID, fileID) +} + +func TestFileCreateMaintenanceKeepsTransientFailuresAndResumesRealQueue(t *testing.T) { + f := newFileCreateFixture(t) + ctx := context.Background() + token := f.token(t, 1) + created, err := f.service.Create(ctx, token, f.server.ID, fileCreateInput("report.txt", "hello")) + if err != nil { + t.Fatal(err) + } + grant, err := f.service.Authorize(ctx, token, f.server.ID) + if err != nil { + t.Fatal(err) + } + if err = f.db.Model(&models.MCPFileCreateGrant{}).Where("token_hash = ?", grant.TokenHash).Update("expires_at", time.Now().Add(-time.Second)).Error; err != nil { + t.Fatal(err) + } + flaky := &flakyFileCreateUploads{UploadRepository: f.repo, fail: true} + cache := memorycache.New() + processing := appprocessing.NewServiceWithRuntime(appprocessing.Dependencies{Config: f.runtime, Repository: f.repo, Cache: cache, Logger: zap.NewNop()}) + uploads := appupload.NewServiceWithRuntime(f.runtime, flaky, zap.NewNop(), appupload.Hooks{EnsureFileCreateProcessing: processing.EnsureFileCreateProcessing}, appconversation.UploadErrorSet(), "test") + uploads.SetObjectStoreProvider(fileCreateStoreProvider{f.store}) + service := appmcp.NewFileCreateService(f.runtime, f.servers, f.repo, uploads, nil, zap.NewNop()) + service.Maintain(ctx) + var count int64 + if err = f.db.Model(&models.MCPFileCreateGrant{}).Count(&count).Error; err != nil || count != 1 { + t.Fatalf("transient failure lost receipt: %d / %v", count, err) + } + flaky.fail = false + service.Maintain(ctx) + service.Maintain(ctx) + messages, err := cache.ReadFileProcessingMessages(ctx, "test-worker") + if err != nil || len(messages) != 1 || messages[0].FileID != created.File.FileID { + t.Fatalf("handoff did not resume: %#v / %v", messages, err) + } + // A paused browser initializer carries uploaded state after an MCP worker started. + if err = f.db.Model(&models.FileObject{}).Where("file_id = ?", created.File.FileID).Updates(map[string]any{"processing_status": "ready", "extract_status": "ready", "preview_text": "preserved"}).Error; err != nil { + t.Fatal(err) + } + stale := created.File + if err = processing.InitializeUploadedFile(ctx, &stale); err != nil { + t.Fatal(err) + } + current, err := f.repo.GetActiveFileObjectByID(ctx, 1, created.File.FileID) + if err != nil || current.ProcessingStatus != "ready" || current.PreviewText != "preserved" { + t.Fatalf("stale browser initializer reset worker: %#v / %v", current, err) + } + service.Maintain(ctx) + if err = f.db.Model(&models.MCPFileCreateGrant{}).Count(&count).Error; err != nil || count != 0 { + t.Fatalf("finished receipt not expired: %d / %v", count, err) + } +} + +func TestFileCreateExpiredCleanupPreservesSharedReferencesAndRetriesRemoval(t *testing.T) { + for _, mode := range []string{"missing metadata", "shared reference", "removal failure"} { + t.Run(mode, func(t *testing.T) { + f := newFileCreateFixture(t) + ctx := context.Background() + token := f.token(t, 1) + result, err := f.service.Create(ctx, token, f.server.ID, fileCreateInput("report.txt", "hello")) + if err != nil { + t.Fatal(err) + } + grant, err := f.service.Authorize(ctx, token, f.server.ID) + if err != nil { + t.Fatal(err) + } + if err = f.db.Unscoped().Where("file_id = ?", result.File.FileID).Delete(&models.FileObject{}).Error; err != nil { + t.Fatal(err) + } + if mode == "shared reference" { + if err = f.db.Create(&models.FileObject{FileID: "shared", UserID: 2, StoragePath: grant.StoragePath, Status: "active"}).Error; err != nil { + t.Fatal(err) + } + } + if err = f.db.Model(&models.MCPFileCreateGrant{}).Where("token_hash = ?", grant.TokenHash).Update("expires_at", time.Unix(0, 0)).Error; err != nil { + t.Fatal(err) + } + if mode == "removal failure" { + err = f.repo.ExpireFileCreateGrant(ctx, grant.TokenHash, time.Now(), func(string) error { return errors.New("storage unavailable") }) + if err == nil { + t.Fatal("failed removal discarded receipt") + } + var count int64 + if err = f.db.Model(&models.MCPFileCreateGrant{}).Count(&count).Error; err != nil || count != 1 { + t.Fatal("lost cleanup evidence") + } + } + f.service.Maintain(ctx) + reader, _, err := f.store.Open(ctx, grant.StoragePath) + if mode == "shared reference" { + if err != nil { + t.Fatal("cleanup deleted shared file") + } + _ = reader.Close() + } else if !errors.Is(err, portstore.ErrNotFound) { + t.Fatalf("candidate remains: %v", err) + } + }) + } +} + +func TestFileCreatePolicyRejectionsLeaveNoFiles(t *testing.T) { + f := newFileCreateFixture(t) + ctx := context.Background() + token := f.token(t, 1) + if _, err := f.service.Create(ctx, token, f.server.ID, fileCreateInput("large.txt", strings.Repeat("x", 1025))); !errors.Is(err, appconversation.ErrFileTooLarge) { + t.Fatalf("size: %v", err) + } + if _, err := f.service.Create(ctx, token, f.server.ID, fileCreateInput("data.pdf", "%PDF-1.7")); !errors.Is(err, appconversation.ErrMIMEBlocked) { + t.Fatalf("type: %v", err) + } + cfg := f.runtime.Snapshot() + cfg.UserStorageQuotaBytes = 2 + f.runtime.Store(cfg) + if _, err := f.service.Create(ctx, token, f.server.ID, fileCreateInput("quota.txt", "hello")); !errors.Is(err, appconversation.ErrStorageQuotaExceeded) { + t.Fatalf("quota: %v", err) + } + var count int64 + if err := f.db.Model(&models.FileObject{}).Count(&count).Error; err != nil || count != 0 { + t.Fatalf("files %d: %v", count, err) + } + if err := filepath.WalkDir(f.root, func(_ string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() { + t.Error("rejected upload wrote an object") + } + return err + }); err != nil { + t.Fatal(err) + } +} + +func TestFileCreateConcurrentIndependentServices(t *testing.T) { + f := newFileCreateFixture(t) + otherDB := openFileCreateDB(t, f.dsn) + checkFileCreateConcurrency(t, f, otherDB) +} + +func checkFileCreateConcurrency(t *testing.T, f *fileCreateFixture, otherDB *gorm.DB) { + t.Helper() + token := f.token(t, 1) + otherRepo := conversationrepo.NewRepo(otherDB) + otherUploads := appupload.NewServiceWithRuntime(f.runtime, otherRepo, zap.NewNop(), appupload.Hooks{}, appconversation.UploadErrorSet(), "test") + otherUploads.SetObjectStoreProvider(fileCreateStoreProvider{storeinfra.NewLocal(f.root)}) + other := appmcp.NewFileCreateService(f.runtime, mcprepo.NewRepo(otherDB), otherRepo, otherUploads, nil, zap.NewNop()) + var wg sync.WaitGroup + results := make(chan *appmcp.FileCreateResult, 2) + errorsCh := make(chan error, 2) + for _, svc := range []*appmcp.FileCreateService{f.service, other} { + wg.Add(1) + go func(svc *appmcp.FileCreateService) { + defer wg.Done() + result, err := svc.Create(context.Background(), token, f.server.ID, fileCreateInput("report.txt", "hello")) + results <- result + errorsCh <- err + }(svc) + } + wg.Wait() + close(results) + close(errorsCh) + for err := range errorsCh { + if err != nil { + t.Fatal(err) + } + } + id := "" + for result := range results { + if id != "" && result.File.FileID != id { + t.Fatal("multiple files created") + } + id = result.File.FileID + } + quota, err := f.repo.GetOrInitUserStorageQuota(context.Background(), 1, 1024) + if err != nil || quota.UsedBytes != 5 { + t.Fatalf("quota: %#v / %v", quota, err) + } +} + +func TestFileCreatePostgresConcurrentInstances(t *testing.T) { + dsn := os.Getenv("DEEIX_TEST_DATABASE_DSN") + if dsn == "" { + t.Skip("set DEEIX_TEST_DATABASE_DSN to test PostgreSQL multi-instance receipt/quota settlement") + } + adminDB, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + t.Fatal(err) + } + schemaName := fmt.Sprintf("deeix_file_create_%d", time.Now().UnixNano()) + if err = adminDB.Exec("CREATE SCHEMA " + schemaName).Error; err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = adminDB.Exec("DROP SCHEMA " + schemaName + " CASCADE").Error + db, _ := adminDB.DB() + _ = db.Close() + }) + if strings.HasPrefix(dsn, "postgres://") || strings.HasPrefix(dsn, "postgresql://") { + parsed, parseErr := url.Parse(dsn) + if parseErr != nil { + t.Fatal(parseErr) + } + query := parsed.Query() + query.Set("search_path", schemaName) + parsed.RawQuery = query.Encode() + dsn = parsed.String() + } else { + dsn += " search_path=" + schemaName + } + open := func() *gorm.DB { + db, openErr := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if openErr != nil { + t.Fatal(openErr) + } + sqlDB, dbErr := db.DB() + if dbErr != nil { + t.Fatal(dbErr) + } + sqlDB.SetMaxOpenConns(8) + t.Cleanup(func() { _ = sqlDB.Close() }) + return db + } + f := initializeFileCreateFixture(t, &fileCreateFixture{db: open(), root: t.TempDir()}) + if err = f.db.Exec("CREATE UNIQUE INDEX file_create_test_content ON file_objects(user_id, sha256, size_bytes) WHERE status = 'active' AND deleted_at IS NULL AND sha256 <> ''").Error; err != nil { + t.Fatal(err) + } + checkFileCreateConcurrency(t, f, open()) +} + +type failingFileCreateRepository struct { + repository.MCPFileCreateRepository + rollback bool + loseResponse bool +} + +func (r failingFileCreateRepository) WithFileCreateGrant(ctx context.Context, hash string, serverID uint, fn func(*domainmcp.FileCreateGrant, repository.UploadRepository) error) error { + err := r.MCPFileCreateRepository.WithFileCreateGrant(ctx, hash, serverID, func(g *domainmcp.FileCreateGrant, uploads repository.UploadRepository) error { + if err := fn(g, uploads); err != nil { + return err + } + if r.rollback { + return errors.New("injected rollback") + } + return nil + }) + if err == nil && r.loseResponse { + return errors.New("commit acknowledgment lost") + } + return err +} + +type acceptedButFailedStore struct{ portstore.Store } + +func (s acceptedButFailedStore) Put(ctx context.Context, key string, body io.Reader, opts portstore.PutOptions) (portstore.ObjectInfo, error) { + info, err := s.Store.Put(ctx, key, body, opts) + if err == nil { + err = errors.New("put acknowledgment lost") + } + return info, err +} + +func TestFileCreateFailureRecoveryAndCleanup(t *testing.T) { + for _, mode := range []string{"rollback", "lost commit response", "lost put response", "expired abandoned"} { + t.Run(mode, func(t *testing.T) { + f := newFileCreateFixture(t) + token := f.token(t, 1) + ctx := context.Background() + failedRepo := failingFileCreateRepository{MCPFileCreateRepository: f.repo, rollback: mode == "rollback" || mode == "expired abandoned", loseResponse: mode == "lost commit response"} + if mode == "lost put response" { + f.uploads.SetObjectStoreProvider(fileCreateStoreProvider{acceptedButFailedStore{f.store}}) + } + failed := appmcp.NewFileCreateService(f.runtime, f.servers, failedRepo, f.uploads, nil, zap.NewNop()) + if _, err := failed.Create(ctx, token, f.server.ID, fileCreateInput("report.txt", "hello")); err == nil { + t.Fatal("expected injected failure") + } + grant, err := f.service.Authorize(ctx, token, f.server.ID) + if err != nil { + t.Fatal(err) + } + reader, _, err := f.store.Open(ctx, grant.StoragePath) + if err != nil { + t.Fatalf("uncertain completion destroyed candidate: %v", err) + } + _ = reader.Close() + if mode == "expired abandoned" { + if err = f.db.Model(&models.MCPFileCreateGrant{}).Where("token_hash = ?", grant.TokenHash).Update("expires_at", time.Now().Add(-time.Second)).Error; err != nil { + t.Fatal(err) + } + f.service.Maintain(ctx) + if _, _, err = f.store.Open(ctx, grant.StoragePath); !errors.Is(err, portstore.ErrNotFound) { + t.Fatalf("abandoned object remains: %v", err) + } + return + } + f.uploads.SetObjectStoreProvider(fileCreateStoreProvider{f.store}) + result, err := f.service.Create(ctx, token, f.server.ID, fileCreateInput("report.txt", "hello")) + if err != nil || result.File.FileID != grant.CandidateID { + t.Fatalf("recovery: %#v / %v", result, err) + } + quota, err := f.repo.GetOrInitUserStorageQuota(ctx, 1, 1024) + if err != nil || quota.UsedBytes != 5 { + t.Fatalf("recovery quota: %#v / %v", quota, err) + } + }) + } +} + +func TestFileCreateHTTPRejectsForeignOwnershipAndExtraParts(t *testing.T) { + f := newFileCreateFixture(t) + token := f.token(t, 1) + h := NewHandler(nil) + h.SetFileCreateService(f.service) + gin.SetMode(gin.TestMode) + router := gin.New() + NewModule(h).RegisterFileCreateRoute(router.Group("/api/v1"), middleware.RateLimit(nil, f.runtime)) + for _, test := range []string{"missing token", "no bearer", "wrong user field", "extra file", "oversized", "normal"} { + t.Run(test, func(t *testing.T) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + part, err := writer.CreateFormFile("file", "report.txt") + if err != nil { + t.Fatal(err) + } + text := "hello" + if test == "oversized" { + text = strings.Repeat("x", 1025) + } + _, _ = io.WriteString(part, text) + if test == "wrong user field" { + _ = writer.WriteField("user_id", "2") + } + if test == "extra file" { + extra, _ := writer.CreateFormFile("file", "second.txt") + _, _ = io.WriteString(extra, "extra") + } + _ = writer.Close() + request := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/v1/mcp/servers/%d/files", f.server.ID), &body) + request.Header.Set("Content-Type", writer.FormDataContentType()) + if test != "missing token" { + request.Header.Set("Authorization", "Bearer "+token) + } + if test == "no bearer" { + request.Header.Set("Authorization", token) + } + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + want := 400 + switch test { + case "missing token", "no bearer": + want = 401 + case "oversized": + want = 413 + case "normal": + want = 200 + } + if recorder.Code != want { + t.Fatalf("status %d, expected %d: %s", recorder.Code, want, recorder.Body.String()) + } + if want == 200 { + var response FileCreateResponseDoc + if err = json.Unmarshal(recorder.Body.Bytes(), &response); err != nil || response.Data.FileID == "" || response.Data.ProcessingStatus != "uploaded" || response.Data.ExtractStatus != "none" { + t.Fatalf("creation/state: %#v / %v", response, err) + } + } + }) + } +} diff --git a/backend/internal/transport/http/mcp/router.go b/backend/internal/transport/http/mcp/router.go index 802e9856d..70e284218 100644 --- a/backend/internal/transport/http/mcp/router.go +++ b/backend/internal/transport/http/mcp/router.go @@ -7,6 +7,10 @@ func (m *Module) RegisterRoutes(authGroup *gin.RouterGroup) { group.GET("/tools", m.Handler.ListAvailableTools) } +func (m *Module) RegisterFileCreateRoute(group *gin.RouterGroup, limit gin.HandlerFunc) { + group.POST("/mcp/servers/:id/files", m.Handler.authorizeFileCreate, limit, m.Handler.CreateFile) +} + func (m *Module) RegisterAdminRoutes(adminGroup *gin.RouterGroup) { group := adminGroup.Group("/mcp") group.GET("/servers", m.Handler.ListServers) diff --git a/backend/internal/transport/http/middleware/ratelimit.go b/backend/internal/transport/http/middleware/ratelimit.go index 253904810..ded225750 100644 --- a/backend/internal/transport/http/middleware/ratelimit.go +++ b/backend/internal/transport/http/middleware/ratelimit.go @@ -199,7 +199,7 @@ func isMediaGenerationRoute(method string, route string) bool { } func isFileUploadRoute(method string, route string) bool { - return method == http.MethodPost && route == "/files" + return method == http.MethodPost && (route == "/files" || route == "/mcp/servers/:id/files") } func isPollingRoute(method string, route string) bool { diff --git a/backend/internal/transport/http/server.go b/backend/internal/transport/http/server.go index cc02f95df..3403f7419 100644 --- a/backend/internal/transport/http/server.go +++ b/backend/internal/transport/http/server.go @@ -207,6 +207,9 @@ func NewEngine(cfg *config.Runtime, log *zap.Logger, modules Modules, hc HealthC } authRequired := api.Group("") + if modules.MCP != nil { + modules.MCP.RegisterFileCreateRoute(api.Group("", middleware.PublicAuthRateLimit(limiter, cfg)), middleware.RateLimit(limiter, cfg)) + } authRequired.Use(middleware.AuthMiddleware(snapshot.JWTSecret, modules.AuthService)) authRequired.Use(middleware.RateLimit(limiter, cfg)) diff --git a/docs/MCP_FILE_CREATE.md b/docs/MCP_FILE_CREATE.md new file mode 100644 index 000000000..fedf969c3 --- /dev/null +++ b/docs/MCP_FILE_CREATE.md @@ -0,0 +1,120 @@ +# MCP 工具产物创建为用户文件（create-only 评审提案） + +关联 [#818](https://github.com/DEEIX-AI/DEEIX-Chat/issues/818)。本文描述本 Draft PR 的可运行契约，**不代表维护者已批准协议**。基线为 `dev@19d60fec8f9356a4bdd997e8b71a5a8a9a0baec9`。 + +## 授权和范围 + +管理员在已有 MCP 服务创建/更新 API 中显式设置 `fileCreateEnabled: true`，授权该服务在持久化聊天的实际 `tools/call` 中自动创建一个当前用户文件。默认关闭；省略此字段的更新保留原值。首版仅通过管理员 API 配置，不增加设置页。 + +这个开关是管理员对自动创建的授权，**不是逐次用户确认**。没有把模型传入的 `confirm=true` 当成用户同意。临时聊天、附件预处理和工具发现不签发凭证。 + +DEEIX 在获取现有工具并发许可后，为一次实际调用生成 32 字节随机 capability，通过 `X-Deeix-File-Create-Token` 仅发给登记服务的 `tools/call`。自动重试沿用同一凭证和调用 ID；签名用户上下文存在时，其 `jti` 使用同一调用 ID。`request_id` 只作请求关联，不是文件幂等键。重新发起聊天或新工具调用取得新的授权，不能当作前一次调用的重试。 + +数据库 `mcp_file_create_grants` 只保存 capability 的 SHA-256，绑定服务、用户、工具、调用、服务撤销代数和过期时间。**只能由 DEEIX 签发和验证**：既不接受共享 HMAC 签名头作为反向授权，也不复用用户 JWT、MCP 出站 token 或模型 `_meta`。外部服务无法自行签发、改归属或扩大操作范围。凭证是 bearer capability；取得它就取得该次受限创建权，不是另一套长期服务身份平台。 + +有效期固定 10 分钟，不能续期。管理员更改服务状态、权限、地址、鉴权 token 或 headers 会增加撤销代数；配置快照过期时拒绝签发新授权。禁用工具、停用用户或关闭全局 MCP 会同时永久失效相关未过期凭证，重新启用不恢复旧授权。验证还读取数据库中的当前全局设置及用户/工具/服务状态；数据库不可用时失败关闭。已进入文件提交事务的调用与禁用操作以锁顺序线性化：先完成的提交可成功，禁用完成之后的新提交不能成功。注销浏览器会话、取消聊天本身不撤销独立的 10 分钟 capability。 + +凭证不进入模型参数、工具发现、初始化握手或静态 headers；带凭证的 MCP 请求禁止跨 origin 重定向。结果中的原文及 JSON 转义凭证在存储/展示前移除。MCP 服务仍必须保护其日志和网络传输，不应把凭证返回到工具结果。生产使用 HTTPS 或受控内部网络。 + +## 字节交接 + +外部服务向其登记 ID 的端点上传字节： + +```http +POST /api/v1/mcp/servers/42/files +Authorization: Bearer <本次 tools/call 收到的 capability> +Content-Type: multipart/form-data; boundary=... + +... 唯一一个名为 file 的文件 part ... +``` + +外部 MCP 的 DEEIX 地址及服务 ID 由部署者提供，不从模型参数或客户端 Host 头推断。示例： + +```sh +curl --fail-with-body \ + -H "Authorization: Bearer $DEEIX_FILE_CREATE_TOKEN" \ + -F 'file=@report.txt;type=text/plain' \ + 'https://chat.example/api/v1/mcp/servers/42/files' +``` + +只允许一个 `file` part，不允许 `user_id`、`purpose`、目标 `file_id`、URL、额外文件/字段或 query 参数。归属来自已验证授权，用途固定为 `mcp_output`。DEEIX 不拉取工具提供的 URL。 + +HTTP 总体积限制为既有 `MaxUploadFileBytes` 加 1 MiB multipart 开销；应用层仍按实际字节校验文件上限、类型允许列表、危险类型和分类大小限制。复用 `UploadService.PrepareTemporaryFile`、对象存储 port、文件元数据创建、内容去重和配额事务。类型判定沿用当前上传策略，包括扩展名归类，**不新增格式完整性/恶意软件扫描保证**。已有策略不允许的格式（例如未加入允许列表的 PPTX）仍被拒绝。 + +新文件成为用户拥有的正式 `file_objects` 行，计入同一存储配额，出现在原有 `/api/v1/files` 列表。有效的同用户相同内容可复用既有文件 ID，不重复扣配额；已有对象缺失或校验失败时返回冲突，委托入口不修复/删除已有用户文件。 + +现有浏览器 `/api/v1/files` 仍只接受登录 JWT/session，不接受 capability 或任意 MCP 签名头。新端点先套用既有按 IP 公开限流，再按授权用户套用 `file_upload` 限流策略；沿用当前默认关闭、缓存错误失败开放等行为，不额外引入一套限流配置。 + +## 返回与重试 + +成功返回 HTTP 200、既有 `errorMsg + data` envelope： + +```json +{ + "errorMsg": "", + "data": { + "file_id": "file_example", + "file_name": "report.txt", + "size_bytes": 123, + "sha256": "<64 hex characters>", + "reused": false, + "replayed": false, + "processing_status": "queued", + "extract_status": "none", + "embed_status": "none", + "processing_ready": false + } +} +``` + +`file_id` 是可靠的创建/复用结果。`reused` 表示内容去重，`replayed` 表示本次 capability 已成功提交过。处理状态从数据库重新读取；创建成功不保证文本提取、OCR、向量化或 RAG 已就绪，状态可随后变化，甚至失败。原有用户文件 UI 继续使用现有处理查询。 + +| 情形 | 结果 | +| --- | --- | +| 未授权、伪造、过期、撤销、错服务 | 401 `mcp.file_create.unauthorized` | +| 非唯一文件、额外字段、非法元数据 | 400 `mcp.file_create.invalid_input` | +| 类型拒绝 | 400，沿用既有文件错误码 | +| 文件或 HTTP 总体积超限 | 413 | +| 配额不足 | 409，沿用既有配额错误码 | +| 同一 capability 改变文件内容、规范化文件名或声明 MIME；既有去重内容不可用 | 409 `mcp.file_create.conflict` | +| 成功结果后来被删除 | 404 `mcp.file_create.result_gone`，不补建文件 | +| 存储/数据库暂时失败或提交结果不确定 | 500，用**同一 capability、同一文件**在有效期内重试 | +| 限流 | 429，沿用既有策略 | + +首个通过文件策略校验的提交，在对象 I/O 前持久绑定规范化文件名、声明 MIME、实际 SHA-256 和大小。后续即使首次配额/存储失败也只能重试这份文件；被类型/大小校验拒绝的请求尚未绑定。没有另一个客户端任意选择的 idempotency key，也不将内容摘要当成请求 ID。 + +提交按数据库 receipt 锁串行，receipt 的 `file_id`、文件行与配额扣减在同一事务中提交。SQLite 使用其已有单写者模型，PostgreSQL 支持共享 DB/对象存储上的多个服务实例。最小实现持有用户配额锁完成最多两分钟的请求/对象 I/O；这是明确的吞吐上限，没有引入通用分布式任务/权限框架。 + +候选 ID 和精确存储 key 在写字节前持久化。对象写入成功但回包丢失、事务回滚、提交确认丢失时，不凭一次错误删除候选：同一提交可校验并恢复同一个 key；不会改用新 ID 或再次扣配额。文件/receipt 提交前不启动处理。 + +每分钟有界巡检最多 100 条记录，复用现有处理服务恢复 `uploaded/queued` 的入队；初始化采用条件状态更新，现有 memory/Redis 处理队列按未结算用户/文件任务去重，worker 仍使用原有 attempt-ID fencing。没有新任务队列。Redis 入队确认丢失可复用仍在 stream 的任务；内存队列重启丢失时 receipt 可补交。凭证过期不延长授权；未完成处理交接的记录保留到可以确认交接，暂时数据库读取失败保留记录。过期候选无活跃引用时清理，删除失败则保留待重试；正式文件和跨用户共享引用不会被清理。 + +过期并完成交接的 receipt 被清理，幂等回放窗口就是凭证的 10 分钟有效期。过期后只能在已有用户文件列表确认产物，没有 MCP 文件查询入口。 + +## 审计、迁移和评审点 + +沿用既有审计服务，动作 `mcp_create_file`，记录目标用户、登记服务/工具 ID、调用 ID、文件 ID、实际大小及 `reused/replayed`；不记录 capability、文件内容或文件名。沿用当前审计的结构化日志与独立 DB 写入流程，不承诺审计和文件提交是同一事务。进程在提交后、审计前退出的窗口仍可能缺少成功审计，receipt 支持恢复创建但不是审计 outbox。 + +使用项目既有 schema migration 注册一个 receipt 表及 MCP 服务的默认关闭开关/撤销代数。没有新 YAML、环境变量、共享存储卷、反向 JWT 或依赖。Swagger 与 `packages/api-contract/src/types.generated.ts` 由 `pnpm api:generate` 生成。 + +维护者需决定的事项： + +1. 是否接受管理员按服务授权后自动创建的 v1 语义，以及临时聊天/预处理暂不授权。 +2. 是否接受每调用一文件、10 分钟 bearer capability、上传端点/头命名与回放窗口。 +3. 是否需要在合并前提供管理员 UI，或将成功审计与文件结算进一步做成原子记录。 + +## 验证入口 + +```sh +make -C backend lint test +pnpm api:check +pnpm --filter @deeix/web check + +# 可选真实 PostgreSQL 和专用 Redis 集成测试 +DEEIX_TEST_DATABASE_DSN='postgres://.../test?sslmode=disable' \ + go -C backend test ./internal/transport/http/mcp -run TestFileCreatePostgresConcurrentInstances + +# 使用 Redis DB 15；必须是专用测试实例，该测试会清空 DB 15。 +DEEIX_TEST_REDIS_ADDR='127.0.0.1:6379' \ + go -C backend test ./internal/infra/cache/redis -run TestFileProcessingRedisRecoveryEnqueueIsIdempotent +``` diff --git a/packages/api-contract/src/types.generated.ts b/packages/api-contract/src/types.generated.ts index 0cad258ef..b5b41d776 100644 --- a/packages/api-contract/src/types.generated.ts +++ b/packages/api-contract/src/types.generated.ts @@ -1374,6 +1374,8 @@ export interface CreateRedemptionCodeRequest { export interface CreateServerRequest { authToken?: string; baseURL: string; + /** FileCreateEnabled authorizes automatic creation of one user file per persistent tool call. */ + fileCreateEnabled?: boolean; headersJSON?: string; name: string; status?: string; @@ -1726,6 +1728,24 @@ export interface FeaturePolicyResponseDoc { errorMsg: string; } +export interface FileCreateResponse { + sha256: string; + embed_status: string; + extract_status: string; + file_id: string; + file_name: string; + processing_ready: boolean; + processing_status: string; + replayed: boolean; + reused: boolean; + size_bytes: number; +} + +export interface FileCreateResponseDoc { + data: FileCreateResponse; + errorMsg: string; +} + export interface FileEmbeddingSkipResponse { fileID: string; reason: string; @@ -4091,6 +4111,7 @@ export interface ServerResponse { activeToolCount: number; baseURL: string; createdAt: string; + fileCreateEnabled: boolean; headersJSON: string; id: number; lastError: string; @@ -10841,6 +10862,30 @@ export namespace Llm { } export namespace Mcp { + /** + * @description 仅接受 DEEIX 在 tools/call 签发的一文件 capability，不接受用户 JWT 或 MCP 共享 HMAC 上下文。重复相同文件返回同一 file_id；创建不代表提取或向量化已完成。 + * @tags mcp + * @name ServersFilesCreate + * @summary 创建 MCP 工具产物文件 + * @request POST:/mcp/servers/{id}/files + */ + export namespace ServersFilesCreate { + export type RequestParams = { + /** 登记的 MCP 服务 ID */ + id: number; + }; + export type RequestQuery = {}; + export type RequestBody = { + /** 唯一文件，不接受其他字段 */ + file: File; + }; + export type RequestHeaders = { + /** Bearer */ + Authorization: string; + }; + export type ResponseBody = FileCreateResponseDoc; + } + /** * @description 获取当前聊天侧可选择的 MCP 工具 * @tags mcp