From 7be0871c0cb099d7a3a7b89e8b38abd0133c5abe Mon Sep 17 00:00:00 2001 From: Jiang Chaoqun Date: Sat, 19 Sep 2026 13:42:32 +0800 Subject: [PATCH] feat: add multi-category device manual management --- .github/workflows/go.yml | 11 +- README.md | 15 + api/accounts/handlers.go | 4 +- api/handler.go | 2 + .../categories_pagination_integration_test.go | 28 + api/manuals/content.go | 85 +++ api/manuals/handlers.go | 310 ++++++++ api/manuals/integration_test.go | 464 ++++++++++++ api/manuals/multipart.go | 43 ++ api/manuals/route.go | 48 ++ api/manuals/route_test.go | 51 ++ ...user_quota_concurrency_integration_test.go | 126 ++++ api/manuals/write_guards_integration_test.go | 263 +++++++ api/middleware/account.go | 2 + app/manuals/application.go | 58 ++ app/manuals/create.go | 106 +++ app/manuals/files.go | 86 +++ app/manuals/items.go | 195 +++++ app/manuals/items_test.go | 28 + app/manuals/policy.go | 73 ++ app/manuals/policy_test.go | 36 + app/manuals/read.go | 283 ++++++++ app/manuals/resource.go | 42 ++ app/manuals/update.go | 296 ++++++++ config/config_example.yaml | 19 + config/global.go | 23 + config/nginx/manuals_locations.conf | 28 + config/runtime.go | 3 +- config/runtime_test.go | 4 +- deploy/pi/README.md | 19 +- deploy/pi/frontend-nginx.conf | 3 +- deploy/pi/gateway-api.conf | 27 + deploy/pi/gateway-frontend.conf | 27 + docs/specs/manual-management/design.md | 155 ++++ .../specs/manual-management/implementation.md | 36 + docs/specs/manual-management/verification.md | 67 ++ domain/dal/manual.go | 326 +++++++++ domain/dal/manual_integration_test.go | 102 +++ domain/dal/manual_test.go | 12 + domain/dal/migrations/20260919_manuals.sql | 53 ++ domain/dal/token.go | 19 + domain/model/manual.go | 173 +++++ domain/service/accounts/manuals_test.go | 27 + domain/service/accounts/service.go | 9 + domain/service/accounts/sessions.go | 40 ++ domain/service/applications/service.go | 5 + domain/service/applications/service_test.go | 6 + .../manuals/pdf_preview_boundary_test.go | 64 ++ domain/service/manuals/service.go | 279 ++++++++ domain/service/manuals/service_test.go | 170 +++++ domain/service/manuals/storage.go | 626 +++++++++++++++++ .../manuals/storage_large_exif_test.go | 28 + domain/service/manuals/types.go | 110 +++ domain/service/passport/token.go | 27 + domain/service/service.go | 4 + front_vue/package.json | 1 + front_vue/src/api/manuals.cjs | 113 +++ front_vue/src/components/MyHeader.vue | 2 + front_vue/src/router/index.js | 27 + front_vue/src/store/index.js | 2 + front_vue/src/utils/applications_behavior.cjs | 2 + front_vue/src/utils/manuals_behavior.cjs | 175 +++++ .../src/utils/web_projects_navigation.cjs | 4 +- front_vue/src/views/Applications.vue | 3 + front_vue/src/views/Index.vue | 11 +- front_vue/src/views/ManualDetail.vue | 75 ++ front_vue/src/views/ManualEditor.vue | 333 +++++++++ front_vue/src/views/ManualList.vue | 135 ++++ front_vue/test/accounts-behavior.test.cjs | 4 + front_vue/test/applications-behavior.test.cjs | 4 +- front_vue/test/manuals-api.test.cjs | 125 ++++ front_vue/test/manuals-behavior.test.cjs | 92 +++ front_vue/test/manuals-browser.cjs | 398 +++++++++++ .../test/web-projects-navigation.test.cjs | 6 + script/test_manuals_skill.py | 485 +++++++++++++ skills/home-server-web-share/SKILL.md | 10 +- .../home-server-web-share/agents/openai.yaml | 6 +- .../references/manuals.md | 111 +++ .../home-server-web-share/scripts/manuals.py | 664 ++++++++++++++++++ utils/ctx.go | 3 +- 80 files changed, 7918 insertions(+), 19 deletions(-) create mode 100644 api/manuals/categories_pagination_integration_test.go create mode 100644 api/manuals/content.go create mode 100644 api/manuals/handlers.go create mode 100644 api/manuals/integration_test.go create mode 100644 api/manuals/multipart.go create mode 100644 api/manuals/route.go create mode 100644 api/manuals/route_test.go create mode 100644 api/manuals/user_quota_concurrency_integration_test.go create mode 100644 api/manuals/write_guards_integration_test.go create mode 100644 app/manuals/application.go create mode 100644 app/manuals/create.go create mode 100644 app/manuals/files.go create mode 100644 app/manuals/items.go create mode 100644 app/manuals/items_test.go create mode 100644 app/manuals/policy.go create mode 100644 app/manuals/policy_test.go create mode 100644 app/manuals/read.go create mode 100644 app/manuals/resource.go create mode 100644 app/manuals/update.go create mode 100644 config/nginx/manuals_locations.conf create mode 100644 docs/specs/manual-management/design.md create mode 100644 docs/specs/manual-management/implementation.md create mode 100644 docs/specs/manual-management/verification.md create mode 100644 domain/dal/manual.go create mode 100644 domain/dal/manual_integration_test.go create mode 100644 domain/dal/manual_test.go create mode 100644 domain/dal/migrations/20260919_manuals.sql create mode 100644 domain/model/manual.go create mode 100644 domain/service/accounts/manuals_test.go create mode 100644 domain/service/manuals/pdf_preview_boundary_test.go create mode 100644 domain/service/manuals/service.go create mode 100644 domain/service/manuals/service_test.go create mode 100644 domain/service/manuals/storage.go create mode 100644 domain/service/manuals/storage_large_exif_test.go create mode 100644 domain/service/manuals/types.go create mode 100644 front_vue/src/api/manuals.cjs create mode 100644 front_vue/src/utils/manuals_behavior.cjs create mode 100644 front_vue/src/views/ManualDetail.vue create mode 100644 front_vue/src/views/ManualEditor.vue create mode 100644 front_vue/src/views/ManualList.vue create mode 100644 front_vue/test/manuals-api.test.cjs create mode 100644 front_vue/test/manuals-behavior.test.cjs create mode 100644 front_vue/test/manuals-browser.cjs create mode 100644 script/test_manuals_skill.py create mode 100644 skills/home-server-web-share/references/manuals.md create mode 100755 skills/home-server-web-share/scripts/manuals.py diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index bda7c1d..235fece 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -42,6 +42,7 @@ jobs: CREATE DATABASE home_server_accounts_runtime_test; CREATE DATABASE home_server_analytics_test_ci; CREATE DATABASE home_server_readcache_test_ci; + CREATE DATABASE home_server_manuals_test_ci CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE USER 'home_server_ci'@'%' IDENTIFIED BY 'ci-isolated-tests'; GRANT ALL ON home_server_accounts_http_test_ci.* TO 'home_server_ci'@'%'; GRANT ALL ON home_server_accounts_migration_test_ci.* TO 'home_server_ci'@'%'; @@ -49,6 +50,7 @@ jobs: GRANT ALL ON home_server_accounts_runtime_test.* TO 'home_server_ci'@'%'; GRANT ALL ON home_server_analytics_test_ci.* TO 'home_server_ci'@'%'; GRANT ALL ON home_server_readcache_test_ci.* TO 'home_server_ci'@'%'; + GRANT ALL ON home_server_manuals_test_ci.* TO 'home_server_ci'@'%'; SQL - name: Exercise availability-first Redis policy @@ -72,13 +74,15 @@ jobs: HOME_SERVER_RUNTIME_TEST_DSN: 'home_server_ci:ci-isolated-tests@tcp(127.0.0.1:3306)/home_server_accounts_runtime_test?parseTime=true' HOME_SERVER_ANALYTICS_TEST_DSN: 'home_server_ci:ci-isolated-tests@tcp(127.0.0.1:3306)/home_server_analytics_test_ci?parseTime=true' HOME_SERVER_READ_CACHE_TEST_DSN: 'home_server_ci:ci-isolated-tests@tcp(127.0.0.1:3306)/home_server_readcache_test_ci?parseTime=true' - run: go test -v ./config ./api/accounts ./api/webprojects ./api/applications ./api/auth ./api/middleware ./api/passport ./api/webdav ./app/... ./domain/model ./domain/repository/... ./domain/service/accounts ./domain/service/webprojects ./domain/service/webcomments ./domain/service/applications ./domain/service/passport ./domain/service/rsa ./internal/accountsmigrate ./cmd/accounts-migrate ./cmd/accounts-maintain ./utils ./utils/md ./utils/log ./utils/ginfmt ./utils/cache ./domain/service/webanalytics + MANUALS_TEST_DSN: 'home_server_ci:ci-isolated-tests@tcp(127.0.0.1:3306)/home_server_manuals_test_ci?parseTime=true' + run: go test -v ./config ./api/accounts ./api/webprojects ./api/manuals ./api/applications ./api/auth ./api/middleware ./api/passport ./api/webdav ./app/... ./domain/model ./domain/repository/... ./domain/service/accounts ./domain/service/manuals ./domain/service/webprojects ./domain/service/webcomments ./domain/service/applications ./domain/service/passport ./domain/service/rsa ./internal/accountsmigrate ./cmd/accounts-migrate ./cmd/accounts-maintain ./utils ./utils/md ./utils/log ./utils/ginfmt ./utils/cache ./domain/service/webanalytics - name: Test isolated DAL cache bindings env: + MANUALS_TEST_DSN: 'home_server_ci:ci-isolated-tests@tcp(127.0.0.1:3306)/home_server_manuals_test_ci?parseTime=true' HOME_SERVER_TEST_REDIS_ADDR: '127.0.0.1:16389' HOME_SERVER_READ_CACHE_TEST_DSN: 'home_server_ci:ci-isolated-tests@tcp(127.0.0.1:3306)/home_server_readcache_test_ci?parseTime=true' - run: go test -v ./domain/dal -run TestReadCache -count=1 + run: go test -v ./domain/dal -run 'Test(ReadCache|Manual)' -count=1 - name: Test AI API client run: python3 script/test_home_server_api.py -v @@ -86,6 +90,9 @@ jobs: - name: Test web hosting skill over isolated HTTPS run: python3 script/test_web_hosting_skill.py -v + - name: Test manuals skill over isolated HTTPS + run: python3 script/test_manuals_skill.py -v + - name: Use public npm registry in the disposable CI checkout working-directory: front_vue run: python3 -c "from pathlib import Path; path = Path('package-lock.json'); path.write_text(path.read_text(encoding='utf-8').replace('http://bnpm.byted.org/', 'https://registry.npmjs.org/'), encoding='utf-8')" diff --git a/README.md b/README.md index d00fe8b..90e00b5 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,21 @@ cq_ddns_client -conf /etc/home_server/client_config.yaml -dry-run 回滚时停用新服务，再启动原 `home_client.service`。切换验证完成前保留旧二进制、配置和 unit。 +## 说明书管理 + +说明书将同一设备的 PDF、图片、文字和链接放在一起，支持每份说明书归属多个个人单层分类、名称包含关键词检索和缩略图列表。新建默认仅自己可见，也可选择登录用户或所有人可见；仅所有者可以编辑和删除。 + +| 操作 | 行为 | +| --- | --- | +| 批量录入 | 图片可多选并多次追加，PDF、文字和 URL 可同时录入；上传失败保留成功项，重新选择或重试失败项 | +| 查阅 | 按分类和名称组合筛选，点击封面进入有序资料列表；PDF 可打开或下载，URL 打开原站 | +| 分享 | 分享说明书地址，访问者按当前可见范围鉴权；缩略图和原件受相同权限控制 | +| 删除 | 逻辑删除后立即停止访问，原始文件保留并继续占用存储空间 | + +系统相册是否支持一次跨相册选择取决于手机；页面会保留多次选择的队列。图片支持 JPEG、PNG、GIF、WebP，HEIC/HEIF 需要先转换。PDF 预览使用 `pdftoppm`；无法生成预览时保留原件并显示占位状态。URL 只保存地址，需要长期归档的内容应上传文件。 + +设计、接口与默认限额见 [说明书管理技术方案](docs/specs/manual-management/design.md)，安装及回滚见 [Pi 部署说明](deploy/pi/README.md)。AI 操作入口包含在 [home-server-web-share skill](skills/home-server-web-share/SKILL.md) 中，使用独立的 `manuals:read` / `manuals:write` scope；旧凭证不会自动扩大权限。 + ## 网页托管 ### 使用方式 diff --git a/api/accounts/handlers.go b/api/accounts/handlers.go index d5661e4..9646ffb 100644 --- a/api/accounts/handlers.go +++ b/api/accounts/handlers.go @@ -113,7 +113,7 @@ type accountView struct { func view(user *model.UserAccount, token string) accountView { runtime := config.Runtime() - return accountView{UserAccount: user, CSRFToken: middleware.CSRFToken(token), Capabilities: map[string]bool{"library": user.LibraryEnabled && runtime.LibraryEnabled, "webdav": user.WebDAVPermission != model.WebDAVNone && runtime.WebDAVEnabled, "applications": runtime.Auth.ApplicationsEnabled, "web_projects": runtime.WebProjects.Enabled}, PasswordPolicy: map[string]int{"min_length": runtime.AccountPolicy.MinPasswordLength}, ApplicationPolicy: map[string]int{"default_credential_ttl_days": runtime.Auth.DefaultCredentialTTLDays, "max_credential_ttl_days": runtime.Auth.MaxCredentialTTLDays, "max_applications_per_user": runtime.Auth.MaxApplicationsPerUser}, SessionPolicy: map[string]int{"max_active_sessions": runtime.AccountPolicy.MaxActiveSessions}} + return accountView{UserAccount: user, CSRFToken: middleware.CSRFToken(token), Capabilities: map[string]bool{"library": user.LibraryEnabled && runtime.LibraryEnabled, "webdav": user.WebDAVPermission != model.WebDAVNone && runtime.WebDAVEnabled, "applications": runtime.Auth.ApplicationsEnabled, "web_projects": runtime.WebProjects.Enabled, "manuals": runtime.Manuals.Enabled}, PasswordPolicy: map[string]int{"min_length": runtime.AccountPolicy.MinPasswordLength}, ApplicationPolicy: map[string]int{"default_credential_ttl_days": runtime.Auth.DefaultCredentialTTLDays, "max_credential_ttl_days": runtime.Auth.MaxCredentialTTLDays, "max_applications_per_user": runtime.Auth.MaxApplicationsPerUser}, SessionPolicy: map[string]int{"max_active_sessions": runtime.AccountPolicy.MaxActiveSessions}} } // login 处理 POST /api/auth/login：验证用户名和密码，建立 HttpOnly 浏览器会话并返回本人资料与 CSRF 信息。 @@ -247,7 +247,7 @@ func publicBootstrap(c *gin.Context) { return } runtime := config.Runtime() - respond(c, map[string]interface{}{"site": map[string]string{"title": runtime.SiteTitle, "notice": runtime.SiteNotice}, "registration": value}, nil) + respond(c, map[string]interface{}{"site": map[string]string{"title": runtime.SiteTitle, "notice": runtime.SiteNotice}, "registration": value, "modules": map[string]bool{"manuals": runtime.Manuals.Enabled}}, nil) } // validateInvitation 处理 POST /api/auth/invitations/validate：在来源限流后校验邀请码，仅返回有效状态与到期时间，不消费邀请码。 diff --git a/api/handler.go b/api/handler.go index 7f268a7..1e221f1 100644 --- a/api/handler.go +++ b/api/handler.go @@ -6,6 +6,7 @@ import ( "github.com/mcoder2014/home_server/api/applications" "github.com/mcoder2014/home_server/api/auth" "github.com/mcoder2014/home_server/api/library" + "github.com/mcoder2014/home_server/api/manuals" "github.com/mcoder2014/home_server/api/middleware" "github.com/mcoder2014/home_server/api/passport" "github.com/mcoder2014/home_server/api/webdav" @@ -35,6 +36,7 @@ func InitRouter() error { webdav.InitRouter, // 静态网页托管相关接口 webprojects.InitRouter, + manuals.InitRouter, auth.InitRouter, applications.InitRouter, } { diff --git a/api/manuals/categories_pagination_integration_test.go b/api/manuals/categories_pagination_integration_test.go new file mode 100644 index 0000000..0dda754 --- /dev/null +++ b/api/manuals/categories_pagination_integration_test.go @@ -0,0 +1,28 @@ +package manuals + +import ( + "net/http" + "net/url" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestManualCategoriesUseBinaryCursorPagination(t *testing.T) { + fixture := newManualHTTPFixture(t) + requireManualSuccess(t, fixture.request(http.MethodPost, "/api/manuals", fixture.owner, map[string]interface{}{ + "name": "分类分页", "categories": []string{"厨房", "a", "A"}, "access_mode": "owner", "client_request_id": "category-page-manual", + }, nil)) + + first := requireManualSuccess(t, fixture.request(http.MethodGet, "/api/manuals/categories?mine=true&limit=2", fixture.owner, nil, nil)) + require.Equal(t, []interface{}{"A", "a"}, first["items"]) + require.Equal(t, true, first["has_more"]) + require.Equal(t, "a", first["next_cursor"]) + + second := requireManualSuccess(t, fixture.request(http.MethodGet, "/api/manuals/categories?mine=true&limit=2&cursor="+url.QueryEscape(first["next_cursor"].(string)), fixture.owner, nil, nil)) + require.Equal(t, []interface{}{"厨房"}, second["items"]) + require.Equal(t, false, second["has_more"]) + require.Equal(t, "", second["next_cursor"]) + + requireManualDenied(t, fixture.request(http.MethodGet, "/api/manuals/categories?mine=true&limit=1001", fixture.owner, nil, nil)) +} diff --git a/api/manuals/content.go b/api/manuals/content.go new file mode 100644 index 0000000..aa2f912 --- /dev/null +++ b/api/manuals/content.go @@ -0,0 +1,85 @@ +package manuals + +import ( + "mime" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + application "github.com/mcoder2014/home_server/app/manuals" + "github.com/mcoder2014/home_server/config" + "github.com/mcoder2014/home_server/domain/model" + service "github.com/mcoder2014/home_server/domain/service/manuals" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/mcoder2014/home_server/utils/ginfmt" +) + +func serveItemResource(c *gin.Context) { + manualID, err := pathID(c, "id") + if err != nil { + ginfmt.Fail(c, err) + return + } + itemID, err := pathID(c, "item_id") + if err != nil { + ginfmt.Fail(c, err) + return + } + principal, err := optionalReader(c) + if err != nil { + ginfmt.Fail(c, err) + return + } + thumbnail := strings.HasSuffix(c.FullPath(), "/thumbnail") + item, err := application.Default.ResourceItem(ginfmt.RPCContext(c), manualID, itemID, principalUserID(principal), thumbnail) + if err != nil { + ginfmt.Fail(c, err) + return + } + key := item.StorageKey + if thumbnail { + key = item.ThumbnailKey + } + if key == nil { + ginfmt.Fail(c, apperrors.ErrNotFound) + return + } + conf := config.Global().Manuals + file, info, err := service.OpenStoredFile(&conf, *key) + if err != nil { + ginfmt.Fail(c, err) + return + } + defer file.Close() + setResourceHeaders(c, item, thumbnail) + http.ServeContent(c.Writer, c.Request, resourceFileName(item, thumbnail), info.ModTime(), file) +} + +func setResourceHeaders(c *gin.Context, item *model.ManualItem, thumbnail bool) { + c.Header("Cache-Control", "private, no-store") + c.Header("X-Content-Type-Options", "nosniff") + c.Header("Cross-Origin-Resource-Policy", "same-origin") + contentType, disposition := item.ContentType, "inline" + if thumbnail { + contentType = "image/jpeg" + } else if c.Query("download") == "1" { + disposition = "attachment" + } + if item.Kind == model.ManualItemPDF { + c.Header("Content-Security-Policy", "sandbox") + } + c.Header("Content-Type", contentType) + if value := mime.FormatMediaType(disposition, map[string]string{"filename": resourceFileName(item, thumbnail)}); value != "" { + c.Header("Content-Disposition", value) + } +} + +func resourceFileName(item *model.ManualItem, thumbnail bool) string { + if thumbnail { + return "thumbnail.jpg" + } + if item.OriginalName != "" { + return item.OriginalName + } + return "manual-item" +} diff --git a/api/manuals/handlers.go b/api/manuals/handlers.go new file mode 100644 index 0000000..7b2a270 --- /dev/null +++ b/api/manuals/handlers.go @@ -0,0 +1,310 @@ +package manuals + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/mcoder2014/home_server/api/middleware" + application "github.com/mcoder2014/home_server/app/manuals" + "github.com/mcoder2014/home_server/config" + service "github.com/mcoder2014/home_server/domain/service/manuals" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/mcoder2014/home_server/utils" + "github.com/mcoder2014/home_server/utils/ginfmt" +) + +func listManuals(c *gin.Context) { + principal, err := optionalReader(c) + if err != nil { + ginfmt.Fail(c, err) + return + } + mine, err := queryBool(c, "mine") + if err != nil { + ginfmt.Fail(c, err) + return + } + if mine && principal == nil { + ginfmt.Fail(c, apperrors.ErrUnauthorized) + return + } + cursor, limit, err := pagination(c) + if err != nil { + ginfmt.Fail(c, err) + return + } + viewerID := principalUserID(principal) + page, err := application.Default.List(ginfmt.RPCContext(c), viewerID, mine, cursor, limit, c.Query("q"), c.Query("category"), c.Request.URL.Query().Has("category"), principal) + respond(c, http.StatusOK, page, err) +} + +func listCategories(c *gin.Context) { + principal, err := optionalReader(c) + if err != nil { + ginfmt.Fail(c, err) + return + } + mine, err := queryBool(c, "mine") + if err != nil { + ginfmt.Fail(c, err) + return + } + if mine && principal == nil { + ginfmt.Fail(c, apperrors.ErrUnauthorized) + return + } + cursor, limit, err := categoryPagination(c) + if err != nil { + ginfmt.Fail(c, err) + return + } + page, err := application.Default.Categories(ginfmt.RPCContext(c), principalUserID(principal), mine, cursor, limit) + respond(c, http.StatusOK, page, err) +} + +func createManual(c *gin.Context) { + var input service.CreateManualInput + if !bindJSON(c, &input, 32<<10) { + return + } + principal := currentPrincipal(c) + manual, err := application.Default.Create(ginfmt.RPCContext(c), principal.UserID, input, principal) + respond(c, http.StatusCreated, manual, err) +} + +func getManual(c *gin.Context) { + manualID, err := pathID(c, "id") + if err != nil { + ginfmt.Fail(c, err) + return + } + principal, err := optionalReader(c) + if err != nil { + ginfmt.Fail(c, err) + return + } + manual, err := application.Default.Get(ginfmt.RPCContext(c), manualID, principalUserID(principal), principal) + respond(c, http.StatusOK, manual, err) +} + +func updateManual(c *gin.Context) { + manualID, err := pathID(c, "id") + if err != nil { + ginfmt.Fail(c, err) + return + } + var input service.UpdateManualInput + if !bindJSON(c, &input, 128<<10) { + return + } + principal := currentPrincipal(c) + manual, err := application.Default.Update(ginfmt.RPCContext(c), principal.UserID, manualID, input, principal) + respond(c, http.StatusOK, manual, err) +} + +func deleteManual(c *gin.Context) { + manualID, err := pathID(c, "id") + if err != nil { + ginfmt.Fail(c, err) + return + } + input := struct { + Revision int64 `json:"revision"` + }{} + if !bindJSON(c, &input, 4096) { + return + } + principal := currentPrincipal(c) + revision, err := application.Default.Delete(ginfmt.RPCContext(c), principal.UserID, manualID, input.Revision, principal) + respond(c, http.StatusOK, gin.H{"id": strconv.FormatInt(manualID, 10), "revision": revision}, err) +} + +func addInlineItem(c *gin.Context) { + manualID, err := pathID(c, "id") + if err != nil { + ginfmt.Fail(c, err) + return + } + var input service.InlineItemInput + if !bindJSON(c, &input, 512<<10) { + return + } + principal := currentPrincipal(c) + result, err := application.Default.AddInline(ginfmt.RPCContext(c), principal.UserID, manualID, input, principal) + respond(c, http.StatusCreated, result, err) +} + +func addFileItem(c *gin.Context) { + manualID, err := pathID(c, "id") + if err != nil { + ginfmt.Fail(c, err) + return + } + principal := currentPrincipal(c) + if err := application.Default.CheckUploadOwner(ginfmt.RPCContext(c), principal.UserID, manualID); err != nil { + ginfmt.Fail(c, err) + return + } + conf := config.Global().Manuals + release, err := application.Default.AcquireUpload(&conf) + if err != nil { + ginfmt.Fail(c, err) + return + } + defer release() + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, conf.MaxFileBytes+(2<<20)) + if err := c.Request.ParseMultipartForm(1 << 20); err != nil { + ginfmt.Fail(c, classifyMultipartError(err)) + return + } + if c.Request.MultipartForm != nil { + defer c.Request.MultipartForm.RemoveAll() + } + title, requestID, header, err := multipartFields(c) + if err != nil { + ginfmt.Fail(c, err) + return + } + file, err := header.Open() + if err != nil { + ginfmt.Fail(c, apperrors.ErrInvalid) + return + } + defer file.Close() + result, err := application.Default.AddReservedFile(ginfmt.RPCContext(c), principal.UserID, manualID, title, requestID, header.Filename, file, principal) + respond(c, http.StatusCreated, result, err) +} + +func deleteItem(c *gin.Context) { + manualID, err := pathID(c, "id") + if err != nil { + ginfmt.Fail(c, err) + return + } + itemID, err := pathID(c, "item_id") + if err != nil { + ginfmt.Fail(c, err) + return + } + input := struct { + Revision int64 `json:"revision"` + }{} + if !bindJSON(c, &input, 4096) { + return + } + principal := currentPrincipal(c) + result, err := application.Default.DeleteItem(ginfmt.RPCContext(c), principal.UserID, manualID, itemID, input.Revision, principal) + respond(c, http.StatusOK, result, err) +} + +func bindJSON(c *gin.Context, value interface{}, limit int64) bool { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, limit) + decoder := json.NewDecoder(c.Request.Body) + decoder.DisallowUnknownFields() + if err := decoder.Decode(value); err != nil { + ginfmt.Fail(c, apperrors.ErrInvalid) + return false + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + ginfmt.Fail(c, apperrors.ErrInvalid) + return false + } + return true +} + +func respond(c *gin.Context, status int, value interface{}, err error) { + if err != nil { + ginfmt.Fail(c, err) + return + } + ginfmt.Success(c, status, value) +} + +func currentPrincipal(c *gin.Context) *utils.Principal { + value, _ := c.Get(utils.CtxKeyPrincipal) + principal, _ := value.(*utils.Principal) + return principal +} + +func principalUserID(principal *utils.Principal) int64 { + if principal == nil { + return 0 + } + return principal.UserID +} + +func pathID(c *gin.Context, name string) (int64, error) { + value, err := strconv.ParseInt(c.Param(name), 10, 64) + if err != nil || value <= 0 || strconv.FormatInt(value, 10) != c.Param(name) { + return 0, apperrors.ErrInvalid + } + return value, nil +} + +func pagination(c *gin.Context) (int64, int, error) { + cursor, limit := int64(0), 20 + var err error + if raw := c.Query("cursor"); raw != "" { + cursor, err = strconv.ParseInt(raw, 10, 64) + if err != nil || cursor <= 0 { + return 0, 0, apperrors.ErrInvalid + } + } + if raw := c.Query("limit"); raw != "" { + limit, err = strconv.Atoi(raw) + if err != nil || limit < 1 || limit > 100 { + return 0, 0, apperrors.ErrInvalid + } + } + return cursor, limit, nil +} + +func categoryPagination(c *gin.Context) (string, int, error) { + cursor, limit := c.Query("cursor"), 200 + if cursor != "" { + categories, err := service.ValidateCategories([]string{cursor}) + if err != nil || len(categories) != 1 || categories[0] != cursor { + return "", 0, apperrors.ErrInvalid + } + } + if raw := c.Query("limit"); raw != "" { + var err error + limit, err = strconv.Atoi(raw) + if err != nil || limit < 1 || limit > 1000 { + return "", 0, apperrors.ErrInvalid + } + } + return cursor, limit, nil +} + +func queryBool(c *gin.Context, name string) (bool, error) { + value := c.Query(name) + if value == "" { + return false, nil + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return false, apperrors.ErrInvalid + } + return parsed, nil +} + +func optionalReader(c *gin.Context) (*utils.Principal, error) { + explicit := len(c.Request.Header.Values("Authorization")) > 0 || len(c.Request.Header.Values(middleware.HeaderKey)) > 0 + hasSession := utils.BrowserSessionToken(c.Request) != "" + if !explicit && !hasSession { + return nil, nil + } + principal, err := middleware.ResolveIdentity(c, "manuals:read", true, false) + if err == nil { + return principal, nil + } + if errors.Is(err, apperrors.ErrDependency) || explicit { + return nil, err + } + return nil, nil +} diff --git a/api/manuals/integration_test.go b/api/manuals/integration_test.go new file mode 100644 index 0000000..15bb450 --- /dev/null +++ b/api/manuals/integration_test.go @@ -0,0 +1,464 @@ +package manuals + +import ( + "bytes" + "context" + "crypto/sha256" + "crypto/tls" + "database/sql" + "encoding/base64" + "encoding/json" + "image" + "image/color" + "image/png" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-sql-driver/mysql" + "github.com/mcoder2014/home_server/api/middleware" + "github.com/mcoder2014/home_server/app/siteconfig" + "github.com/mcoder2014/home_server/config" + "github.com/mcoder2014/home_server/data" + "github.com/mcoder2014/home_server/domain/dal/migrations" + "github.com/mcoder2014/home_server/domain/db" + applicationservice "github.com/mcoder2014/home_server/domain/service/applications" + manualservice "github.com/mcoder2014/home_server/domain/service/manuals" + "github.com/mcoder2014/home_server/utils" + "github.com/stretchr/testify/require" + "gorm.io/gorm/logger" +) + +const manualTestOrigin = "https://manuals.example.test" + +type manualHTTPSession struct { + cookie *http.Cookie + csrf string +} + +type manualHTTPResponse struct { + status int + code int + data map[string]interface{} + body []byte + header http.Header +} + +type manualHTTPFixture struct { + t *testing.T + router *gin.Engine + database *sql.DB + storageRoot string + owner, member *manualHTTPSession + readToken string + ownerID int64 + originalConfig config.Config + originalRoutes map[string]map[string]data.HttpRoute +} + +func newManualHTTPFixture(t *testing.T) *manualHTTPFixture { + t.Helper() + dsn := os.Getenv("MANUALS_TEST_DSN") + if dsn == "" { + t.Skip("MANUALS_TEST_DSN is required for manual HTTP integration tests") + } + parsed, err := mysql.ParseDSN(dsn) + require.NoError(t, err) + require.True(t, strings.HasPrefix(parsed.DBName, "home_server_manuals_test_"), "refuse non-fixture database") + parsed.MultiStatements = false + parsed.ParseTime = true + sqlDB, err := sql.Open("mysql", parsed.FormatDSN()) + require.NoError(t, err) + dropManualTestTables(t, sqlDB, parsed.DBName) + for _, ddl := range []string{manualAccountTestDDL, manualSessionTestDDL} { + _, err = sqlDB.Exec(ddl) + require.NoError(t, err) + } + for _, name := range []string{"20260913_runtime_config.sql", "20260912_applications.sql", "20260919_manuals.sql"} { + raw, readErr := migrations.SQL.ReadFile(name) + require.NoError(t, readErr) + executeManualTestDDL(t, sqlDB, string(raw)) + } + + root := t.TempDir() + conf := config.Config{IdentitySource: "database", ConfigSource: "database"} + conf.Auth.ApplicationsEnabled = true + conf.Auth.SiteOrigin = manualTestOrigin + conf.Manuals.Enabled = true + conf.Manuals.StorageRoot = filepath.Join(root, "manuals") + require.NoError(t, manualservice.Init(&conf.Manuals, "", "")) + seedManualRuntimeConfig(t, sqlDB, conf) + seedManualIdentities(t, sqlDB) + readToken := seedManualApplication(t, sqlDB) + + originalConfig, originalRoutes := config.Global(), data.RouterMap + config.SetGlobalConfig(conf) + require.NoError(t, db.InitDatabase(parsed.FormatDSN())) + db.MasterDB().Logger = logger.Discard + runtimeService := siteconfig.New(db.MasterDB(), conf) + require.NoError(t, runtimeService.Initialize(context.Background())) + require.NoError(t, applicationservice.Init(conf.Auth)) + require.NoError(t, middleware.ConfigureAuthentication(conf.Auth)) + + data.RouterMap = map[string]map[string]data.HttpRoute{} + require.NoError(t, InitRouter()) + gin.SetMode(gin.TestMode) + router := gin.New() + router.Use(gin.RecoveryWithWriter(io.Discard), middleware.AddLogID) + data.ForRange(func(method, path string, handlers ...gin.HandlerFunc) { + router.Handle(method, path, handlers...) + }) + fixture := &manualHTTPFixture{ + t: t, router: router, database: sqlDB, storageRoot: conf.Manuals.StorageRoot, + owner: manualBrowserSession(t, sqlDB, 1001, "owner-session-token"), + member: manualBrowserSession(t, sqlDB, 1002, "member-session-token"), + readToken: readToken, ownerID: 1001, originalConfig: originalConfig, originalRoutes: originalRoutes, + } + t.Cleanup(func() { + config.SetGlobalConfig(fixture.originalConfig) + data.RouterMap = fixture.originalRoutes + if connection, connectionErr := db.MasterDB().DB(); connectionErr == nil { + _ = connection.Close() + } + _ = sqlDB.Close() + }) + return fixture +} + +func dropManualTestTables(t *testing.T, database *sql.DB, schema string) { + t.Helper() + rows, err := database.Query("SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?", schema) + require.NoError(t, err) + var tables []string + for rows.Next() { + var table string + require.NoError(t, rows.Scan(&table)) + tables = append(tables, table) + } + require.NoError(t, rows.Close()) + for _, table := range tables { + _, err = database.Exec("DROP TABLE `" + strings.ReplaceAll(table, "`", "``") + "`") + require.NoError(t, err) + } +} + +func executeManualTestDDL(t *testing.T, database *sql.DB, raw string) { + t.Helper() + lines := make([]string, 0) + for _, line := range strings.Split(raw, "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "--") { + lines = append(lines, line) + } + } + for _, statement := range strings.Split(strings.Join(lines, "\n"), ";") { + statement = strings.TrimSpace(statement) + if statement == "" { + continue + } + require.False(t, strings.HasPrefix(strings.ToUpper(statement), "USE ")) + _, err := database.Exec(statement) + require.NoError(t, err) + } +} + +func seedManualRuntimeConfig(t *testing.T, database *sql.DB, conf config.Config) { + t.Helper() + now := time.Now().UTC() + _, err := database.Exec("INSERT INTO site_runtime_state (id, config_generation, registration_epoch, revision, update_time) VALUES (1, 1, 1, 1, ?)", now) + require.NoError(t, err) + for namespace, values := range config.DefaultRuntimeValues(conf) { + raw, marshalErr := json.Marshal(values) + require.NoError(t, marshalErr) + digest := sha256.Sum256(raw) + _, err = database.Exec("INSERT INTO site_config_current (namespace, revision, schema_version, values_json, values_sha256, updated_by, update_time) VALUES (?, 1, 1, ?, ?, 1001, ?)", namespace, string(raw), digest[:], now) + require.NoError(t, err) + } +} + +func seedManualIdentities(t *testing.T, database *sql.DB) { + t.Helper() + now := time.Now().UTC() + for id, name := range map[int64]string{1001: "owner", 1002: "member"} { + _, err := database.Exec("INSERT INTO user_account (id, username, username_key, display_name, avatar_version, contact_email, contact_mobile, password_hash, status, role, library_enabled, webdav_permission, auth_version, revision, must_change_password, invite_eligible_at, source, create_time, update_time) VALUES (?, ?, ?, ?, 0, '', '', '', 'active', 'user', FALSE, 'none', 1, 1, FALSE, ?, 'test', ?, ?)", id, name, name, name, now, now, now) + require.NoError(t, err) + } +} + +func manualBrowserSession(t *testing.T, database *sql.DB, userID int64, token string) *manualHTTPSession { + t.Helper() + now := time.Now().UTC() + digest := sha256.Sum256([]byte(token)) + _, err := database.Exec("INSERT INTO login_token (id, user_id, token_digest, auth_version, purpose, is_expired, authenticated_at, login_ip, user_agent, client_name, os_name, device_type, login_source, expire_time, create_time, update_time) VALUES (?, ?, ?, 1, 'user', 0, ?, '127.0.0.1', 'manual-test', 'test', 'test', 'test', 'test', ?, ?, ?)", userID, userID, digest[:], now, now.Add(time.Hour), now, now) + require.NoError(t, err) + return &manualHTTPSession{cookie: &http.Cookie{Name: utils.SessionCookieName, Value: token, Path: "/", Secure: true, HttpOnly: true}, csrf: middleware.CSRFToken(token)} +} + +func seedManualApplication(t *testing.T, database *sql.DB) string { + t.Helper() + rawToken := "at_cq_" + base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{7}, 32)) + digest := sha256.Sum256([]byte(rawToken)) + now := time.Now().UTC() + scopes := `["manuals:read"]` + _, err := database.Exec("INSERT INTO application (id, owner_user_id, name, description, access_key, secret_digest, scopes, status, active_slot, revision, secret_version, expires_at, create_time, update_time) VALUES (2001, 1001, 'reader', '', 'ak_cq_AAAAAAAAAAAAAAAAAAAAAA', ?, ?, 1, 1, 1, 1, ?, ?, ?)", make([]byte, 32), scopes, now.Add(time.Hour), now, now) + require.NoError(t, err) + _, err = database.Exec("INSERT INTO application_access_token (id, application_id, token_digest, secret_version, application_revision, scope_snapshot, expired_at, create_time) VALUES (3001, 2001, ?, 1, 1, ?, ?, ?)", digest[:], scopes, now.Add(time.Hour), now) + require.NoError(t, err) + return rawToken +} + +func (fixture *manualHTTPFixture) request(method, path string, session *manualHTTPSession, body interface{}, headers map[string]string) manualHTTPResponse { + fixture.t.Helper() + var reader io.Reader + switch value := body.(type) { + case nil: + reader = http.NoBody + case io.Reader: + reader = value + case []byte: + reader = bytes.NewReader(value) + default: + raw, err := json.Marshal(value) + require.NoError(fixture.t, err) + reader = bytes.NewReader(raw) + } + request := httptest.NewRequest(method, manualTestOrigin+path, reader) + request.TLS = &tls.ConnectionState{HandshakeComplete: true} + request.RemoteAddr = "192.0.2.10:45000" + request.Header.Set("Origin", manualTestOrigin) + request.Header.Set("Content-Type", "application/json") + if session != nil { + request.AddCookie(session.cookie) + request.Header.Set("X-CSRF-Token", session.csrf) + } + for key, value := range headers { + if value == "" { + request.Header.Del(key) + } else { + request.Header.Set(key, value) + } + } + recorder := httptest.NewRecorder() + fixture.router.ServeHTTP(recorder, request) + response := manualHTTPResponse{status: recorder.Code, body: append([]byte(nil), recorder.Body.Bytes()...), header: recorder.Header()} + var envelope struct { + Code int `json:"code"` + Data map[string]interface{} `json:"data"` + } + if json.Unmarshal(response.body, &envelope) == nil { + response.code, response.data = envelope.Code, envelope.Data + } + return response +} + +func (fixture *manualHTTPFixture) multipart(fields map[string]string, filename string, content []byte) (*bytes.Buffer, string) { + fixture.t.Helper() + buffer := &bytes.Buffer{} + writer := multipart.NewWriter(buffer) + for name, value := range fields { + require.NoError(fixture.t, writer.WriteField(name, value)) + } + part, err := writer.CreateFormFile("file", filename) + require.NoError(fixture.t, err) + _, err = part.Write(content) + require.NoError(fixture.t, err) + require.NoError(fixture.t, writer.Close()) + return buffer, writer.FormDataContentType() +} + +func requireManualSuccess(t *testing.T, response manualHTTPResponse) map[string]interface{} { + t.Helper() + if response.status < 200 || response.status >= 300 || response.code != 0 { + t.Fatalf("manual request failed: status=%d code=%d body=%s", response.status, response.code, response.body) + } + return response.data +} + +func requireManualDenied(t *testing.T, response manualHTTPResponse) { + t.Helper() + if response.status < 400 || response.status >= 500 || response.code == 0 { + t.Fatalf("manual request was not safely denied: status=%d code=%d body=%s", response.status, response.code, response.body) + } +} + +func manualRevision(value map[string]interface{}) int64 { + revision, _ := value["revision"].(float64) + return int64(revision) +} + +func TestManualHTTPPermissionsIdempotencyAndProtectedFiles(t *testing.T) { + fixture := newManualHTTPFixture(t) + create := map[string]interface{}{"name": "冰箱100%_!", "description": "家庭说明", "categories": []string{"厨房", "Kitchen"}, "access_mode": "owner", "client_request_id": "create-1"} + requireManualDenied(t, fixture.request(http.MethodPost, "/api/manuals", fixture.owner, create, map[string]string{"X-CSRF-Token": ""})) + created := requireManualSuccess(t, fixture.request(http.MethodPost, "/api/manuals", fixture.owner, create, nil)) + manualID := created["id"].(string) + require.Equal(t, int64(1), manualRevision(created)) + require.ElementsMatch(t, []interface{}{"Kitchen", "厨房"}, created["categories"].([]interface{})) + replayed := requireManualSuccess(t, fixture.request(http.MethodPost, "/api/manuals", fixture.owner, create, nil)) + require.Equal(t, manualID, replayed["id"]) + require.Equal(t, int64(1), manualRevision(replayed)) + conflictingCreate := map[string]interface{}{"name": "另一份", "categories": []string{}, "access_mode": "owner", "client_request_id": "create-1"} + requireManualDenied(t, fixture.request(http.MethodPost, "/api/manuals", fixture.owner, conflictingCreate, nil)) + + inline := map[string]interface{}{"kind": "text", "title": "保养", "text": "每月清洁滤网", "client_request_id": "inline-1"} + inlineResult := requireManualSuccess(t, fixture.request(http.MethodPost, "/api/manuals/"+manualID+"/items", fixture.owner, inline, nil)) + inlineItem := inlineResult["item"].(map[string]interface{}) + require.Equal(t, int64(2), manualRevision(inlineResult)) + inlineReplay := requireManualSuccess(t, fixture.request(http.MethodPost, "/api/manuals/"+manualID+"/items", fixture.owner, inline, nil)) + require.Equal(t, inlineItem["id"], inlineReplay["item"].(map[string]interface{})["id"]) + require.Equal(t, int64(2), manualRevision(inlineReplay)) + + private := requireManualSuccess(t, fixture.request(http.MethodPatch, "/api/manuals/"+manualID, fixture.owner, map[string]interface{}{"revision": 2, "status": "active"}, nil)) + require.Equal(t, int64(3), manualRevision(private)) + requireManualSuccess(t, fixture.request(http.MethodGet, "/api/manuals/"+manualID, fixture.owner, nil, nil)) + requireManualDenied(t, fixture.request(http.MethodGet, "/api/manuals/"+manualID, fixture.member, nil, nil)) + requireManualDenied(t, fixture.request(http.MethodGet, "/api/manuals/"+manualID, nil, nil, nil)) + requireManualSuccess(t, fixture.request(http.MethodGet, "/api/manuals/"+manualID, nil, nil, map[string]string{"Authorization": "Bearer " + fixture.readToken})) + requireManualDenied(t, fixture.request(http.MethodPost, "/api/manuals/"+manualID+"/items", nil, map[string]interface{}{"kind": "url", "url": "https://example.com", "client_request_id": "app-write"}, map[string]string{"Authorization": "Bearer " + fixture.readToken})) + + authenticated := requireManualSuccess(t, fixture.request(http.MethodPatch, "/api/manuals/"+manualID, fixture.owner, map[string]interface{}{"revision": 3, "access_mode": "authenticated"}, nil)) + require.Equal(t, int64(4), manualRevision(authenticated)) + requireManualSuccess(t, fixture.request(http.MethodGet, "/api/manuals/"+manualID, fixture.member, nil, nil)) + requireManualDenied(t, fixture.request(http.MethodGet, "/api/manuals/"+manualID, nil, nil, nil)) + public := requireManualSuccess(t, fixture.request(http.MethodPatch, "/api/manuals/"+manualID, fixture.owner, map[string]interface{}{"revision": 4, "access_mode": "public"}, nil)) + require.Equal(t, int64(5), manualRevision(public)) + requireManualSuccess(t, fixture.request(http.MethodGet, "/api/manuals/"+manualID, nil, nil, nil)) + requireManualDenied(t, fixture.request(http.MethodGet, "/api/manuals/"+manualID, nil, nil, map[string]string{"Authorization": "Bearer invalid"})) + stale := &manualHTTPSession{cookie: &http.Cookie{Name: utils.SessionCookieName, Value: "missing-session-token", Path: "/"}} + requireManualSuccess(t, fixture.request(http.MethodGet, "/api/manuals/"+manualID, stale, nil, nil)) + private = requireManualSuccess(t, fixture.request(http.MethodPatch, "/api/manuals/"+manualID, fixture.owner, map[string]interface{}{"revision": 5, "access_mode": "owner"}, nil)) + require.Equal(t, int64(6), manualRevision(private)) + requireManualDenied(t, fixture.request(http.MethodGet, "/api/manuals/"+manualID, nil, nil, nil)) + public = requireManualSuccess(t, fixture.request(http.MethodPatch, "/api/manuals/"+manualID, fixture.owner, map[string]interface{}{"revision": 6, "access_mode": "public"}, nil)) + require.Equal(t, int64(7), manualRevision(public)) + + pngBytes := manualPNG(t) + upload, contentType := fixture.multipart(map[string]string{"title": "铭牌", "client_request_id": "image-1"}, "plate.png", pngBytes) + uploaded := requireManualSuccess(t, fixture.request(http.MethodPost, "/api/manuals/"+manualID+"/files", fixture.owner, upload, map[string]string{"Content-Type": contentType})) + imageItem := uploaded["item"].(map[string]interface{}) + imageID := imageItem["id"].(string) + require.Equal(t, "ready", imageItem["preview_status"]) + require.Equal(t, int64(8), manualRevision(uploaded)) + upload, contentType = fixture.multipart(map[string]string{"title": "铭牌", "client_request_id": "image-1"}, "plate.png", pngBytes) + uploadReplay := requireManualSuccess(t, fixture.request(http.MethodPost, "/api/manuals/"+manualID+"/files", fixture.owner, upload, map[string]string{"Content-Type": contentType})) + require.Equal(t, imageID, uploadReplay["item"].(map[string]interface{})["id"]) + require.Equal(t, int64(8), manualRevision(uploadReplay)) + + contentPath := "/api/manuals/" + manualID + "/items/" + imageID + "/content" + thumbnailPath := "/api/manuals/" + manualID + "/items/" + imageID + "/thumbnail" + original := fixture.request(http.MethodGet, contentPath, nil, nil, nil) + require.Equal(t, http.StatusOK, original.status) + require.Equal(t, pngBytes, original.body) + require.Equal(t, "private, no-store", original.header.Get("Cache-Control")) + require.Equal(t, "nosniff", original.header.Get("X-Content-Type-Options")) + head := fixture.request(http.MethodHead, contentPath, nil, nil, nil) + require.Equal(t, http.StatusOK, head.status) + require.Empty(t, head.body) + partial := fixture.request(http.MethodGet, contentPath, nil, nil, map[string]string{"Range": "bytes=0-7"}) + require.Equal(t, http.StatusPartialContent, partial.status) + require.Equal(t, pngBytes[:8], partial.body) + thumbnail := fixture.request(http.MethodGet, thumbnailPath, nil, nil, nil) + require.Equal(t, http.StatusOK, thumbnail.status) + require.Equal(t, "image/jpeg", thumbnail.header.Get("Content-Type")) + require.Greater(t, len(thumbnail.body), 10) + + private = requireManualSuccess(t, fixture.request(http.MethodPatch, "/api/manuals/"+manualID, fixture.owner, map[string]interface{}{"revision": 8, "access_mode": "owner"}, nil)) + require.Equal(t, int64(9), manualRevision(private)) + requireManualDenied(t, fixture.request(http.MethodGet, contentPath, nil, nil, map[string]string{"Range": "bytes=0-7"})) + require.Equal(t, http.StatusPartialContent, fixture.request(http.MethodGet, contentPath, fixture.owner, nil, map[string]string{"Range": "bytes=0-7"}).status) + public = requireManualSuccess(t, fixture.request(http.MethodPatch, "/api/manuals/"+manualID, fixture.owner, map[string]interface{}{"revision": 9, "access_mode": "public"}, nil)) + require.Equal(t, int64(10), manualRevision(public)) + + textBytes := []byte("第一行\n第二行") + textUpload, textContentType := fixture.multipart(map[string]string{"title": "备注", "client_request_id": "text-file-1"}, "notes.txt", textBytes) + textResult := requireManualSuccess(t, fixture.request(http.MethodPost, "/api/manuals/"+manualID+"/files", fixture.owner, textUpload, map[string]string{"Content-Type": textContentType})) + textItem := textResult["item"].(map[string]interface{}) + require.Equal(t, "text", textItem["kind"]) + require.Equal(t, string(textBytes), textItem["text"]) + require.Empty(t, textItem["content_url"]) + require.Equal(t, int64(11), manualRevision(textResult)) + _, err := os.Stat(filepath.Join(fixture.storageRoot, strconv.FormatInt(fixture.ownerID, 10), manualID, textItem["id"].(string))) + require.ErrorIs(t, err, os.ErrNotExist) + + itemDirectory := filepath.Join(fixture.storageRoot, strconv.FormatInt(fixture.ownerID, 10), manualID, imageID) + require.DirExists(t, itemDirectory) + deleted := requireManualSuccess(t, fixture.request(http.MethodDelete, "/api/manuals/"+manualID+"/items/"+imageID, fixture.owner, map[string]interface{}{"revision": 11}, nil)) + require.Equal(t, int64(12), manualRevision(deleted)) + require.DirExists(t, itemDirectory, "logical deletion must retain original bytes") + requireManualDenied(t, fixture.request(http.MethodGet, contentPath, fixture.owner, nil, nil)) + upload, contentType = fixture.multipart(map[string]string{"title": "铭牌", "client_request_id": "image-1"}, "plate.png", pngBytes) + requireManualDenied(t, fixture.request(http.MethodPost, "/api/manuals/"+manualID+"/files", fixture.owner, upload, map[string]string{"Content-Type": contentType})) + + _, err = fixture.database.Exec("UPDATE user_account SET status = 'banned' WHERE id = ?", fixture.ownerID) + require.NoError(t, err) + requireManualDenied(t, fixture.request(http.MethodGet, "/api/manuals/"+manualID, nil, nil, nil)) +} + +func manualPNG(t *testing.T) []byte { + t.Helper() + imageData := image.NewRGBA(image.Rect(0, 0, 4, 3)) + imageData.Set(0, 0, color.RGBA{R: 255, A: 255}) + imageData.Set(3, 2, color.RGBA{B: 255, A: 255}) + var buffer bytes.Buffer + require.NoError(t, png.Encode(&buffer, imageData)) + return buffer.Bytes() +} + +const manualAccountTestDDL = ` +CREATE TABLE user_account ( + id BIGINT NOT NULL PRIMARY KEY, + username VARCHAR(64) NOT NULL, + username_key VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + display_name VARCHAR(64) NOT NULL DEFAULT '', + avatar_version BIGINT NOT NULL DEFAULT 0, + contact_email VARCHAR(254) NOT NULL DEFAULT '', + contact_mobile VARCHAR(32) NOT NULL DEFAULT '', + password_hash VARCHAR(255) NOT NULL DEFAULT '', + status VARCHAR(16) NOT NULL DEFAULT 'active', + role VARCHAR(16) NOT NULL DEFAULT 'user', + library_enabled BOOLEAN NOT NULL DEFAULT FALSE, + webdav_permission VARCHAR(16) NOT NULL DEFAULT 'none', + auth_version BIGINT NOT NULL DEFAULT 1, + revision BIGINT NOT NULL DEFAULT 1, + must_change_password BOOLEAN NOT NULL DEFAULT FALSE, + password_expires_at DATETIME(6) NULL, + invite_eligible_at DATETIME(6) NOT NULL, + source VARCHAR(16) NOT NULL, + invited_by_user_id BIGINT NULL, + created_by_user_id BIGINT NULL, + imported_at DATETIME(6) NULL, + last_login_at DATETIME(6) NULL, + deleted_at DATETIME(6) NULL, + create_time DATETIME(6) NOT NULL, + update_time DATETIME(6) NOT NULL, + UNIQUE KEY uk_account_username (username_key) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4` + +const manualSessionTestDDL = ` +CREATE TABLE login_token ( + id BIGINT NOT NULL PRIMARY KEY, + user_id BIGINT NOT NULL, + token_digest BINARY(32) NOT NULL, + auth_version BIGINT NOT NULL, + purpose VARCHAR(32) NOT NULL, + is_expired INT NOT NULL DEFAULT 0, + authenticated_at DATETIME(6) NOT NULL, + login_ip VARCHAR(128) NOT NULL, + user_agent VARCHAR(512) NOT NULL, + client_name VARCHAR(128) NOT NULL, + os_name VARCHAR(128) NOT NULL, + device_type VARCHAR(128) NOT NULL, + login_source VARCHAR(64) NOT NULL, + expire_time DATETIME(6) NOT NULL, + create_time DATETIME(6) NOT NULL, + update_time DATETIME(6) NOT NULL, + UNIQUE KEY uk_login_token_digest (token_digest), + KEY idx_login_token_user (user_id, is_expired, expire_time) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4` diff --git a/api/manuals/multipart.go b/api/manuals/multipart.go new file mode 100644 index 0000000..a0e07cb --- /dev/null +++ b/api/manuals/multipart.go @@ -0,0 +1,43 @@ +package manuals + +import ( + "errors" + "mime/multipart" + "net/http" + + "github.com/gin-gonic/gin" + apperrors "github.com/mcoder2014/home_server/errors" +) + +func multipartFields(c *gin.Context) (string, string, *multipart.FileHeader, error) { + form := c.Request.MultipartForm + if form == nil || len(form.File) != 1 || len(form.File["file"]) != 1 { + return "", "", nil, apperrors.ErrInvalid + } + for name := range form.File { + if name != "file" { + return "", "", nil, apperrors.ErrInvalid + } + } + for name, values := range form.Value { + if (name != "title" && name != "client_request_id") || len(values) != 1 { + return "", "", nil, apperrors.ErrInvalid + } + } + if len(form.Value["client_request_id"]) != 1 || len(form.Value["title"]) > 1 { + return "", "", nil, apperrors.ErrInvalid + } + title := "" + if len(form.Value["title"]) == 1 { + title = form.Value["title"][0] + } + return title, form.Value["client_request_id"][0], form.File["file"][0], nil +} + +func classifyMultipartError(err error) error { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return apperrors.ErrTooLarge + } + return apperrors.ErrInvalid +} diff --git a/api/manuals/route.go b/api/manuals/route.go new file mode 100644 index 0000000..3613519 --- /dev/null +++ b/api/manuals/route.go @@ -0,0 +1,48 @@ +package manuals + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/mcoder2014/home_server/api/middleware" + "github.com/mcoder2014/home_server/data" + "github.com/mcoder2014/home_server/domain/service/accounts" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/mcoder2014/home_server/utils/ginfmt" +) + +func InitRouter() error { + write := middleware.RequireIdentity("manuals:write", false) + data.AddRoute(http.MethodGet, "/api/manuals", requireModule, listManuals) + data.AddRoute(http.MethodGet, "/api/manuals/categories", requireModule, listCategories) + data.AddRoute(http.MethodPost, "/api/manuals", requireModule, write, createManual) + data.AddRoute(http.MethodGet, "/api/manuals/:id", requireModule, getManual) + data.AddRoute(http.MethodPatch, "/api/manuals/:id", requireModule, write, updateManual) + data.AddRoute(http.MethodDelete, "/api/manuals/:id", requireModule, write, deleteManual) + data.AddRoute(http.MethodPost, "/api/manuals/:id/items", requireModule, write, addInlineItem) + data.AddRoute(http.MethodPost, "/api/manuals/:id/files", requireModule, write, addFileItem) + data.AddRoute(http.MethodDelete, "/api/manuals/:id/items/:item_id", requireModule, write, deleteItem) + data.AddRouteV2([]string{http.MethodGet, http.MethodHead}, []string{"/api/manuals/:id/items/:item_id/content", "/api/manuals/:id/items/:item_id/thumbnail"}, requireModule, serveItemResource) + return nil +} + +func requireModule(c *gin.Context) { + c.Header("Cache-Control", "no-store") + middleware.EnableReadSnapshot(c) + enabled, err := accounts.ModuleEnabled(ginfmt.RPCContext(c), "manuals") + if err != nil { + ginfmt.Fail(c, apperrors.ErrDependency) + c.Abort() + return + } + if !enabled { + if c.Request.Method == http.MethodGet || c.Request.Method == http.MethodHead { + ginfmt.Fail(c, apperrors.ErrNotFound) + } else { + ginfmt.Fail(c, apperrors.WithMessage(apperrors.ErrForbidden, "功能暂未开放")) + } + c.Abort() + return + } + c.Next() +} diff --git a/api/manuals/route_test.go b/api/manuals/route_test.go new file mode 100644 index 0000000..235d7bb --- /dev/null +++ b/api/manuals/route_test.go @@ -0,0 +1,51 @@ +package manuals + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/mcoder2014/home_server/config" + "github.com/stretchr/testify/require" +) + +func TestManualRoutesSetNoStoreBeforeModuleDecision(t *testing.T) { + before := config.Global() + mode := gin.Mode() + t.Cleanup(func() { + config.SetGlobalConfig(before) + gin.SetMode(mode) + }) + gin.SetMode(gin.TestMode) + for _, enabled := range []bool{false, true} { + config.SetGlobalConfig(config.Config{Manuals: config.ManualsConfig{Enabled: enabled}}) + router := gin.New() + router.Use(requireModule) + router.GET("/api/manuals", func(c *gin.Context) { + c.Status(http.StatusNoContent) + }) + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/api/manuals", nil) + router.ServeHTTP(response, request) + require.Equal(t, "no-store", response.Header().Get("Cache-Control")) + if enabled { + require.Equal(t, http.StatusNoContent, response.Code) + } else { + require.NotEqual(t, http.StatusNoContent, response.Code) + } + } +} + +func TestManualOptionalReaderRejectsExplicitEmptyCredential(t *testing.T) { + context, _ := gin.CreateTestContext(httptest.NewRecorder()) + context.Request = httptest.NewRequest(http.MethodGet, "/api/manuals", nil) + principal, err := optionalReader(context) + require.NoError(t, err) + require.Nil(t, principal) + + context.Request.Header["Authorization"] = []string{""} + principal, err = optionalReader(context) + require.Error(t, err) + require.Nil(t, principal) +} diff --git a/api/manuals/user_quota_concurrency_integration_test.go b/api/manuals/user_quota_concurrency_integration_test.go new file mode 100644 index 0000000..7cea8fa --- /dev/null +++ b/api/manuals/user_quota_concurrency_integration_test.go @@ -0,0 +1,126 @@ +package manuals + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "net/http" + "strconv" + "testing" + "time" + + manualapplication "github.com/mcoder2014/home_server/app/manuals" + "github.com/mcoder2014/home_server/config" + manualservice "github.com/mcoder2014/home_server/domain/service/manuals" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/mcoder2014/home_server/utils" + "github.com/stretchr/testify/require" +) + +func TestManualConcurrentUserByteQuotaAcrossManuals(t *testing.T) { + fixture := newManualHTTPFixture(t) + manualIDs := []string{ + createManualForWriteGuard(t, fixture, "user-byte-manual-1"), + createManualForWriteGuard(t, fixture, "user-byte-manual-2"), + } + conf := config.Global() + conf.Manuals.MaxUserBytes = 5 + config.SetGlobalConfig(conf) + + bodies := make([]*manualBlockedBody, 2) + done := make([]<-chan manualHTTPResponse, 2) + for index, manualID := range manualIDs { + payload, err := json.Marshal(map[string]interface{}{ + "kind": "text", "title": "用户配额竞争", "text": "12345", "client_request_id": "user-byte-item-" + strconv.Itoa(index+1), + }) + require.NoError(t, err) + bodies[index], done[index] = beginBlockedManualRequest(t, fixture, http.MethodPost, "/api/manuals/"+manualID+"/items", fixture.owner, payload, nil) + } + for _, body := range bodies { + body.unblock() + } + responses := []manualHTTPResponse{ + finishBlockedManualRequest(t, bodies[0], done[0]), + finishBlockedManualRequest(t, bodies[1], done[1]), + } + + successes, limited := 0, 0 + for _, response := range responses { + switch response.status { + case http.StatusCreated: + successes++ + case http.StatusTooManyRequests: + limited++ + default: + t.Fatalf("unexpected user-quota response: status=%d code=%d body=%s", response.status, response.code, response.body) + } + } + require.Equal(t, 1, successes) + require.Equal(t, 1, limited) + require.Equal(t, int64(1), countManualRows(t, fixture, "SELECT COUNT(*) FROM manual_items WHERE manual_id IN (?, ?)", manualIDs[0], manualIDs[1])) + var totalBytes, revisionSum int64 + require.NoError(t, fixture.database.QueryRow("SELECT COALESCE(SUM(size_bytes), 0) FROM manual_items WHERE manual_id IN (?, ?)", manualIDs[0], manualIDs[1]).Scan(&totalBytes)) + require.NoError(t, fixture.database.QueryRow("SELECT SUM(revision) FROM manuals WHERE id IN (?, ?)", manualIDs[0], manualIDs[1]).Scan(&revisionSum)) + require.Equal(t, int64(5), totalBytes) + require.Equal(t, int64(3), revisionSum, "one successful append must increment exactly one manual revision") +} + +func TestManualDatabaseUserByteQuotaAcrossSessionsAndApplicationInstances(t *testing.T) { + fixture := newManualHTTPFixture(t) + manualIDs := make([]int64, 2) + for index, requestID := range []string{"database-user-byte-manual-1", "database-user-byte-manual-2"} { + value, err := strconv.ParseInt(createManualForWriteGuard(t, fixture, requestID), 10, 64) + require.NoError(t, err) + manualIDs[index] = value + } + conf := config.Global() + conf.Manuals.MaxUserBytes = 5 + config.SetGlobalConfig(conf) + + secondToken := "second-owner-session-token" + digest := sha256.Sum256([]byte(secondToken)) + now := time.Now().UTC() + _, err := fixture.database.Exec("INSERT INTO login_token (id, user_id, token_digest, auth_version, purpose, is_expired, authenticated_at, login_ip, user_agent, client_name, os_name, device_type, login_source, expire_time, create_time, update_time) VALUES (?, ?, ?, 1, 'user', 0, ?, '127.0.0.1', 'manual-test', 'test', 'test', 'test', 'test', ?, ?, ?)", fixture.ownerID+10_000, fixture.ownerID, digest[:], now, now.Add(time.Hour), now, now) + require.NoError(t, err) + + tokens := []string{fixture.owner.cookie.Value, secondToken} + applications := []*manualapplication.Application{manualapplication.New(), manualapplication.New()} + start := make(chan struct{}) + ready := make(chan struct{}, len(applications)) + results := make(chan error, len(applications)) + for index := range applications { + go func(index int) { + ready <- struct{}{} + <-start + ctx := context.WithValue(context.Background(), utils.CtxKeyLoginToken, tokens[index]) + principal := &utils.Principal{Kind: "user", UserID: fixture.ownerID, AuthVersion: 1, TokenExpiresAt: now.Add(time.Hour)} + _, addErr := applications[index].AddInline(ctx, fixture.ownerID, manualIDs[index], manualservice.InlineItemInput{ + Kind: "text", Title: "跨实例用户配额竞争", Text: "12345", ClientRequestID: "database-user-byte-item-" + strconv.Itoa(index+1), + }, principal) + results <- addErr + }(index) + } + for range applications { + <-ready + } + close(start) + + successes, limited := 0, 0 + for range applications { + switch addErr := <-results; { + case addErr == nil: + successes++ + case errors.Is(addErr, apperrors.ErrRateLimited): + limited++ + default: + t.Fatalf("unexpected cross-instance quota result: %v", addErr) + } + } + require.Equal(t, 1, successes) + require.Equal(t, 1, limited) + require.Equal(t, int64(1), countManualRows(t, fixture, "SELECT COUNT(*) FROM manual_items WHERE manual_id IN (?, ?)", manualIDs[0], manualIDs[1])) + var totalBytes int64 + require.NoError(t, fixture.database.QueryRow("SELECT COALESCE(SUM(size_bytes), 0) FROM manual_items WHERE manual_id IN (?, ?)", manualIDs[0], manualIDs[1]).Scan(&totalBytes)) + require.Equal(t, int64(5), totalBytes) +} diff --git a/api/manuals/write_guards_integration_test.go b/api/manuals/write_guards_integration_test.go new file mode 100644 index 0000000..7ce71a6 --- /dev/null +++ b/api/manuals/write_guards_integration_test.go @@ -0,0 +1,263 @@ +package manuals + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + "time" + + "github.com/mcoder2014/home_server/api/middleware" + "github.com/mcoder2014/home_server/config" + "github.com/mcoder2014/home_server/domain/model" + accountservice "github.com/mcoder2014/home_server/domain/service/accounts" + passportservice "github.com/mcoder2014/home_server/domain/service/passport" + "github.com/mcoder2014/home_server/utils" + "github.com/stretchr/testify/require" +) + +// manualBlockedBody pauses the first handler read. Authentication middleware +// has completed at that point, while no request payload has reached a write. +type manualBlockedBody struct { + reader io.Reader + entered chan struct{} + release chan struct{} + enteredOnce sync.Once + releaseOnce sync.Once +} + +func (body *manualBlockedBody) Read(buffer []byte) (int, error) { + body.enteredOnce.Do(func() { close(body.entered) }) + <-body.release + return body.reader.Read(buffer) +} + +func (body *manualBlockedBody) unblock() { + body.releaseOnce.Do(func() { close(body.release) }) +} + +func beginBlockedManualRequest(t *testing.T, fixture *manualHTTPFixture, method, path string, session *manualHTTPSession, payload []byte, headers map[string]string) (*manualBlockedBody, <-chan manualHTTPResponse) { + t.Helper() + body := &manualBlockedBody{reader: bytes.NewReader(payload), entered: make(chan struct{}), release: make(chan struct{})} + done := make(chan manualHTTPResponse, 1) + go func() { + done <- fixture.request(method, path, session, body, headers) + }() + t.Cleanup(body.unblock) + select { + case <-body.entered: + case response := <-done: + t.Fatalf("manual request ended before authenticated body read: status=%d code=%d", response.status, response.code) + case <-time.After(5 * time.Second): + t.Fatal("manual request did not reach authenticated body read") + } + return body, done +} + +func finishBlockedManualRequest(t *testing.T, body *manualBlockedBody, done <-chan manualHTTPResponse) manualHTTPResponse { + t.Helper() + body.unblock() + select { + case response := <-done: + return response + case <-time.After(10 * time.Second): + t.Fatal("manual request did not finish after body release") + return manualHTTPResponse{} + } +} + +func createManualForWriteGuard(t *testing.T, fixture *manualHTTPFixture, requestID string) string { + t.Helper() + created := requireManualSuccess(t, fixture.request(http.MethodPost, "/api/manuals", fixture.owner, map[string]interface{}{ + "name": "并发测试说明书", "categories": []string{"测试"}, "access_mode": "owner", "client_request_id": requestID, + }, nil)) + return created["id"].(string) +} + +func countManualRows(t *testing.T, fixture *manualHTTPFixture, query string, arguments ...interface{}) int64 { + t.Helper() + var count int64 + require.NoError(t, fixture.database.QueryRow(query, arguments...).Scan(&count)) + return count +} + +func TestManualCreateRejectsSessionRevokedAfterAuthentication(t *testing.T) { + fixture := newManualHTTPFixture(t) + payload, err := json.Marshal(map[string]interface{}{ + "name": "不得创建", "categories": []string{}, "access_mode": "owner", "client_request_id": "revoked-create", + }) + require.NoError(t, err) + body, done := beginBlockedManualRequest(t, fixture, http.MethodPost, "/api/manuals", fixture.owner, payload, nil) + require.NoError(t, accountservice.Logout(context.Background(), fixture.owner.cookie.Value)) + + response := finishBlockedManualRequest(t, body, done) + require.Equal(t, http.StatusUnauthorized, response.status) + require.Zero(t, countManualRows(t, fixture, "SELECT COUNT(*) FROM manuals")) +} + +func TestManualDuplicateCreateDoesNotHideSessionRevocation(t *testing.T) { + fixture := newManualHTTPFixture(t) + payload, err := json.Marshal(map[string]interface{}{ + "name": "幂等创建", "categories": []string{"测试"}, "access_mode": "owner", "client_request_id": "duplicate-create", + }) + require.NoError(t, err) + created := requireManualSuccess(t, fixture.request(http.MethodPost, "/api/manuals", fixture.owner, payload, nil)) + require.Equal(t, int64(1), manualRevision(created)) + body, done := beginBlockedManualRequest(t, fixture, http.MethodPost, "/api/manuals", fixture.owner, payload, nil) + require.NoError(t, accountservice.Logout(context.Background(), fixture.owner.cookie.Value)) + + response := finishBlockedManualRequest(t, body, done) + require.Equal(t, http.StatusUnauthorized, response.status) + require.Equal(t, int64(1), countManualRows(t, fixture, "SELECT COUNT(*) FROM manuals")) +} + +func TestManualLegacyCreateRejectsSessionRevokedAfterAuthentication(t *testing.T) { + fixture := newManualHTTPFixture(t) + identity, err := json.Marshal([]*model.UserIdentity{{ID: fixture.ownerID, UserName: "legacy-owner", AuthVersion: 1}}) + require.NoError(t, err) + require.NoError(t, passportservice.GetMockData().LoadConf(string(identity))) + t.Cleanup(func() { _ = passportservice.GetMockData().LoadConf("[]") }) + _, err = fixture.database.Exec("ALTER TABLE login_token ADD COLUMN token VARCHAR(512) NOT NULL DEFAULT ''") + require.NoError(t, err) + legacyToken := "legacy-manual-session-token" + _, err = fixture.database.Exec("UPDATE login_token SET token = ? WHERE id = ?", legacyToken, fixture.ownerID) + require.NoError(t, err) + conf := config.Global() + conf.IdentitySource = "config" + config.SetGlobalConfig(conf) + session := &manualHTTPSession{ + cookie: &http.Cookie{Name: utils.SessionCookieName, Value: legacyToken, Path: "/", Secure: true, HttpOnly: true}, + csrf: middleware.CSRFToken(legacyToken), + } + payload, err := json.Marshal(map[string]interface{}{ + "name": "旧身份不得创建", "categories": []string{}, "access_mode": "owner", "client_request_id": "legacy-revoked-create", + }) + require.NoError(t, err) + body, done := beginBlockedManualRequest(t, fixture, http.MethodPost, "/api/manuals", session, payload, nil) + require.NoError(t, passportservice.DeleteToken(context.Background(), legacyToken)) + + response := finishBlockedManualRequest(t, body, done) + require.Equal(t, http.StatusUnauthorized, response.status) + require.Zero(t, countManualRows(t, fixture, "SELECT COUNT(*) FROM manuals")) +} + +func TestManualFileRejectsSessionRevokedDuringUpload(t *testing.T) { + fixture := newManualHTTPFixture(t) + manualID := createManualForWriteGuard(t, fixture, "file-revoke-manual") + upload, contentType := fixture.multipart(map[string]string{"title": "不得写入", "client_request_id": "revoked-file"}, "plate.png", manualPNG(t)) + body, done := beginBlockedManualRequest(t, fixture, http.MethodPost, "/api/manuals/"+manualID+"/files", fixture.owner, upload.Bytes(), map[string]string{"Content-Type": contentType}) + require.NoError(t, accountservice.Logout(context.Background(), fixture.owner.cookie.Value)) + + response := finishBlockedManualRequest(t, body, done) + require.Equal(t, http.StatusUnauthorized, response.status) + require.Zero(t, countManualRows(t, fixture, "SELECT COUNT(*) FROM manual_items WHERE manual_id = ?", manualID)) + var revision int64 + require.NoError(t, fixture.database.QueryRow("SELECT revision FROM manuals WHERE id = ?", manualID).Scan(&revision)) + require.Equal(t, int64(1), revision) + directory := filepath.Join(fixture.storageRoot, strconv.FormatInt(fixture.ownerID, 10), manualID) + entries, err := os.ReadDir(directory) + if !os.IsNotExist(err) { + require.NoError(t, err) + require.Empty(t, entries, "rejected upload left a stored item directory") + } +} + +func TestManualDuplicateFileDoesNotHideSessionRevocation(t *testing.T) { + fixture := newManualHTTPFixture(t) + manualID := createManualForWriteGuard(t, fixture, "duplicate-revoke-manual") + pngData := manualPNG(t) + first, contentType := fixture.multipart(map[string]string{"title": "铭牌", "client_request_id": "duplicate-file"}, "plate.png", pngData) + created := requireManualSuccess(t, fixture.request(http.MethodPost, "/api/manuals/"+manualID+"/files", fixture.owner, first, map[string]string{"Content-Type": contentType})) + require.Equal(t, int64(2), manualRevision(created)) + + replay, replayType := fixture.multipart(map[string]string{"title": "铭牌", "client_request_id": "duplicate-file"}, "plate.png", pngData) + body, done := beginBlockedManualRequest(t, fixture, http.MethodPost, "/api/manuals/"+manualID+"/files", fixture.owner, replay.Bytes(), map[string]string{"Content-Type": replayType}) + require.NoError(t, accountservice.Logout(context.Background(), fixture.owner.cookie.Value)) + + response := finishBlockedManualRequest(t, body, done) + require.Equal(t, http.StatusUnauthorized, response.status) + require.Equal(t, int64(1), countManualRows(t, fixture, "SELECT COUNT(*) FROM manual_items WHERE manual_id = ?", manualID)) + var revision int64 + require.NoError(t, fixture.database.QueryRow("SELECT revision FROM manuals WHERE id = ?", manualID).Scan(&revision)) + require.Equal(t, int64(2), revision) + directory := filepath.Join(fixture.storageRoot, strconv.FormatInt(fixture.ownerID, 10), manualID) + entries, err := os.ReadDir(directory) + require.NoError(t, err) + require.Len(t, entries, 1, "duplicate retry left a second stored directory") +} + +func TestManualDuplicateFileDoesNotHideConcurrentManualDeletion(t *testing.T) { + fixture := newManualHTTPFixture(t) + manualID := createManualForWriteGuard(t, fixture, "duplicate-delete-manual") + pngData := manualPNG(t) + first, contentType := fixture.multipart(map[string]string{"title": "铭牌", "client_request_id": "delete-race-file"}, "plate.png", pngData) + created := requireManualSuccess(t, fixture.request(http.MethodPost, "/api/manuals/"+manualID+"/files", fixture.owner, first, map[string]string{"Content-Type": contentType})) + require.Equal(t, int64(2), manualRevision(created)) + + replay, replayType := fixture.multipart(map[string]string{"title": "铭牌", "client_request_id": "delete-race-file"}, "plate.png", pngData) + body, done := beginBlockedManualRequest(t, fixture, http.MethodPost, "/api/manuals/"+manualID+"/files", fixture.owner, replay.Bytes(), map[string]string{"Content-Type": replayType}) + deleted := requireManualSuccess(t, fixture.request(http.MethodDelete, "/api/manuals/"+manualID, fixture.owner, map[string]interface{}{"revision": 2}, nil)) + require.Equal(t, int64(3), manualRevision(deleted)) + + response := finishBlockedManualRequest(t, body, done) + require.Equal(t, http.StatusNotFound, response.status) + var status model.ManualStatus + var revision int64 + require.NoError(t, fixture.database.QueryRow("SELECT status, revision FROM manuals WHERE id = ?", manualID).Scan(&status, &revision)) + require.Equal(t, model.ManualStatusDeleted, status) + require.Equal(t, int64(3), revision) + require.Equal(t, int64(1), countManualRows(t, fixture, "SELECT COUNT(*) FROM manual_items WHERE manual_id = ?", manualID)) + directory := filepath.Join(fixture.storageRoot, strconv.FormatInt(fixture.ownerID, 10), manualID) + entries, err := os.ReadDir(directory) + require.NoError(t, err) + require.Len(t, entries, 1, "failed replay left a second stored directory") +} + +func TestManualConcurrentItemQuotaSerializesRevision(t *testing.T) { + fixture := newManualHTTPFixture(t) + manualID := createManualForWriteGuard(t, fixture, "concurrent-quota-manual") + conf := config.Global() + conf.Manuals.MaxItemsPerManual = 1 + config.SetGlobalConfig(conf) + + payloads := make([][]byte, 2) + for index := range payloads { + payload, err := json.Marshal(map[string]interface{}{ + "kind": "text", "title": "竞争写入", "text": "正文", "client_request_id": "concurrent-item-" + strconv.Itoa(index+1), + }) + require.NoError(t, err) + payloads[index] = payload + } + firstBody, firstDone := beginBlockedManualRequest(t, fixture, http.MethodPost, "/api/manuals/"+manualID+"/items", fixture.owner, payloads[0], nil) + secondBody, secondDone := beginBlockedManualRequest(t, fixture, http.MethodPost, "/api/manuals/"+manualID+"/items", fixture.owner, payloads[1], nil) + firstBody.unblock() + secondBody.unblock() + responses := []manualHTTPResponse{ + finishBlockedManualRequest(t, firstBody, firstDone), + finishBlockedManualRequest(t, secondBody, secondDone), + } + + successes, limited := 0, 0 + for _, response := range responses { + switch response.status { + case http.StatusCreated: + successes++ + case http.StatusTooManyRequests: + limited++ + default: + t.Fatalf("unexpected concurrent response: status=%d code=%d body=%s", response.status, response.code, response.body) + } + } + require.Equal(t, 1, successes) + require.Equal(t, 1, limited) + require.Equal(t, int64(1), countManualRows(t, fixture, "SELECT COUNT(*) FROM manual_items WHERE manual_id = ?", manualID)) + var revision int64 + require.NoError(t, fixture.database.QueryRow("SELECT revision FROM manuals WHERE id = ?", manualID).Scan(&revision)) + require.Equal(t, int64(2), revision) +} diff --git a/api/middleware/account.go b/api/middleware/account.go index f5b7cf6..682ae5d 100644 --- a/api/middleware/account.go +++ b/api/middleware/account.go @@ -202,6 +202,8 @@ func AuthorizeCapability(ctx context.Context, principal *utils.Principal, scope module = "webdav" } else if strings.HasPrefix(scope, "web-projects:") || strings.HasPrefix(scope, "web-comments:") { module = "web_projects" + } else if strings.HasPrefix(scope, "manuals:") { + module = "manuals" } if module == "" { return nil diff --git a/app/manuals/application.go b/app/manuals/application.go new file mode 100644 index 0000000..84de736 --- /dev/null +++ b/app/manuals/application.go @@ -0,0 +1,58 @@ +package manuals + +import ( + "context" + "errors" + "sync" + + "github.com/go-sql-driver/mysql" + "github.com/mcoder2014/home_server/domain/db" + manualservice "github.com/mcoder2014/home_server/domain/service/manuals" + apperrors "github.com/mcoder2014/home_server/errors" + "gorm.io/gorm" +) + +type Application struct { + ownerLocks [64]sync.Mutex + uploads uploadReservations + diskFree func(string) (uint64, error) +} + +type uploadReservations struct { + sync.Mutex + reservedBytes uint64 +} + +var Default = New() + +func New() *Application { + return &Application{diskFree: manualservice.DiskFreeBytes} +} + +func database(ctx context.Context) (*gorm.DB, error) { + if db.MasterDB() == nil { + return nil, apperrors.ErrDependency + } + return db.MasterDB().WithContext(ctx), nil +} + +func (application *Application) lockOwner(ownerID int64) func() { + index := uint64(ownerID) % uint64(len(application.ownerLocks)) + application.ownerLocks[index].Lock() + return application.ownerLocks[index].Unlock +} + +func persistenceError(err error) error { + if err == nil { + return nil + } + var apiError *apperrors.APIError + if errors.As(err, &apiError) { + return err + } + var mysqlError *mysql.MySQLError + if errors.As(err, &mysqlError) && mysqlError.Number == 1062 { + return apperrors.ErrConflict + } + return apperrors.ErrDependency +} diff --git a/app/manuals/create.go b/app/manuals/create.go new file mode 100644 index 0000000..48c425a --- /dev/null +++ b/app/manuals/create.go @@ -0,0 +1,106 @@ +package manuals + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "time" + + "github.com/mcoder2014/home_server/config" + "github.com/mcoder2014/home_server/domain/dal" + "github.com/mcoder2014/home_server/domain/model" + service "github.com/mcoder2014/home_server/domain/service/manuals" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/mcoder2014/home_server/utils" + "gorm.io/gorm" +) + +func (application *Application) Create(ctx context.Context, ownerID int64, input service.CreateManualInput, principal *utils.Principal) (*service.ManualView, error) { + name, description, access, err := service.ValidateManualFields(input.Name, input.Description, input.AccessMode) + if err != nil || !service.ValidRequestID(input.ClientRequestID) { + return nil, apperrors.ErrInvalid + } + categories, err := service.ValidateCategories(input.Categories) + if err != nil { + return nil, err + } + hash := inputHash(struct { + Name string + Description string + Categories []string + AccessMode string + }{name, description, categories, access.String()}) + database, err := database(ctx) + if err != nil { + return nil, err + } + manualID, err := newID() + if err != nil { + return nil, apperrors.ErrDependency + } + now := time.Now() + manual := &model.Manual{ID: manualID, OwnerUserID: ownerID, Name: name, Description: description, AccessMode: access, Status: model.ManualStatusDraft, Revision: 1, ClientRequestID: input.ClientRequestID, RequestHash: hash, CreateTime: now, UpdateTime: now} + replayed := false + unlock := application.lockOwner(ownerID) + defer unlock() + err = database.Transaction(func(tx *gorm.DB) error { + if err := requireWritePolicy(ctx, tx, ownerID, principal); err != nil { + return err + } + existing, err := dal.FindManualByRequestID(tx, ownerID, input.ClientRequestID, true) + if err != nil { + return err + } + if existing != nil { + manual = existing + if existing.RequestHash != hash || existing.Status == model.ManualStatusDeleted { + return apperrors.ErrConflict + } + replayed = true + return nil + } + count, err := dal.CountUserManuals(tx, ownerID) + if err != nil { + return err + } + if count >= int64(config.Global().Manuals.MaxManualsPerUser) { + return apperrors.ErrRateLimited + } + if err := checkWriteExpiry(principal); err != nil { + return err + } + if err := dal.InsertManual(tx, manual); err != nil { + return err + } + return dal.InsertManualCategories(tx, manual.ID, categories) + }) + if err != nil { + return nil, persistenceError(err) + } + if replayed { + return application.Get(ctx, manual.ID, ownerID, principal) + } + return manualView(manual, categories, 0, nil, []*model.ManualItem{}, principal), nil +} + +func inputHash(value interface{}) string { + encoded, _ := json.Marshal(value) + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]) +} + +func newID() (int64, error) { + for { + var data [8]byte + if _, err := rand.Read(data[:]); err != nil { + return 0, err + } + id := int64(binary.BigEndian.Uint64(data[:]) & uint64(^uint64(0)>>1)) + if id > 0 { + return id, nil + } + } +} diff --git a/app/manuals/files.go b/app/manuals/files.go new file mode 100644 index 0000000..57981b3 --- /dev/null +++ b/app/manuals/files.go @@ -0,0 +1,86 @@ +package manuals + +import ( + "context" + "errors" + "io" + "time" + + "github.com/mcoder2014/home_server/config" + "github.com/mcoder2014/home_server/domain/dal" + "github.com/mcoder2014/home_server/domain/model" + service "github.com/mcoder2014/home_server/domain/service/manuals" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/mcoder2014/home_server/utils" + "gorm.io/gorm" +) + +// AddReservedFile consumes a file body after the API layer has validated the +// manual owner and reserved worst-case disk space for this upload. +func (application *Application) AddReservedFile(ctx context.Context, ownerID, manualID int64, title, requestID, originalName string, source io.Reader, principal *utils.Principal) (*service.ItemMutationResult, error) { + title, err := service.ValidateFileMetadata(title, requestID) + if err != nil { + return nil, err + } + conf := config.Global().Manuals + itemID, err := newID() + if err != nil { + return nil, apperrors.ErrDependency + } + storedFile, err := service.StoreFile(&conf, ownerID, manualID, itemID, originalName, source) + if err != nil { + return nil, err + } + hash := inputHash(struct{ Kind, Title, OriginalName, SHA256 string }{storedFile.Kind.String(), title, storedFile.OriginalName, storedFile.SHA256}) + item := &model.ManualItem{ID: itemID, ManualID: manualID, Kind: storedFile.Kind, Title: title, Text: storedFile.Text, StorageKey: stringPointer(storedFile.StorageKey), ThumbnailKey: stringPointer(storedFile.ThumbnailKey), ContentType: storedFile.ContentType, OriginalName: storedFile.OriginalName, SizeBytes: storedFile.SizeBytes, SHA256: storedFile.SHA256, PreviewStatus: storedFile.PreviewStatus, ClientRequestID: requestID, RequestHash: hash, CreateTime: time.Now()} + database, err := database(ctx) + if err != nil { + if storedFile.StorageKey != "" { + _ = service.RemoveStoredFile(&conf, ownerID, manualID, itemID) + } + return nil, err + } + result, revision, appendErr := application.appendItem(ctx, database, ownerID, item, principal) + if appendErr == nil { + if result.ID != itemID && storedFile.StorageKey != "" { + _ = service.RemoveStoredFile(&conf, ownerID, manualID, itemID) + } + return &service.ItemMutationResult{Item: itemView(result), Revision: revision}, nil + } + return application.resolveFileCommit(ctx, database, &conf, ownerID, manualID, item, appendErr) +} + +func (application *Application) resolveFileCommit(ctx context.Context, database *gorm.DB, conf *config.ManualsConfig, ownerID, manualID int64, item *model.ManualItem, appendErr error) (*service.ItemMutationResult, error) { + if !errors.Is(appendErr, apperrors.ErrDependency) { + if item.StorageKey != nil { + _ = service.RemoveStoredFile(conf, ownerID, manualID, item.ID) + } + return nil, appendErr + } + existing, queryErr := dal.FindManualItemByRequestID(database.WithContext(ctx), manualID, item.ClientRequestID, false) + if queryErr != nil { + // The transaction outcome cannot be proven. Keep the generated directory + // so a committed row can never be left pointing at deleted bytes. + return nil, apperrors.ErrDependency + } + if existing == nil { + if item.StorageKey != nil { + _ = service.RemoveStoredFile(conf, ownerID, manualID, item.ID) + } + return nil, appendErr + } + if existing.ID != item.ID { + if item.StorageKey != nil { + _ = service.RemoveStoredFile(conf, ownerID, manualID, item.ID) + } + return nil, appendErr + } + if existing.DeletedAt != nil || existing.RequestHash != item.RequestHash { + return nil, apperrors.ErrConflict + } + revision, err := application.currentRevision(ctx, manualID) + if err != nil { + return nil, err + } + return &service.ItemMutationResult{Item: itemView(existing), Revision: revision}, nil +} diff --git a/app/manuals/items.go b/app/manuals/items.go new file mode 100644 index 0000000..9c2b47d --- /dev/null +++ b/app/manuals/items.go @@ -0,0 +1,195 @@ +package manuals + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "math" + "time" + + "github.com/mcoder2014/home_server/config" + "github.com/mcoder2014/home_server/domain/dal" + "github.com/mcoder2014/home_server/domain/model" + service "github.com/mcoder2014/home_server/domain/service/manuals" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/mcoder2014/home_server/utils" + "gorm.io/gorm" +) + +func (application *Application) AddInline(ctx context.Context, ownerID, manualID int64, input service.InlineItemInput, principal *utils.Principal) (*service.ItemMutationResult, error) { + validated, err := service.ValidateInlineItem(input) + if err != nil { + return nil, err + } + itemID, err := newID() + if err != nil { + return nil, apperrors.ErrDependency + } + body := validated.Text + if validated.Kind == model.ManualItemURL { + body = validated.URL + } + digest := sha256.Sum256([]byte(body)) + hash := inputHash(struct{ Kind, Title, Text, URL string }{validated.Kind.String(), validated.Title, validated.Text, validated.URL}) + item := &model.ManualItem{ID: itemID, ManualID: manualID, Kind: validated.Kind, Title: validated.Title, Text: validated.Text, URL: validated.URL, SizeBytes: int64(len(validated.Text)), SHA256: hex.EncodeToString(digest[:]), PreviewStatus: model.ManualPreviewNone, ClientRequestID: validated.ClientRequestID, RequestHash: hash, CreateTime: time.Now()} + database, err := database(ctx) + if err != nil { + return nil, err + } + stored, revision, err := application.appendItem(ctx, database, ownerID, item, principal) + if err != nil { + return nil, err + } + return &service.ItemMutationResult{Item: itemView(stored), Revision: revision}, nil +} + +func (application *Application) appendItem(ctx context.Context, database *gorm.DB, ownerID int64, item *model.ManualItem, principal *utils.Principal) (*model.ManualItem, int64, error) { + var stored *model.ManualItem + var revision int64 + unlock := application.lockOwner(ownerID) + defer unlock() + err := database.Transaction(func(tx *gorm.DB) error { + if err := requireWritePolicy(ctx, tx, ownerID, principal); err != nil { + return err + } + manual, err := dal.FindManual(tx, item.ManualID, true) + if err != nil { + return err + } + if manual == nil || manual.OwnerUserID != ownerID || manual.Status == model.ManualStatusDeleted { + return apperrors.ErrNotFound + } + existing, err := dal.FindManualItemByRequestID(tx, item.ManualID, item.ClientRequestID, true) + if err != nil { + return err + } + if existing != nil { + if existing.DeletedAt != nil || existing.RequestHash != item.RequestHash { + return apperrors.ErrConflict + } + stored, revision = existing, manual.Revision + return nil + } + if err := checkItemQuota(tx, ownerID, item.ManualID, item.SizeBytes); err != nil { + return err + } + count, err := dal.CountActiveManualItems(tx, item.ManualID) + if err != nil { + return err + } + item.Position = int(count) + 1 + if err := checkWriteExpiry(principal); err != nil { + return err + } + if err := dal.InsertManualItem(tx, item); err != nil { + return err + } + updated, err := dal.UpdateManualFieldsCAS(tx, ownerID, item.ManualID, manual.Revision, map[string]interface{}{"revision": gorm.Expr("revision + 1"), "update_time": time.Now()}) + if err != nil { + return err + } + if !updated { + return apperrors.ErrConflict + } + stored, revision = item, manual.Revision+1 + return nil + }) + if err != nil { + return nil, 0, persistenceError(err) + } + return stored, revision, nil +} + +func checkItemQuota(tx *gorm.DB, ownerID, manualID, addition int64) error { + limits := config.Global().Manuals + count, err := dal.CountActiveManualItems(tx, manualID) + if err != nil { + return err + } + if count >= int64(limits.MaxItemsPerManual) { + return apperrors.ErrRateLimited + } + manualBytes, err := dal.SumManualItemBytes(tx, manualID) + if err != nil { + return err + } + userBytes, err := dal.SumUserManualItemBytes(tx, ownerID) + if err != nil { + return err + } + if addition < 0 || manualBytes > math.MaxInt64-addition || userBytes > math.MaxInt64-addition { + return apperrors.ErrDependency + } + if manualBytes+addition > limits.MaxManualBytes || userBytes+addition > limits.MaxUserBytes { + return apperrors.ErrRateLimited + } + return nil +} + +func (application *Application) CheckUploadOwner(ctx context.Context, ownerID, manualID int64) error { + database, err := database(ctx) + if err != nil { + return err + } + manual, err := dal.FindManual(database, manualID, false) + if err != nil { + return apperrors.ErrDependency + } + if manual == nil || manual.OwnerUserID != ownerID || manual.Status == model.ManualStatusDeleted { + return apperrors.ErrNotFound + } + return nil +} + +func (application *Application) AcquireUpload(conf *config.ManualsConfig) (func(), error) { + if conf == nil || !conf.Enabled || conf.MaxFileBytes <= 0 || conf.PDFPreviewOutputLimitBytes <= 0 { + return nil, apperrors.ErrDependency + } + reservation := uint64(conf.MaxFileBytes) + preview := uint64(conf.PDFPreviewOutputLimitBytes) + if reservation > math.MaxUint64-preview { + return nil, apperrors.ErrDependency + } + reservation += preview + application.uploads.Lock() + defer application.uploads.Unlock() + free, err := application.diskFree(conf.StorageRoot) + if err != nil { + return nil, apperrors.ErrDependency + } + minimum := uint64(conf.MinFreeDiskBytes) + if minimum > free || application.uploads.reservedBytes > free-minimum || reservation > free-minimum-application.uploads.reservedBytes { + return nil, apperrors.ErrRateLimited + } + application.uploads.reservedBytes += reservation + released := false + return func() { + application.uploads.Lock() + defer application.uploads.Unlock() + if released { + return + } + released = true + application.uploads.reservedBytes -= reservation + }, nil +} + +func stringPointer(value string) *string { + if value == "" { + return nil + } + copy := value + return © +} + +func (application *Application) currentRevision(ctx context.Context, manualID int64) (int64, error) { + database, err := database(ctx) + if err != nil { + return 0, err + } + manual, err := dal.FindManual(database, manualID, false) + if err != nil || manual == nil { + return 0, apperrors.ErrDependency + } + return manual.Revision, nil +} diff --git a/app/manuals/items_test.go b/app/manuals/items_test.go new file mode 100644 index 0000000..ef758fa --- /dev/null +++ b/app/manuals/items_test.go @@ -0,0 +1,28 @@ +package manuals + +import ( + "testing" + + "github.com/mcoder2014/home_server/config" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/stretchr/testify/require" +) + +func TestManualUploadReservationPreventsConcurrentDiskOvercommit(t *testing.T) { + application := New() + application.diskFree = func(string) (uint64, error) { + return 200, nil + } + conf := config.ManualsConfig{Enabled: true, StorageRoot: "/unused", MaxFileBytes: 100, MinFreeDiskBytes: 10, PDFPreviewOutputLimitBytes: 50} + + release, err := application.AcquireUpload(&conf) + require.NoError(t, err) + _, err = application.AcquireUpload(&conf) + require.ErrorIs(t, err, apperrors.ErrRateLimited) + release() + release() + + secondRelease, err := application.AcquireUpload(&conf) + require.NoError(t, err) + secondRelease() +} diff --git a/app/manuals/policy.go b/app/manuals/policy.go new file mode 100644 index 0000000..2c11730 --- /dev/null +++ b/app/manuals/policy.go @@ -0,0 +1,73 @@ +package manuals + +import ( + "context" + "time" + + "github.com/mcoder2014/home_server/config" + "github.com/mcoder2014/home_server/domain/service/accounts" + "github.com/mcoder2014/home_server/domain/service/passport" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/mcoder2014/home_server/utils" + "gorm.io/gorm" +) + +// requireWritePolicy revalidates the authenticated snapshot after any upload +// work and before taking resource locks or committing side effects. +func requireWritePolicy(ctx context.Context, tx *gorm.DB, ownerID int64, principal *utils.Principal) error { + if !config.Global().Manuals.Enabled { + return apperrors.WithMessage(apperrors.ErrForbidden, "功能暂未开放") + } + if principal == nil || principal.UserID != ownerID || (principal.Kind != "user" && principal.Kind != "application") { + return apperrors.ErrUnauthorized + } + if err := checkWriteExpiry(principal); err != nil { + return err + } + if principal.Kind == "application" { + enabled, err := accounts.EnabledTx(tx, "auth", "applications_enabled", true) + if err != nil { + return err + } + if !enabled { + return apperrors.ErrForbidden + } + } + if !accounts.DatabaseMode() { + if principal.Kind == "user" { + return passport.RequireConfigSessionTx(ctx, tx, utils.GetTokenFromCtx(ctx), ownerID, principal.AuthVersion) + } + identity, err := passport.GetByID(ctx, ownerID) + if err != nil { + return apperrors.ErrDependency + } + if identity == nil { + return apperrors.ErrUnauthorized + } + return accounts.RequireApplicationSnapshotTx(tx, principal, "manuals:write") + } + if principal.AuthVersion <= 0 { + return apperrors.ErrUnauthorized + } + if principal.Kind == "user" { + user, err := accounts.RequireUserSessionTx(tx, utils.GetTokenFromCtx(ctx), ownerID) + if err != nil { + return err + } + if user.AuthVersion != principal.AuthVersion { + return apperrors.ErrUnauthorized + } + return nil + } + if _, err := accounts.RequireUserTx(tx, ownerID, principal.AuthVersion, false); err != nil { + return err + } + return accounts.RequireApplicationSnapshotTx(tx, principal, "manuals:write") +} + +func checkWriteExpiry(principal *utils.Principal) error { + if principal == nil || principal.TokenExpiresAt.IsZero() || !principal.TokenExpiresAt.After(time.Now()) { + return apperrors.ErrUnauthorized + } + return nil +} diff --git a/app/manuals/policy_test.go b/app/manuals/policy_test.go new file mode 100644 index 0000000..7f504f9 --- /dev/null +++ b/app/manuals/policy_test.go @@ -0,0 +1,36 @@ +package manuals + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/mcoder2014/home_server/config" + "github.com/mcoder2014/home_server/domain/model" + "github.com/mcoder2014/home_server/domain/service/passport" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/mcoder2014/home_server/utils" + "github.com/stretchr/testify/require" +) + +func TestManualWritePolicyLegacyIdentityRejectsMissingTokenAndRechecksExpiry(t *testing.T) { + before := config.Global() + t.Cleanup(func() { + config.SetGlobalConfig(before) + _ = passport.GetMockData().LoadConf("[]") + }) + config.SetGlobalConfig(config.Config{IdentitySource: "config", Manuals: config.ManualsConfig{Enabled: true}}) + encoded, err := json.Marshal([]*model.UserIdentity{{ID: 101, UserName: "owner"}}) + require.NoError(t, err) + require.NoError(t, passport.GetMockData().LoadConf(string(encoded))) + principal := &utils.Principal{Kind: "user", UserID: 101, TokenExpiresAt: time.Now().Add(time.Minute)} + + require.ErrorIs(t, requireWritePolicy(context.Background(), nil, 101, principal), apperrors.ErrUnauthorized) + wrongTokenType := context.WithValue(context.Background(), utils.CtxKeyLoginToken, 101) + require.ErrorIs(t, requireWritePolicy(wrongTokenType, nil, 101, principal), apperrors.ErrUnauthorized) + principal.TokenExpiresAt = time.Now().Add(-time.Second) + require.ErrorIs(t, requireWritePolicy(context.Background(), nil, 101, principal), apperrors.ErrUnauthorized) + principal.TokenExpiresAt = time.Now().Add(time.Minute) + require.ErrorIs(t, requireWritePolicy(context.Background(), nil, 202, principal), apperrors.ErrUnauthorized) +} diff --git a/app/manuals/read.go b/app/manuals/read.go new file mode 100644 index 0000000..2a9d6fe --- /dev/null +++ b/app/manuals/read.go @@ -0,0 +1,283 @@ +package manuals + +import ( + "context" + "net/url" + "strconv" + "strings" + "unicode/utf8" + + "github.com/mcoder2014/home_server/domain/dal" + "github.com/mcoder2014/home_server/domain/model" + "github.com/mcoder2014/home_server/domain/service/accounts" + service "github.com/mcoder2014/home_server/domain/service/manuals" + "github.com/mcoder2014/home_server/domain/service/passport" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/mcoder2014/home_server/utils" + "gorm.io/gorm" +) + +func (application *Application) List(ctx context.Context, viewerID int64, mine bool, cursor int64, limit int, queryText, category string, categorySet bool, principal *utils.Principal) (*service.ManualPage, error) { + if limit == 0 { + limit = 20 + } + queryText, category = strings.TrimSpace(queryText), strings.TrimSpace(category) + if limit < 1 || limit > 100 || cursor < 0 || (mine && viewerID <= 0) || !utf8.ValidString(queryText) || utf8.RuneCountInString(queryText) > service.MaxNameRunes || !utf8.ValidString(category) || utf8.RuneCountInString(category) > service.MaxCategoryRunes { + return nil, apperrors.ErrInvalid + } + database, err := database(ctx) + if err != nil { + return nil, err + } + filter := dal.ManualListFilter{ViewerUserID: viewerID, Mine: mine, Cursor: cursor, Limit: limit + 1, Query: queryText, Category: category, CategorySet: categorySet} + if accounts.DatabaseMode() { + filter.RequireActiveOwner = true + } else { + filter.ActiveOwnerIDs, err = activeOwnerIDs(ctx) + if err != nil { + return nil, err + } + } + manualRows, err := dal.ListManuals(database, filter) + if err != nil { + return nil, apperrors.ErrDependency + } + page := &service.ManualPage{Items: []*service.ManualView{}} + if len(manualRows) > limit { + page.HasMore = true + manualRows = manualRows[:limit] + } + counts, covers, categories, err := listDecorations(database, manualRows) + if err != nil { + return nil, err + } + for _, manual := range manualRows { + page.Items = append(page.Items, manualView(manual, categories[manual.ID], counts[manual.ID], covers[manual.ID], nil, principal)) + } + if page.HasMore { + page.NextCursor = strconv.FormatInt(manualRows[len(manualRows)-1].ID, 10) + } + return page, nil +} + +func (application *Application) Categories(ctx context.Context, viewerID int64, mine bool, cursor string, limit int) (*service.CategoryPage, error) { + if limit == 0 { + limit = 200 + } + if mine && viewerID <= 0 { + return nil, apperrors.ErrUnauthorized + } + if limit < 1 || limit > 1000 || !utf8.ValidString(cursor) || utf8.RuneCountInString(cursor) > service.MaxCategoryRunes || strings.TrimSpace(cursor) != cursor { + return nil, apperrors.ErrInvalid + } + database, err := database(ctx) + if err != nil { + return nil, err + } + filter := dal.ManualListFilter{ViewerUserID: viewerID, Mine: mine} + if accounts.DatabaseMode() { + filter.RequireActiveOwner = true + } else if filter.ActiveOwnerIDs, err = activeOwnerIDs(ctx); err != nil { + return nil, err + } + items, err := dal.ListManualCategoriesAfter(database, filter, cursor, limit+1) + if err != nil { + return nil, apperrors.ErrDependency + } + if items == nil { + items = []string{} + } + page := &service.CategoryPage{Items: []string{}} + if len(items) > limit { + page.HasMore = true + items = items[:limit] + } + page.Items = items + if page.HasMore { + page.NextCursor = items[len(items)-1] + } + return page, nil +} + +func (application *Application) Get(ctx context.Context, manualID, viewerID int64, principal *utils.Principal) (*service.ManualView, error) { + database, err := database(ctx) + if err != nil { + return nil, err + } + manual, err := dal.FindManual(database, manualID, false) + if err != nil { + return nil, apperrors.ErrDependency + } + if manual == nil { + return nil, apperrors.ErrNotFound + } + active, err := ownerActive(ctx, manual.OwnerUserID) + if err != nil { + return nil, err + } + if !active || !service.CanReadManual(manual.AccessMode, manual.Status, manual.OwnerUserID, viewerID) { + return nil, apperrors.ErrNotFound + } + items, err := dal.ListManualItems(database, manual.ID, false) + if err != nil { + return nil, apperrors.ErrDependency + } + categoryRows, err := dal.ListManualCategoriesByManualIDs(database, []int64{manual.ID}) + if err != nil { + return nil, apperrors.ErrDependency + } + cover := chooseDetailCover(manual.CoverItemID, items) + return manualView(manual, categoriesByManualID(categoryRows)[manual.ID], len(items), cover, items, principal), nil +} + +func activeOwnerIDs(ctx context.Context) ([]int64, error) { + users, err := passport.ListUsersWithError(ctx) + if err != nil { + return nil, apperrors.ErrDependency + } + ids := make([]int64, 0, len(users)) + for _, user := range users { + if user != nil && user.ID > 0 { + ids = append(ids, user.ID) + } + } + if len(ids) == 0 { + ids = append(ids, -1) + } + return ids, nil +} + +func ownerActive(ctx context.Context, ownerID int64) (bool, error) { + owner, err := passport.GetByID(ctx, ownerID) + if err != nil { + return false, apperrors.ErrDependency + } + return owner != nil, nil +} + +func listDecorations(database *gorm.DB, manuals []*model.Manual) (map[int64]int, map[int64]*dal.ManualItemPreview, map[int64][]string, error) { + manualIDs, explicitItemIDs, autoManualIDs := make([]int64, 0, len(manuals)), make([]int64, 0, len(manuals)), make([]int64, 0, len(manuals)) + for _, manual := range manuals { + manualIDs = append(manualIDs, manual.ID) + if manual.CoverItemID == nil { + autoManualIDs = append(autoManualIDs, manual.ID) + } else { + explicitItemIDs = append(explicitItemIDs, *manual.CoverItemID) + } + } + counts, err := dal.ListManualItemCounts(database, manualIDs) + if err != nil { + return nil, nil, nil, apperrors.ErrDependency + } + explicit, err := dal.ListManualItemPreviewsByIDs(database, explicitItemIDs) + if err != nil { + return nil, nil, nil, apperrors.ErrDependency + } + automatic, err := dal.ListAutomaticManualCovers(database, autoManualIDs) + if err != nil { + return nil, nil, nil, apperrors.ErrDependency + } + categoryRows, err := dal.ListManualCategoriesByManualIDs(database, manualIDs) + if err != nil { + return nil, nil, nil, apperrors.ErrDependency + } + countMap, coverMap := make(map[int64]int, len(counts)), make(map[int64]*dal.ManualItemPreview, len(manuals)) + for _, count := range counts { + countMap[count.ManualID] = count.Count + } + for _, cover := range append(explicit, automatic...) { + coverMap[cover.ManualID] = cover + } + return countMap, coverMap, categoriesByManualID(categoryRows), nil +} + +func categoriesByManualID(rows []*model.ManualCategory) map[int64][]string { + categories := make(map[int64][]string) + for _, row := range rows { + categories[row.ManualID] = append(categories[row.ManualID], row.Category) + } + return categories +} + +func chooseDetailCover(explicitID *int64, items []*model.ManualItem) *dal.ManualItemPreview { + if explicitID != nil { + for _, item := range items { + if item.ID == *explicitID { + return previewFromItem(item) + } + } + } + for _, item := range items { + if (item.Kind == model.ManualItemImage || item.Kind == model.ManualItemPDF) && item.PreviewStatus == model.ManualPreviewReady { + return previewFromItem(item) + } + } + if len(items) > 0 { + return previewFromItem(items[0]) + } + return nil +} + +func previewFromItem(item *model.ManualItem) *dal.ManualItemPreview { + return &dal.ManualItemPreview{ID: item.ID, ManualID: item.ManualID, Kind: item.Kind, Title: item.Title, TextExcerpt: excerpt(item.Text, 160), URL: item.URL, ThumbnailKey: item.ThumbnailKey, PreviewStatus: item.PreviewStatus, Position: item.Position} +} + +func manualView(manual *model.Manual, categories []string, itemCount int, cover *dal.ManualItemPreview, items []*model.ManualItem, principal *utils.Principal) *service.ManualView { + if categories == nil { + categories = []string{} + } + view := &service.ManualView{ID: strconv.FormatInt(manual.ID, 10), Name: manual.Name, Description: manual.Description, Categories: categories, AccessMode: manual.AccessMode.String(), Status: manual.Status.String(), Revision: manual.Revision, ItemCount: itemCount, CanEdit: canEdit(manual.OwnerUserID, principal), CreateTime: manual.CreateTime, UpdateTime: manual.UpdateTime} + if manual.CoverItemID != nil { + value := strconv.FormatInt(*manual.CoverItemID, 10) + view.CoverItemID = &value + } + if cover != nil { + view.Cover = coverView(manual.ID, cover) + view.CoverURL = view.Cover.ThumbnailURL + } + if items != nil { + view.Items = make([]*service.ItemView, 0, len(items)) + for _, item := range items { + view.Items = append(view.Items, itemView(item)) + } + } + return view +} + +func coverView(manualID int64, item *dal.ManualItemPreview) *service.CoverView { + view := &service.CoverView{ID: strconv.FormatInt(item.ID, 10), Kind: item.Kind.String(), Title: item.Title, TextExcerpt: excerpt(item.TextExcerpt, 160), PreviewStatus: item.PreviewStatus.String()} + if item.ThumbnailKey != nil && item.PreviewStatus == model.ManualPreviewReady { + view.ThumbnailURL = itemResourceURL(manualID, item.ID, "thumbnail") + } + if parsed, err := url.Parse(item.URL); err == nil { + view.URLHost = parsed.Hostname() + } + return view +} + +func itemView(item *model.ManualItem) *service.ItemView { + view := &service.ItemView{ID: strconv.FormatInt(item.ID, 10), Kind: item.Kind.String(), Title: item.Title, Text: item.Text, URL: item.URL, OriginalName: item.OriginalName, ContentType: item.ContentType, SizeBytes: item.SizeBytes, Position: item.Position, PreviewStatus: item.PreviewStatus.String()} + if item.StorageKey != nil { + view.ContentURL = itemResourceURL(item.ManualID, item.ID, "content") + } + if item.ThumbnailKey != nil && item.PreviewStatus == model.ManualPreviewReady { + view.ThumbnailURL = itemResourceURL(item.ManualID, item.ID, "thumbnail") + } + return view +} + +func itemResourceURL(manualID, itemID int64, resource string) string { + return "/api/manuals/" + strconv.FormatInt(manualID, 10) + "/items/" + strconv.FormatInt(itemID, 10) + "/" + resource +} + +func canEdit(ownerID int64, principal *utils.Principal) bool { + return principal != nil && principal.UserID == ownerID && (principal.Kind == "user" || principal.Allows("manuals:write")) +} + +func excerpt(value string, limit int) string { + runes := []rune(value) + if len(runes) > limit { + runes = runes[:limit] + } + return string(runes) +} diff --git a/app/manuals/resource.go b/app/manuals/resource.go new file mode 100644 index 0000000..a431ebd --- /dev/null +++ b/app/manuals/resource.go @@ -0,0 +1,42 @@ +package manuals + +import ( + "context" + + "github.com/mcoder2014/home_server/domain/dal" + "github.com/mcoder2014/home_server/domain/model" + service "github.com/mcoder2014/home_server/domain/service/manuals" + apperrors "github.com/mcoder2014/home_server/errors" +) + +func (application *Application) ResourceItem(ctx context.Context, manualID, itemID, viewerID int64, thumbnail bool) (*model.ManualItem, error) { + database, err := database(ctx) + if err != nil { + return nil, err + } + manual, err := dal.FindManual(database, manualID, false) + if err != nil { + return nil, apperrors.ErrDependency + } + if manual == nil { + return nil, apperrors.ErrNotFound + } + active, err := ownerActive(ctx, manual.OwnerUserID) + if err != nil { + return nil, err + } + if !active || !service.CanReadManual(manual.AccessMode, manual.Status, manual.OwnerUserID, viewerID) { + return nil, apperrors.ErrNotFound + } + item, err := dal.FindManualItem(database, manualID, itemID, false) + if err != nil { + return nil, apperrors.ErrDependency + } + if item == nil || item.StorageKey == nil { + return nil, apperrors.ErrNotFound + } + if thumbnail && (item.ThumbnailKey == nil || item.PreviewStatus != model.ManualPreviewReady) { + return nil, apperrors.ErrNotFound + } + return item, nil +} diff --git a/app/manuals/update.go b/app/manuals/update.go new file mode 100644 index 0000000..d2cffd4 --- /dev/null +++ b/app/manuals/update.go @@ -0,0 +1,296 @@ +package manuals + +import ( + "context" + "strconv" + "time" + + "github.com/mcoder2014/home_server/domain/dal" + "github.com/mcoder2014/home_server/domain/model" + service "github.com/mcoder2014/home_server/domain/service/manuals" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/mcoder2014/home_server/utils" + "gorm.io/gorm" +) + +func (application *Application) Update(ctx context.Context, ownerID, manualID int64, input service.UpdateManualInput, principal *utils.Principal) (*service.ManualView, error) { + if input.Revision <= 0 || !hasManualPatch(input) { + return nil, apperrors.ErrInvalid + } + database, err := database(ctx) + if err != nil { + return nil, err + } + unlock := application.lockOwner(ownerID) + defer unlock() + err = database.Transaction(func(tx *gorm.DB) error { + if err := requireWritePolicy(ctx, tx, ownerID, principal); err != nil { + return err + } + manual, err := dal.FindManual(tx, manualID, true) + if err != nil { + return err + } + if manual == nil || manual.OwnerUserID != ownerID || manual.Status == model.ManualStatusDeleted { + return apperrors.ErrNotFound + } + if manual.Revision != input.Revision { + return apperrors.ErrConflict + } + return applyManualPatch(tx, manual, input, principal) + }) + if err != nil { + return nil, persistenceError(err) + } + return application.Get(ctx, manualID, ownerID, principal) +} + +func hasManualPatch(input service.UpdateManualInput) bool { + return input.Name != nil || input.Description != nil || input.Categories != nil || input.AccessMode != nil || input.Status != nil || input.CoverItemID.Set || input.ItemIDs != nil +} + +func applyManualPatch(tx *gorm.DB, manual *model.Manual, input service.UpdateManualInput, principal *utils.Principal) error { + name, description, accessMode := manual.Name, manual.Description, manual.AccessMode.String() + if input.Name != nil { + name = *input.Name + } + if input.Description != nil { + description = *input.Description + } + if input.AccessMode != nil { + accessMode = *input.AccessMode + } + name, description, access, err := service.ValidateManualFields(name, description, accessMode) + if err != nil { + return err + } + var categories []string + if input.Categories != nil { + categories, err = service.ValidateCategories(*input.Categories) + if err != nil { + return err + } + } + fields := map[string]interface{}{"name": name, "description": description, "access_mode": access, "revision": gorm.Expr("revision + 1"), "update_time": time.Now()} + items, err := patchItems(tx, manual, input, fields) + if err != nil { + return err + } + if input.Status != nil { + status, ok := model.ParseManualStatus(*input.Status) + if !ok || status == model.ManualStatusDeleted || (status != manual.Status && !(manual.Status == model.ManualStatusDraft && status == model.ManualStatusActive)) { + return apperrors.ErrInvalid + } + if status == model.ManualStatusActive && len(items) == 0 { + return apperrors.ErrUnprocessable + } + fields["status"] = status + } + if err := checkWriteExpiry(principal); err != nil { + return err + } + updated, err := dal.UpdateManualFieldsCAS(tx, manual.OwnerUserID, manual.ID, manual.Revision, fields) + if err != nil { + return err + } + if !updated { + return apperrors.ErrConflict + } + if input.Categories != nil { + return dal.ReplaceManualCategories(tx, manual.ID, categories) + } + return nil +} + +func patchItems(tx *gorm.DB, manual *model.Manual, input service.UpdateManualInput, fields map[string]interface{}) ([]*model.ManualItem, error) { + needItems := input.ItemIDs != nil || input.CoverItemID.Set || (input.Status != nil && *input.Status == "active") + if !needItems { + return nil, nil + } + items, err := dal.ListManualItems(tx, manual.ID, true) + if err != nil { + return nil, err + } + byID := make(map[int64]*model.ManualItem, len(items)) + for _, item := range items { + byID[item.ID] = item + } + if input.ItemIDs != nil { + ordered, err := parseCompleteItemOrder(*input.ItemIDs, byID) + if err != nil { + return nil, err + } + positions := make(map[int64]int) + for index, id := range ordered { + if byID[id].Position != index+1 { + positions[id] = index + 1 + } + } + updated, err := dal.UpdateManualItemPositions(tx, manual.ID, positions) + if err != nil { + return nil, err + } + if !updated { + return nil, apperrors.ErrConflict + } + } + if input.CoverItemID.Set { + if input.CoverItemID.Value == nil { + fields["cover_item_id"] = nil + } else { + coverID, err := parsePositiveID(*input.CoverItemID.Value) + if err != nil || byID[coverID] == nil { + return nil, apperrors.ErrInvalid + } + fields["cover_item_id"] = coverID + } + } + return items, nil +} + +func parseCompleteItemOrder(values []string, items map[int64]*model.ManualItem) ([]int64, error) { + if len(values) != len(items) { + return nil, apperrors.ErrInvalid + } + ordered, seen := make([]int64, 0, len(values)), make(map[int64]bool, len(values)) + for _, value := range values { + id, err := parsePositiveID(value) + if err != nil || seen[id] || items[id] == nil { + return nil, apperrors.ErrInvalid + } + seen[id] = true + ordered = append(ordered, id) + } + return ordered, nil +} + +func (application *Application) Delete(ctx context.Context, ownerID, manualID, revision int64, principal *utils.Principal) (int64, error) { + if revision <= 0 { + return 0, apperrors.ErrInvalid + } + database, err := database(ctx) + if err != nil { + return 0, err + } + unlock := application.lockOwner(ownerID) + defer unlock() + err = database.Transaction(func(tx *gorm.DB) error { + if err := requireWritePolicy(ctx, tx, ownerID, principal); err != nil { + return err + } + manual, err := dal.FindManual(tx, manualID, true) + if err != nil { + return err + } + if manual == nil || manual.OwnerUserID != ownerID || manual.Status == model.ManualStatusDeleted { + return apperrors.ErrNotFound + } + if manual.Revision != revision { + return apperrors.ErrConflict + } + if err := checkWriteExpiry(principal); err != nil { + return err + } + updated, err := dal.UpdateManualFieldsCAS(tx, ownerID, manualID, revision, map[string]interface{}{"status": model.ManualStatusDeleted, "revision": gorm.Expr("revision + 1"), "update_time": time.Now()}) + if err != nil { + return err + } + if !updated { + return apperrors.ErrConflict + } + return nil + }) + if err != nil { + return 0, persistenceError(err) + } + return revision + 1, nil +} + +func (application *Application) DeleteItem(ctx context.Context, ownerID, manualID, itemID, revision int64, principal *utils.Principal) (*service.DeleteItemResult, error) { + if revision <= 0 { + return nil, apperrors.ErrInvalid + } + database, err := database(ctx) + if err != nil { + return nil, err + } + var coverID *int64 + unlock := application.lockOwner(ownerID) + defer unlock() + err = database.Transaction(func(tx *gorm.DB) error { + if err := requireWritePolicy(ctx, tx, ownerID, principal); err != nil { + return err + } + manual, err := dal.FindManual(tx, manualID, true) + if err != nil { + return err + } + if manual == nil || manual.OwnerUserID != ownerID || manual.Status == model.ManualStatusDeleted { + return apperrors.ErrNotFound + } + if manual.Revision != revision { + return apperrors.ErrConflict + } + items, err := dal.ListManualItems(tx, manualID, true) + if err != nil { + return err + } + item := findItem(items, itemID) + if item == nil { + return apperrors.ErrNotFound + } + if manual.Status == model.ManualStatusActive && len(items) == 1 { + return apperrors.ErrUnprocessable + } + now := time.Now() + if err := dal.DeleteManualItem(tx, manualID, itemID, now); err != nil { + return err + } + if err := dal.CompactManualItemPositions(tx, manualID, item.Position); err != nil { + return err + } + coverID = manual.CoverItemID + fields := map[string]interface{}{"revision": gorm.Expr("revision + 1"), "update_time": now} + if coverID != nil && *coverID == itemID { + coverID = nil + fields["cover_item_id"] = nil + } + if err := checkWriteExpiry(principal); err != nil { + return err + } + updated, err := dal.UpdateManualFieldsCAS(tx, ownerID, manualID, revision, fields) + if err != nil { + return err + } + if !updated { + return apperrors.ErrConflict + } + return nil + }) + if err != nil { + return nil, persistenceError(err) + } + result := &service.DeleteItemResult{Revision: revision + 1} + if coverID != nil { + value := strconv.FormatInt(*coverID, 10) + result.CoverItemID = &value + } + return result, nil +} + +func findItem(items []*model.ManualItem, id int64) *model.ManualItem { + for _, item := range items { + if item.ID == id { + return item + } + } + return nil +} + +func parsePositiveID(value string) (int64, error) { + id, err := strconv.ParseInt(value, 10, 64) + if err != nil || id <= 0 || strconv.FormatInt(id, 10) != value { + return 0, apperrors.ErrInvalid + } + return id, nil +} diff --git a/config/config_example.yaml b/config/config_example.yaml index d877402..5352e3d 100644 --- a/config/config_example.yaml +++ b/config/config_example.yaml @@ -39,6 +39,25 @@ web_projects: max_concurrent_extracts: 4 delete_retention_days: 7 +manuals: + # Keep disabled until the migration and private storage directory are ready. + enabled: false + storage_root: /var/lib/home_server/manuals + max_file_bytes: 52428800 + max_items_per_manual: 100 + max_manual_bytes: 524288000 + max_manuals_per_user: 1000 + max_user_bytes: 10737418240 + min_free_disk_bytes: 2147483648 + pdftoppm_path: /usr/bin/pdftoppm + prlimit_path: /usr/bin/prlimit + pdf_preview_timeout_seconds: 10 + max_concurrent_pdf_previews: 1 + pdf_preview_memory_limit_bytes: 536870912 + pdf_preview_cpu_seconds: 10 + pdf_preview_output_limit_bytes: 16777216 + pdf_preview_open_files_limit: 32 + auth: # Enable after application tables and same-origin HTTPS proxy are installed. applications_enabled: false diff --git a/config/global.go b/config/global.go index c9bf401..1961a79 100644 --- a/config/global.go +++ b/config/global.go @@ -50,6 +50,7 @@ type Config struct { SharePath string `json:"share_path" yaml:"share_path"` } `json:"webdav" yaml:"webdav"` WebProjects WebProjectsConfig `json:"web_projects" yaml:"web_projects"` + Manuals ManualsConfig `json:"manuals" yaml:"manuals"` Auth AuthConfig `json:"auth" yaml:"auth"` } @@ -87,6 +88,28 @@ type WebProjectsConfig struct { DeleteRetentionDays int `json:"delete_retention_days" yaml:"delete_retention_days"` } +// ManualsConfig contains deployment and capacity boundaries for private +// manual files. It is intentionally absent from the database configuration +// registry so existing complete snapshots remain valid. +type ManualsConfig struct { + Enabled bool `json:"enabled" yaml:"enabled"` + StorageRoot string `json:"storage_root" yaml:"storage_root"` + MaxFileBytes int64 `json:"max_file_bytes" yaml:"max_file_bytes"` + MaxItemsPerManual int `json:"max_items_per_manual" yaml:"max_items_per_manual"` + MaxManualBytes int64 `json:"max_manual_bytes" yaml:"max_manual_bytes"` + MaxManualsPerUser int `json:"max_manuals_per_user" yaml:"max_manuals_per_user"` + MaxUserBytes int64 `json:"max_user_bytes" yaml:"max_user_bytes"` + MinFreeDiskBytes int64 `json:"min_free_disk_bytes" yaml:"min_free_disk_bytes"` + PDFToPPMPath string `json:"pdftoppm_path" yaml:"pdftoppm_path"` + PRLimitPath string `json:"prlimit_path" yaml:"prlimit_path"` + PDFPreviewTimeoutSeconds int `json:"pdf_preview_timeout_seconds" yaml:"pdf_preview_timeout_seconds"` + MaxConcurrentPDFPreviews int `json:"max_concurrent_pdf_previews" yaml:"max_concurrent_pdf_previews"` + PDFPreviewMemoryLimitBytes int64 `json:"pdf_preview_memory_limit_bytes" yaml:"pdf_preview_memory_limit_bytes"` + PDFPreviewCPUSeconds int `json:"pdf_preview_cpu_seconds" yaml:"pdf_preview_cpu_seconds"` + PDFPreviewOutputLimitBytes int64 `json:"pdf_preview_output_limit_bytes" yaml:"pdf_preview_output_limit_bytes"` + PDFPreviewOpenFilesLimit int `json:"pdf_preview_open_files_limit" yaml:"pdf_preview_open_files_limit"` +} + // 全局配置 var globalConfig = Config{} var globalConfigLock sync.RWMutex diff --git a/config/nginx/manuals_locations.conf b/config/nginx/manuals_locations.conf new file mode 100644 index 0000000..99cad18 --- /dev/null +++ b/config/nginx/manuals_locations.conf @@ -0,0 +1,28 @@ +# All manual metadata, originals and thumbnails go through backend authorization. +# Do not add a static alias for manuals.storage_root. +location = /api/manuals { + client_max_body_size 512k; + proxy_pass http://127.0.0.1:18080; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $remote_addr; + access_log off; + proxy_cache off; + proxy_intercept_errors off; + add_header Cache-Control "no-store" always; +} + +location ^~ /api/manuals/ { + # 50 MiB file limit plus multipart overhead; the application checks each body. + client_max_body_size 52m; + proxy_read_timeout 120s; + proxy_request_buffering off; + proxy_pass http://127.0.0.1:18080; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $remote_addr; + access_log off; + proxy_cache off; + proxy_intercept_errors off; + add_header Cache-Control "no-store" always; +} diff --git a/config/runtime.go b/config/runtime.go index 76ac7cc..eb2c901 100644 --- a/config/runtime.go +++ b/config/runtime.go @@ -20,6 +20,7 @@ type AccountPolicyConfig struct { type RuntimeConfig struct { Auth AuthConfig WebProjects WebProjectsConfig + Manuals ManualsConfig RegistrationEnabled, LibraryEnabled, WebDAVEnabled, CleanupEnabled bool SiteTitle, SiteNotice string AccountPolicy AccountPolicyConfig @@ -46,7 +47,7 @@ func BuildRuntimeSnapshot(conf Config, generation int64, revisions map[string]in } validated[schema.Namespace] = v } - result := RuntimeConfig{Auth: conf.Auth, WebProjects: conf.WebProjects, Generation: generation, Revisions: revisions} + result := RuntimeConfig{Auth: conf.Auth, WebProjects: conf.WebProjects, Manuals: conf.Manuals, Generation: generation, Revisions: revisions} for namespace, target := range map[string]interface{}{"auth": &result.Auth, "web_projects": &result.WebProjects, "account_policy": &result.AccountPolicy} { data, err := json.Marshal(validated[namespace]) if err != nil { diff --git a/config/runtime_test.go b/config/runtime_test.go index dfba81f..a14fe49 100644 --- a/config/runtime_test.go +++ b/config/runtime_test.go @@ -37,6 +37,8 @@ func TestRuntimeSnapshotIsCompleteImmutableAndMonotonic(t *testing.T) { conf := Config{} conf.Auth.SiteOrigins = []string{"https://home.example.com"} conf.WebProjects.StorageRoot = "/private/example" + conf.Manuals.Enabled = true + conf.Manuals.StorageRoot = "/private/manuals" values := DefaultRuntimeValues(conf) revisions := map[string]int64{} for name := range values { @@ -52,7 +54,7 @@ func TestRuntimeSnapshotIsCompleteImmutableAndMonotonic(t *testing.T) { snapshot.Auth.SiteOrigins[0] = "https://attacker.example" snapshot.Revisions["site"] = 99 loaded := Runtime() - if loaded.Auth.SiteOrigins[0] != "https://home.example.com" || loaded.Revisions["site"] != 1 || loaded.WebProjects.StorageRoot != "/private/example" { + if loaded.Auth.SiteOrigins[0] != "https://home.example.com" || loaded.Revisions["site"] != 1 || loaded.WebProjects.StorageRoot != "/private/example" || !loaded.Manuals.Enabled || loaded.Manuals.StorageRoot != "/private/manuals" { t.Fatal("snapshot retained mutable data or discarded bootstrap path") } loaded.Auth.SiteOrigins[0] = "https://mutated.example" diff --git a/deploy/pi/README.md b/deploy/pi/README.md index 51a2220..fe0f827 100644 --- a/deploy/pi/README.md +++ b/deploy/pi/README.md @@ -41,11 +41,28 @@ 仓库静态回归命令为 `node --test front_vue/test/web-projects-nginx.test.cjs`，覆盖新增路径、请求体上限、转发头、禁缓存、双入口和 WebDAV 规则。安装现场仍须对实际配置执行独立前端 Nginx 的 `-t -c /etc/home_server/frontend-nginx.conf` 及网关的 `nginx -t`，再验证两个入口下的 `/api/site/bootstrap`、已登录个人中心、管理员配置请求，以及原有 WebDAV Basic 访问。 +## 说明书模块 + +说明书的部署配置独立于数据库中的动态 namespace，缺省关闭。启用前先执行精确的 `domain/dal/migrations/20260919_manuals.sql` 增量建表，检查配置与数据备份，然后在服务配置中设置 `manuals.enabled` 和 `manuals.storage_root`。存储目录建议 `/var/lib/home_server/manuals`，权限 0700、归服务用户所有，不能位于 WebDAV、网页托管或前端静态目录中。 + +安装 `config/nginx/manuals_locations.conf` 到 `/etc/home_server/locations/`，在 Pi 副本中将 `$scheme` / `$remote_addr` 转为现有的 `$cq_forwarded_proto` / `$cq_client_ip`，并让独立前端 Nginx include 该文件。API 根路径限制 512 KiB，子路径允许 52 MiB 以容纳 50 MiB 文件和 multipart 元数据；后端仍按请求类型独立限制大小。两份网关模板也增加了 `/api/manuals` 精确匹配与 `/api/manuals/` 优先前缀，保留原有目标端口及 TLS 头。 + +| 检查 | 要求 | +| --- | --- | +| PDF 预览 | 核对 `pdftoppm` 与外部进程资源限制工具可用，使用服务账号验证首页缩略图；缺工具或转换失败不得让 PDF 上传状态冒充预览成功 | +| 私有存储 | 目录可写、容量及保留空间满足限制；禁止为原件或缩略图增加静态 alias | +| 数据与搜索 | 自动封面查询使用窗口函数，已核对环境为 MariaDB 10.11；其他数据库版本需先核对窗口函数兼容性；新表精确增量创建；在隔离数据中检查列表、分类及名字匹配 SQL 的 EXPLAIN | +| 身份 | 应用凭证显式加入 `manuals:read/write`；保持原应用其他 scopes，不提取用户会话替代应用授权 | +| 升级 | 构建匹配的前后端 release、记录源码 SHA 与产物哈希，Nginx 校验通过后切 current 并重启 | +| 回滚 | 切回完整旧 release 和旧配置；保留新增表及全部说明书文件，不在程序回滚时 DROP 或覆盖数据 | + +部署后除 `/ping` 外，还需验证公开目录、私有说明书未登录拒绝、原件/缩略图鉴权、PDF Range、图片多选和名称＋分类过滤，以及原网页托管、登录、藏书与 WebDAV 入口。真实手机相册的交互应记录实际验收设备，不能以桌面文件选择测试冒充。 + ## 安装顺序 1. 核对并备份既有数据库、配置、服务单元和二进制；备份含凭据，只保存在目标服务器的私有目录。 2. 核对表结构与索引，再执行缺失的增量 DDL。首次启用运行 `20260909_web_projects.sql` 与 `20260912_applications.sql`；已有 token 索引时跳过重复创建。已有旧网页表时按其真实结构选择升级脚本，不混用首次建表与旧表升级。 -3. 从指定 master revision 构建 release，记录源码 revision、构建参数及产物 SHA-256；不要覆盖正在执行的二进制文件内容。 +3. 从通过验证的指定 revision 构建 release（未合并功能使用明确的 PR 提交 SHA），记录源码 revision、构建参数及产物 SHA-256；不要覆盖正在执行的二进制文件内容。 4. 安装后端配置、前端配置与三个 systemd 文件，运行 `systemd-analyze verify` 和独立 Nginx 配置检查，再切换 `current` 链接。 5. `systemctl daemon-reload`，启用前后端与 target，再启动或重启。通过 `.10:18081` 检查 SPA、同源 API 和静态资源。 6. 备份网关原站点文件，将两个网关模板安装到对应站点的实际目标文件；`nginx -t` 成功后 reload。通过域名 SNI 验证 TLS、登录入口、API 和原始路径防护。 diff --git a/deploy/pi/frontend-nginx.conf b/deploy/pi/frontend-nginx.conf index 2de6b85..bf03aee 100644 --- a/deploy/pi/frontend-nginx.conf +++ b/deploy/pi/frontend-nginx.conf @@ -28,12 +28,13 @@ http { server_name _; root /opt/home_server/current/frontend; index index.html; - # Installed copies of the repo's two API snippets use + # Installed copies of the repo's API snippets use # $cq_forwarded_proto and $cq_client_ip in proxy headers. # api_auth_locations also routes /api/site/, /api/account/ and /api/admin/; # keep these specific locations ahead of the generic /api/ rejection. include /etc/home_server/locations/web_projects_locations.conf; include /etc/home_server/locations/api_auth_locations.conf; + include /etc/home_server/locations/manuals_locations.conf; location ^~ /passport/ { client_max_body_size 16k; proxy_pass http://127.0.0.1:18080; diff --git a/deploy/pi/gateway-api.conf b/deploy/pi/gateway-api.conf index 5b6f6dd..899095d 100644 --- a/deploy/pi/gateway-api.conf +++ b/deploy/pi/gateway-api.conf @@ -17,6 +17,33 @@ server { return 400; } + # Exact root and ^~ prefix keep manual originals and thumbnails behind auth. + location = /api/manuals { + client_max_body_size 512k; + proxy_pass http://192.0.2.10:18080; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_cache off; + proxy_intercept_errors off; + add_header Cache-Control "no-store" always; + } + + location ^~ /api/manuals/ { + client_max_body_size 52m; + proxy_pass http://192.0.2.10:18080; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Connection ""; + proxy_request_buffering off; + proxy_read_timeout 120s; + proxy_cache off; + proxy_intercept_errors off; + add_header Cache-Control "no-store" always; + } + # Keep only the new account namespaces on a bounded, uncached API route. # Preserve the existing gateway destination and raw Host (including port). location = /api/account/avatar { diff --git a/deploy/pi/gateway-frontend.conf b/deploy/pi/gateway-frontend.conf index db97a61..c659af0 100644 --- a/deploy/pi/gateway-frontend.conf +++ b/deploy/pi/gateway-frontend.conf @@ -17,6 +17,33 @@ server { return 400; } + # Exact root and ^~ prefix keep manual originals and thumbnails behind auth. + location = /api/manuals { + client_max_body_size 512k; + proxy_pass http://192.0.2.10:18081; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_cache off; + proxy_intercept_errors off; + add_header Cache-Control "no-store" always; + } + + location ^~ /api/manuals/ { + client_max_body_size 52m; + proxy_pass http://192.0.2.10:18081; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Connection ""; + proxy_request_buffering off; + proxy_read_timeout 120s; + proxy_cache off; + proxy_intercept_errors off; + add_header Cache-Control "no-store" always; + } + # Keep only the new account namespaces on a bounded, uncached API route. # Preserve the existing gateway destination and raw Host (including port). location = /api/account/avatar { diff --git a/docs/specs/manual-management/design.md b/docs/specs/manual-management/design.md new file mode 100644 index 0000000..be1caf2 --- /dev/null +++ b/docs/specs/manual-management/design.md @@ -0,0 +1,155 @@ +# 说明书管理：系统设计与技术方案 + +日期：2026-09-19。设计基线：master `176bce3`。 + +## 1. 目标与范围 + +将家庭设备的纸质照片、PDF、文字和官网链接组织成可检索的说明书。一份说明书包含多条有序资料，不要求用户先合成 PDF 或 HTML。 + +| 场景 | 目标行为 | +| --- | --- | +| 买到设备 | 输入名称、分类，批量选照片，可继续从其他相册追加；同时加入多个 PDF、文本和 URL | +| 找使用方法 | 按分类筛选、按名称包含关键词检索，浏览封面缩略图，进入详情按顺序查阅 | +| 分享 | 选择仅自己、登录用户、所有人可见，默认仅自己；分享链接不携带凭据 | +| 管理 | 所有者编辑名称、分类、资料顺序及可见范围，删除资料或整份说明书 | +| AI 操作 | 通过现有应用凭证机制与 skill 完成创建、上传、查询和编辑 | + +采用个人单层多分类，每份说明书可属于多个分类，空数组表示未分类。分类名称去除首尾空白后精确去重，大小写不同视为不同分类；最多 20 个，每个最多 100 字符。分类关系在说明书内维护，筛选项从当前访问者可见的说明书中提取，不引入独立分类管理系统。名称允许重复，以 ID 唯一定位。匿名访问者可浏览公开说明书目录；登录用户看到自己及对其开放的说明书。 + +首期不含 OCR、正文全文搜索、自动抓取 URL 内容、设备资产/保修管理、协同编辑或版本发布系统。URL 只保存链接，外部网站下线后不能保证内容继续存在；需要长期留存的资料应上传文件。 + +## 2. 已核对的现状 + +| 真实来源 | 已确认事实及复用方式 | +| --- | --- | +| `api/webprojects/route.go`、`app/webprojects/projects.go` | 网页托管有草稿、版本发布、访问控制；说明书独立建模，复用身份及错误响应模式 | +| `api/middleware/identity.go`、`account.go` | 支持 Cookie、用户 token、应用 Bearer；Cookie 写请求带 CSRF/Origin 检查，显式错误凭据不降级匿名 | +| `domain/service/applications/service.go` | scope 是显式白名单；新增 `manuals:read`、`manuals:write`，write 包含 read | +| `config/runtime.go`、`registry.go` | 数据库配置快照要求所有 namespace 完整；首期说明书使用独立部署配置，避免添加 namespace 使已有快照失效 | +| `front_vue/src/router/index.js`、`permission.js` | Vue 3、Element Plus、现有登录守卫；说明书列表与阅读页面允许匿名，新建和编辑需要登录 | +| `skills/home-server-web-share/`、`script/home_server_api.py` | 已有 Python 标准库 HTTPS 客户端及凭据约束；扩展该 skill，保留原命令兼容性 | +| `.github/workflows/go.yml` | 有 Linux Go 构建、MariaDB/Redis 隔离集成测试、Node 和 Python 测试 | +| `deploy/pi/README.md` | 使用 release 目录和 current 软链部署，前后端统一切换；新增数据不随程序回滚删除 | + +2026-09-19 只读现场检查：SSH 主机 `pi` 是 x86_64；前后端服务 active；当前 release 为 account-center PR #26 版本；存储盘可用约 126 GiB；已有 `/usr/bin/pdftoppm` 和 `/usr/bin/prlimit`；验证及构建使用临时独立的 Go 1.21.13 工具链，不改系统工具链。主机名不代表 ARM 架构。 + +## 3. 方案选择 + +| 方案 | 代价 | 判断 | +| --- | --- | --- | +| 独立说明书模块，共用身份与客户端 | 新增三张表、一组 API 和列表、编辑、阅读页面，存储及权限边界明确 | 采用 | +| 把资料自动拼成网页发布版本 | 图片/PDF 需要转换、重新发布，分类检索与单项编辑难以自然表达 | 不采用 | +| 接入完整文档管理系统 | 多一个部署、账号与备份体系，家庭场景收益不足 | 不采用 | + +核心关系是“说明书 1:N 资料项”。一条资料只有一种类型，不拆成四套业务流程；文件类型差异集中在校验、缩略图生成与阅读展示。现有网页托管表、发布链路和存储路径保持原有语义。 + +```mermaid +flowchart LR + A["浏览器：列表、编辑、阅读"] --> C["说明书 API 与权限检查"] + B["现有 skill：应用 Bearer"] --> C + C --> D["说明书服务：元数据、资料顺序、配额"] + D --> E["MySQL：说明书、资料、分类关系"] + D --> F["私有文件目录：原件与缩略图"] + F --> G["受限预览处理：图片缩放、PDF 首页"] +``` + +## 4. 数据与存储 + +| 数据表 | 主要字段 | 约束 | +| --- | --- | --- | +| `manuals` | id、owner_user_id、name、description、access_mode、status、cover_item_id、revision、client_request_id、request_hash、create_time、update_time | access_mode 为 owner/authenticated/public；status 为 draft/active/deleted；唯一 `(owner_user_id, client_request_id)` | +| `manual_items` | id、manual_id、kind、title、text、url、storage_key、thumbnail_key、content_type、original_name、size_bytes、sha256、position、preview_status、client_request_id、request_hash、deleted_at、create_time | kind 为 image/pdf/text/url；唯一 `(manual_id, client_request_id)`；文件路径不作为外部 API 返回字段 | +| `manual_categories` | manual_id、category、create_time | 主键 `(manual_id, category)`；反查索引 `(category, manual_id)`；category 使用 utf8mb4_bin，与应用精确去重一致 | + +ID 通过 JSON 字符串传输，避免 JavaScript 精度丢失。revision 是整数，元数据修改、排序和删除通过条件更新检测并发，冲突返回 409。创建与资料追加必须携带 client_request_id，使用 1—128 位 ASCII 字母、数字或 `._:-`（建议 UUID）；相同 ID 相同输入返回原结果，相同 ID 不同输入返回 409。 + +存储配置 `manuals.storage_root`，生产为 `/var/lib/home_server/manuals`。使用服务生成的 ID 组成 `///original` 和 `thumbnail.jpg`，不拼接用户文件名；目录不位于 WebDAV、前端静态根或网页托管目录内。配置必须检查目录重叠和软链接。原件保留用于下载；缩略图重新编码并移除元数据。 + +文件先写到同文件系统临时目录，完成体积/格式检查及缩略图处理后原子移动到最终目录，再提交资料记录。失败清理本次已知临时文件；数据库提交结果不确定时按 request ID 与本次生成的 item ID 同时核对，不能误删可能已提交的原件；旧幂等条目不能把当前授权或资源状态错误转换为成功。崩溃遗留文件只允许离线核对后回收，不添加不可靠的周期删除器。 + +删除采用逻辑删除，立即禁止读取；保留文件以支持人工恢复与回滚。已删除文件继续占用配额，界面说明删除不立即释放物理空间。存储备份同时包含数据库与原件，缩略图可重新生成。 + +## 5. 上传和查阅 + +### 5.1 多选、多次追加与失败恢复 + +图片使用 `input[type=file][multiple]` 唤起系统相册，文件入口选择 PDF/文本文件。每次选择追加到当前队列，取消选择不清空已有队列。支持上传前移除、调整顺序，并显示每项进度与错误。一次保存可处理多个不同类型资料，客户端按队列逐项调用 API，不把全部原件拼成一个超大请求。 + +系统选择器决定是否能在单次操作里切换相册；网页保证多次选择的合并效果，不能承诺接管系统相册。重复文件提示由文件名、大小、修改时间辅助识别，不据此擅自丢弃文件。原生多选依据见 [MDN file input](https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/input/file)。 + +新建先保存私有草稿，再逐项上传并记录成功项 ID；全部成功后按用户选择的范围激活。失败保留草稿和队列，仅重试失败项。刷新后可从草稿继续添加，但浏览器不能自动恢复用户尚未上传的本地文件，页面应提示重新选择。编辑现有说明书时，成功追加的资料即时可见，失败项明确标记，不能把部分成功提示为全部完成。 + +### 5.2 格式与缩略图 + +| 类型 | 输入及阅读 | 列表预览 | +| --- | --- | --- | +| 图片 | JPEG、PNG、GIF、WebP；原件下载与缩放查看 | 服务端生成最长边 480 px 的 JPEG；GIF 使用首帧 | +| PDF | 上传 PDF，浏览器内置查看器打开，另提供下载链接 | pi 已安装的 pdftoppm 渲染首页；失败显示 PDF 占位及预览不可用状态，原件保留 | +| 文本 | 多段 UTF-8 文本或 `.txt` 文件；文件上传解码后按相同 100000 字符上限保存为 kind=text，保留 original_name 和 UTF-8 size_bytes，不另存原件；纯文本显示、保留换行 | 文本摘要卡片 | +| URL | 多个 HTTP/HTTPS 地址与可选标题；外部新标签打开 | 域名与标题卡片；不抓取远程网页或 favicon | + +默认第一条可用图片/PDF 缩略图为封面，所有者可选择其他资料作为封面。数据库 cover_item_id=NULL 表示自动选择，非 NULL 表示用户指定；自动结果在读取时计算，不写回该字段。PATCH 省略该字段保留原意图，传 null 恢复自动。删除指定封面后回到自动选择。没有图像型封面时使用文本或链接摘要。HEIC/HEIF 不冒充 JPEG；不能解码时单项拒绝并提示转换成 JPEG/PNG，不影响其他文件。图片按实际解码格式验证，前端 accept 和 MIME 不能替代服务端检查。 + +图片缩略图先应用 JPEG EXIF 方向，再缩放重新编码，避免手机竖拍图片侧倒。预览处理要限制图片像素数（40 MP）、输出尺寸、图片与 PDF 共用的预览处理并发（全站 1）及 PDF 超时（10 秒）；外部程序使用参数数组，不拼 shell 命令，并使用 pi 已有 prlimit 限制内存、CPU、输出文件和文件描述符。运行资源限制需在部署验证中确认。PDF 原件保存和预览成功分别报告。 + +默认限制：名称 120 字、每份 20 个分类且每个 100 字、描述 2000 字；每段文本 100000 字符、URL 2048 字节；单文件 50 MiB、每份 100 条资料和 500 MiB、每用户 1000 份说明书及 10 GiB，磁盘保留 2 GiB。后端拒绝超限，代理正文限制留 multipart 余量（52 MiB）。计数及字节配额必须在事务内串行检查，磁盘保护包含处理中上传的预留量。 + +### 5.3 列表与查询 + +列表卡片包含封面、名称、分类、资料数量、范围和更新时间；过滤项为“我可访问的 / 我的”、分类和名称关键词。分类与关键词可组合；省略 category 参数表示全部，显式 `category=` 表示仅未分类，其他值表示包含指定分类。分类筛选用 EXISTS，未分类用 NOT EXISTS，避免多分类关联造成列表重复。按 ID 倒序游标分页，默认 20、最大 100，不提供大 OFFSET。 + +名称模糊匹配定义为大小写不敏感的字面子串匹配，中文可用，`%` 和 `_` 作为普通字符。name 列显式使用 utf8mb4_unicode_ci；参数化 LIKE 同时转义 `%`、`_` 和选定转义符，并用真实 MariaDB 查询验证。名称不建误导性的 B-tree 模糊索引：SQL 先利用 owner/status/access 及分类关系索引限制候选，再执行名称过滤；这种残余过滤仍会扫描候选，不能宣称为索引命中。通过隔离数据库的 EXPLAIN 和代表性数据确认扫描量；不为家庭数量级引入搜索引擎。 + +分类建议及计数必须应用相同可见性条件，不能通过筛选项泄漏别人的私有分类。分类建议按 utf8mb4_bin 顺序分页，默认 200、最大 1000；cursor 是上一页返回的最后一个分类文本，需 URL 编码。has_more=false 时 next_cursor 为空；前端取全可见分类后展示，不静默截断。列表不逐条查 SQL、不返回大段文本及全部附件，按当前页 ID 批量补齐封面、资料数量和分类，禁止逐份查询。创建与修改分类和说明书元数据在同一事务提交；PATCH 省略 categories 保留现有分类，传空数组清空分类。 + +## 6. API 契约 + +沿用现有 ginfmt 响应封装及错误码；下面是封装内部的数据。路径以 `/api/manuals` 为前缀，浏览器与 skill 共用，数值 ID 均为字符串。 + +| 方法与路径 | 输入 / 输出 | 权限 | +| --- | --- | --- | +| GET `/api/manuals` | `q, category, mine, cursor, limit`；返回 `items, has_more, next_cursor` | 匿名仅 public active，登录按可见范围 | +| GET `/api/manuals/categories` | `mine, cursor, limit`；返回 `items: [分类名], has_more, next_cursor` | 仅包含有权看到的分类 | +| POST `/api/manuals` | `name, description, categories, access_mode, client_request_id`；返回草稿详情 | 登录/`manuals:write` | +| GET `/api/manuals/:id` | 返回说明书和有序 `items`，带 `can_edit` | 所有者可看草稿，其他人只读 active | +| PATCH `/api/manuals/:id` | `revision`，可选 name/description/categories/access_mode/status/cover_item_id/item_ids | 仅所有者；status 只允许 draft→active，active 至少有一项；item_ids 必须是完整有效项集合 | +| DELETE `/api/manuals/:id` | JSON `revision` | 仅所有者，逻辑删除 | +| POST `/api/manuals/:id/items` | `kind: text/url, title, text/url, client_request_id`；返回 `item, revision` | 仅所有者，幂等追加一项 | +| POST `/api/manuals/:id/files` | multipart `file, title, client_request_id` | 仅所有者，幂等追加一份文件；返回 `item, revision` | +| DELETE `/api/manuals/:id/items/:item_id` | JSON `revision` | 仅所有者；删除封面后重选默认封面，active 的最后一项不可删除 | +| GET/HEAD `/api/manuals/:id/items/:item_id/content` | 原件或 `?download=1`，支持 Range | 与说明书同一权限 | +| GET/HEAD `/api/manuals/:id/items/:item_id/thumbnail` | 缩略图 | 与说明书同一权限 | + +详情字段：`id, name, description, categories, access_mode, status, revision, cover_item_id, cover_url, cover, item_count, can_edit, create_time, update_time, items`。资料字段：`id, kind, title, text, url, original_name, content_type, size_bytes, position, content_url, thumbnail_url, preview_status`。列表不包含 items 正文，使用有界 cover 对象：`id, kind, title, thumbnail_url, text_excerpt, url_host, preview_status`，其中 id 为当前实际封面资料 ID，text_excerpt 最多 160 字符。封面未显式指定时优先第一张可用图片/PDF 缩略图，否则首项；显式指定文本/URL 时显示该项摘要。 + +两种资料追加 API 都在说明书行锁内分配 position、将 revision 加一并返回 `{item, revision}`。相同请求的幂等重放不重复追加、不增加 revision，返回原资料及当前 revision。客户端每次成功后更新 revision，再用最新快照排序或激活；409 不自动覆盖其他终端的变更。 + +草稿 access_mode 可记录用户目标范围，但 status=draft 始终仅所有者可见。应用令牌对说明书读写需要各自 scope，并按应用所有者身份执行；应用并不扩大资源可见范围。所有者失效、账号禁用、强制改密、token 撤销与 Cookie CSRF 继续受原账号机制约束；匿名公开内容也不得绕过所有者账号状态规则。 + +所有写事务在取得账号/资源锁后复核账号状态、auth_version、精确用户会话是否已撤销、token 到期及应用授权快照，不能只依赖上传开始时的路由鉴权。数据库身份模式在任何非锁定数据库读取前，先锁定已知 owner 账号，再锁精确会话；这样 REPEATABLE READ 下后续配额查询能看到前一个事务的提交，不能用锁前会话查询建立旧快照。旧配置身份模式补充进程内用户锁，仅支持现有单实例部署。未删除的说明书和资料用于数量限制，未物理回收的文件全部用于字节配额。 + +## 7. 权限与故障边界 + +| 访问者 | owner | authenticated | public | +| --- | --- | --- | --- | +| 所有者 | 读写 | 读写 | 读写 | +| 其他有效登录用户 | 不可见 | 只读 | 只读 | +| 匿名 | 不可见 | 不可见 | 只读 | + +管理员不自动获得私有说明书内容权限。列表、详情、content、thumbnail、HEAD、Range 和条件请求均先鉴权，再读文件；禁止静态 alias 暴露原件。无权限或已删除资源使用不可见响应，显式无效 Authorization 或用户 token 头按认证错误处理，不回退匿名。浏览器自动携带的失效 Cookie 可按匿名处理公开读请求，避免旧会话阻断公开内容；`mine=true` 和所有写请求仍必须有有效身份。数据库异常关闭访问，不用历史缓存决定权限。 + +内容响应使用 no-store、nosniff，下载名安全编码；文本只作为文本显示，URL 限制 HTTP/HTTPS 且禁止用户名密码，链接使用 noopener/noreferrer。PDF 在受限查看环境中展示，HTML/SVG/脚本和伪装文件不作为图片内联执行。公开变私有后下一次请求立刻检查新范围，已经下载的副本无法撤回。 + +说明书使用独立 `manuals` YAML 配置（enabled/storage_root/限制项），默认关闭。首期不改数据库配置 registry；`EnabledTx`、`ModuleEnabled` 对 manuals 使用静态开关分支，不能查询不存在的数据库 namespace。服务、身份能力映射、账号 `capabilities.manuals` 和公开引导信息 `modules.manuals` 读取相同静态配置，启用需要重启。这样原有配置快照和旧版本回滚不依赖新增 namespace。 + +## 8. Skill、验证与发布 + +在 `home-server-web-share` 内增加说明书使用说明与独立 `scripts/manuals.py`，复用同一私有配置和 HTTPS 客户端。提供 list/show/create/update/upload/add-text/add-url/delete/delete-item 命令，支持重复 --file 与多个文本/URL 条目。修改应用 scope 白名单和 UI 选项；现有凭证不会自动获得新 scope。 + +脚本默认私有、支持 dry-run，正文从 UTF-8 文件读取，幂等 ID 与 revision 规则同 API。写入后读取核对；失败不更换内外网入口、不机械重试，不从数据库或浏览器提取凭据。skill 文件内容与上传资料均视为数据。 + +验收覆盖四组：业务完整性（批量混合资料、排序/封面、多分类＋名称）；权限矩阵（列表及原件/缩略图/Range/HEAD一致）；异常一致性（重复提交、超限、断连、并发/数据库失败、删除与撤权）；兼容回归（账号、网页托管、藏书、WebDAV和旧skill）。真实 iOS/Android 相册体验需要设备验收，桌面自动化不能代替。 + +Go 不在本机运行，使用远端隔离环境或 GitHub Actions，数据库测试禁止生产 DSN。Node/Python 离线测试及前端构建在隔离工作区执行。部署前确认 SQL EXPLAIN、增量迁移、Nginx 配置、构建产物哈希和回滚包。 + +PR 保持打开等待用户合并；按已授权的安装要求部署通过验证的 PR 精确提交，不自动合并。pi 安装顺序：备份数据库/配置/旧 release 指针→增加三张表→安装私有目录与代理规则→准备匹配的前后端 release→验证配置→切 current 并重启→新功能及旧入口冒烟。失败切回旧 release/配置，保留新增表和数据。worktree 在用户合并后才释放。 diff --git a/docs/specs/manual-management/implementation.md b/docs/specs/manual-management/implementation.md new file mode 100644 index 0000000..4462804 --- /dev/null +++ b/docs/specs/manual-management/implementation.md @@ -0,0 +1,36 @@ +# 说明书管理实施计划 + +目标：按 [系统设计](design.md) 交付说明书管理、skill、验证、PR 与 pi 更新。基线 `176bce3`，分支 `feat/cq/manuals`，工作区 `worktrees/home_server-manuals`。 + +## 执行分工与依赖 + +开发、验证与独立 Code Review 均使用 `gpt-5.6-sol`、`max` 子 Agent。先固定 API 契约，再并行实现后端、前端、skill；主 Agent 负责设计一致性、集成、发布和部署。共享文件由明确负责人修改，其他 Agent 只反馈需求。 + +| 阶段 | 文件责任范围 | 完成标准 | +| --- | --- | --- | +| 后端 | `api/manuals/`、`app/manuals/`、`domain/model/manual.go`、`domain/dal/manual*.go`、`domain/service/manuals/`、迁移、Go 配置/身份集成 | 三表与完整 API、格式/缩略图/配额/权限、真实可执行 Go 测试 | +| 前端 | `front_vue/src/views/Manual*.vue`、`api/manuals.cjs`、必要工具与导航/路由、前端测试 | 手机布局、队列追加/重试、混合资料、筛选、阅读、可见范围、应用 scope UI | +| Skill | `skills/home-server-web-share/`、对应 Python 测试 | 新 CLI 全部命令、dry-run、批量输入、幂等、回读核对；旧命令回归 | +| 集成 | 代理模板、README、部署说明、CI、迁移和部署工具（如必要） | 无静态旁路，52 MiB 上传规则，两入口兼容，旧配置可启动 | +| 验证 | 独立 Linux 构建/测试、浏览器与 HTTPS 集成、EXPLAIN | 记录执行环境、命令、结果及未覆盖边界 | +| 审查 | 完整 diff 与权限/存储/事务链路 | 独立审查，阻断问题全部修复并复验 | +| 发布 | commit、push、GitHub PR、pi release | PR 给用户合并；部署精确 SHA、健康检查、旧入口回归与可回滚记录 | + +## 实施状态 + +| 范围 | 已完成内容 | 验收状态 | +| --- | --- | --- | +| 后端 | 三表、多分类、字面名称检索、草稿/发布、混合资料、封面、幂等及 revision | Go 构建、完整主测试、DAL 和跨实例配额重复测试通过 | +| 权限与文件 | 精确 session 事务复核、应用 scope、各内容入口共用权限、格式/尺寸/配额、私有存储 | HTTP 权限矩阵、撤销/重复请求/删除竞态、EXIF/PDF 实际处理通过；故障注入边界见验证记录 | +| 前端 | 列表/编辑/阅读、移动布局、多次选图队列、局部重试、分类分页、保护路由 | Node 92/92、说明书浏览器 8/8、账号浏览器 6/6、生产构建通过；系统相册待实机验收 | +| Skill | 9 个说明书命令、批量输入、dry-run、写后回读、逐项稳定 ID 恢复 | 新旧 Python/HTTPS 33/33 与独立前向验证通过 | +| 集成与审查 | 导航/scopes、CI、双入口代理、部署及回滚说明、完整产物校验 | 独立静态审查无未处理 P1/P2；候选网关实际 Nginx 校验通过 | +| 发布 | 提交并推送、创建 PR、GitHub CI、按精确 SHA 更新 pi 与本地稳定 skill | 以 PR 和发布记录中的最终提交、CI 与现场验收为准；PR 保持打开由用户合并 | + +生产发布顺序：固定通过验证的 PR 提交 → 从提交构建匹配的前后端 → 校验整个 release → 备份数据库/配置/旧指针 → 增量建表和私有目录 → 检查实际 Nginx → 切换并验证新旧入口。发布前禁止使用中间验证快照的二进制。工作树在用户合并 PR 后释放。 + +## 证据要求 + +验证结果单独记录于 `verification.md`，必须区分已运行、通过、失败和未覆盖。禁止把 Go 编译替代权限集成测试、把静态源码匹配替代浏览器行为、把 `/ping` 替代新功能验证。 + +部署记录只包含版本号、哈希、路径和状态，不包含私有域名凭据、DSN、AK/SK 或 Cookie。原仓库未跟踪的 `agents.md` 不纳入提交。 diff --git a/docs/specs/manual-management/verification.md b/docs/specs/manual-management/verification.md new file mode 100644 index 0000000..da3cfab --- /dev/null +++ b/docs/specs/manual-management/verification.md @@ -0,0 +1,67 @@ +# 说明书管理验证记录 + +日期：2026-09-19。基线：`176bce3`。本记录是提交前验证快照；GitHub CI 与生产发布验收另以 PR 和部署记录中的精确提交 SHA 为准。 + +## 最终功能验证 + +| 对象 | 环境 / 命令 | 结果 | +| --- | --- | --- | +| Go 构建 | pi 隔离源码 v3，Go 1.21.13，`go build -p 2 ./...` 和主程序构建 | 通过，Linux x86_64 产物生成；发布时仍从最终提交重新构建 | +| 完整 Go CI 主测试 | 隔离 MariaDB 10.11 / Redis，复用 CI 包与环境变量清单 | 34 包、325 个顶层用例通过，20 个既有 opt-in 用例跳过，0 失败；169.52 秒 | +| DAL 回归 | 主测试结束后串行执行 `go test ./domain/dal -run 'Test(ReadCache|Manual)' -count=1` | 13 个顶层用例通过，0 失败；包含真实三表迁移、字面名称、多分类及未分类检索 | +| 跨实例容量配额 | 同 owner、不同精确会话、两个独立 Application 实例，定向 `-count=10` | 10 次通过、0 失败；每次恰好一项写入成功，另一项受配额限制 | +| 独立文件边界 | pi 实际 pdftoppm 与格式解码 | PDF 预览最长边 ≤480、损坏 PNG 拒绝且无最终目录、损坏 PDF 保留原件并标记不可预览、超限文件拒绝均通过 | +| 上传预留并发 | 32 路并发、10 轮 | 每轮 1 成功、31 限流；该项与数据库配额测试分别验证 | +| 前端行为 | macOS，`npm run test:frontend` | 92/92 通过 | +| 说明书浏览器 | 实际 Vue 页面与 Chrome，`npm run test:manuals-browser` | 8/8 通过，覆盖 320 px、多次选图追加、慢上传期间写互斥、部分失败重试、409、revision 类型及混合资料阅读 | +| 账号浏览器回归 | Chrome，`npm run test:account-center-browser` | 6/6 通过 | +| 前端构建与语法 | 变更文件 ESLint；`VUE_APP_API_BASE_URL=/ npm run build` | 通过；保留既有 bundle 大小及 caniuse-lite 警告 | +| Python 与 skill | 实际本机合成 HTTPS fixture，`test_manuals_skill`、`test_web_hosting_skill`、`test_home_server_api` | 33/33 通过：新 skill 5 项、旧 skill 15 项、客户端 13 项 | +| Skill 独立前向验证 | 合成 HTTPS、假应用凭据及临时 CA | 2 张图片、PDF、文本、URL、多分类、失败恢复、排序/封面/激活、筛选与旧命令兼容通过；未请求生产凭据 | +| 网关候选配置 | 实际 TLS 网关，备份原配置并对候选配置执行 `nginx -t` | 通过；提交前未替换生效配置或 reload | +| 文档与差异 | mmdc + Chrome、公开部署模板检查、`git diff --check` | 1 张 Mermaid 成功渲染，模板与格式检查通过 | + +完整主测试覆盖 DB 与 legacy/config 会话撤销、重复创建回放、上传中撤销、重复上传时撤销或删除、同说明书条目配额、跨说明书用户字节配额、分类二页游标、真实 pdftoppm PDF 最长边不超过 480 px，以及超过 4 KiB 的 APP1 EXIF 方向处理。测试中的真实权限矩阵同时检查原件、缩略图、HEAD、Range、所有者禁用和草稿/删除状态。 + +多个 pending 项恢复测试逐项保留原 request ID，断言无重复资料且最终顺序完整；不能把剩余多项重新组合成批次重编号。独立前向验证另外实际执行了 8 次 token exchange 和 16 次 HTTPS 业务调用。 + +### 独立复核 + +开发、验证与 Code Review 均由 `gpt-5.6-sol / max` 子 Agent 执行。参与 Go 修复的审查者没有把自审计作独立结论，修复由验证 Agent 重新读码和运行测试；前端及 skill 另有非实现者复核。最终静态审查没有未处理的 P1/P2。 + +| 风险 | 最终规则与证据 | +| --- | --- | +| 撤销会话后慢请求仍写入 | 写事务重新锁定并校验精确 session；JSON、文件、重复创建与旧配置身份均有撤销后拒绝用例 | +| RR 旧快照突破用户配额 | 先锁已知 owner，再锁 session，避免锁前非锁定读；追加测试曾实证两项均成功，修复后跨实例连续 10 次通过 | +| 文件失败与预览边界 | 未知提交只回查本次生成的 item ID；确定性撤权/删除错误不转换为旧条目成功；EXIF 大段和 PDF 严格尺寸回归通过 | +| 分类遗漏与上传竞态 | 分类 cursor 分页取全，异常游标拒绝；上传期间禁止其他写操作；独立前端静态检查及目标 Node 10/10 通过 | +| 发布包不一致 | 在 DDL/配置/切换前核对完整目录摘要，覆盖二进制、全部前端文件、SQL 和代理片段；嵌套 JS 变化可检出，symlink 拒绝 | + +## 查询执行计划 + +在隔离 MariaDB 10.11 写入 10,002 份说明书和 8,336 条分类关系后执行真实 EXPLAIN。公开列表使用 `idx_manuals_access_status_id`，估算候选 8,000 行；分类加名称子串先取得相同候选，再用分类表 PRIMARY 做 `eq_ref`。名称包含及字面 `%` 匹配均为残余过滤，没有名称索引命中。未分类查询会物化扫描 8,336 条分类主键记录。这是家庭规模的明确成本边界，不能宣称子串检索具有索引查找性能。 + +## 源码与日志证据 + +Go v3 使用明确允许的源文件清单同步；46/46 文件 SHA-256 匹配。生产配置和 DSN、node_modules、前端 dist、未列入清单的工作区文件不进入远端测试副本。API 与 DAL 用例共享合成数据库但严格分步串行运行，避免并发 DROP。 + +| 证据 | SHA-256 | +| --- | --- | +| v3 源文件摘要清单 | `50d786494a52d56e121763caf183809e529da5b69fe38ce29593f1ed10064e4c` | +| v3 全仓构建日志 | `505d5f7502ba6c84d50d993c08cc3a931b26c7cec2a222d96ddb4b4138befac8` | +| v3 完整主测试日志 | `98525fde3d7a44c91b65cd9853a1d8e11a4d190b13e38c1de3252e0f8f2ea1c7` | +| v3 DAL 日志 | `a14cf86e16df90367f8209e20d14ba6c7bbd6cfa5dac8717deff505197b1d7e6` | +| v3 跨实例配额 10 次日志 | `0f332087e36431e15b96ddca168cbaa8baf1ad42a1d956f18ac594fa1f4f0a87` | +| v3 独立文件与预留边界 | `53bbd78d96a2710146d45760b8a8f54b263c942c8eb1c4eea2936d0874589eca` | +| v3 EXPLAIN 日志 | `2af98119c338b785a9dab9664d3544e30fdef9225caa6ad0d7ff140f2c5c79b6` | + +Go 测试限制包并发 2、CPU 150%、内存 3 GiB；数据库和 Redis 仅绑定 loopback 并设置资源上限。主机系统 Go 和生产服务不参与验证环境改造。 + +## 基线与验证边界 + +修改前精确 `git archive 176bce3` 的关联 Go 基线为 16 包、208 PASS、20 SKIP、0 FAIL；Node 82/82、Python 客户端 13/13、旧 skill 15/15、原代理静态测试 5/5 通过。这些基线只用于识别回归，不计作新功能覆盖。 + +- 20 个跳过项是原 CI 未启用的显式 opt-in 数据库或 live fixture，不能算作已覆盖。 +- iOS/Android 系统相册选择器尚未实机验证；Chrome 窄屏与文件选择测试不替代系统相册验收。HEIC/HEIF 需要先转换为 JPEG/PNG。 +- 未执行数据库断连、进程崩溃或 commit 结果未知的故障注入；相关补偿路径已独立代码审查，一般幂等和撤权用例不等同故障注入证明。 +- 生产发布需固定 PR 提交、备份、增量迁移、Nginx 校验与新旧入口冒烟；提交前的隔离测试不替代该记录。 diff --git a/domain/dal/manual.go b/domain/dal/manual.go new file mode 100644 index 0000000..b93ff8f --- /dev/null +++ b/domain/dal/manual.go @@ -0,0 +1,326 @@ +package dal + +import ( + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/mcoder2014/home_server/domain/model" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +const ( + ManualTable = "manuals" + ManualCategoryTable = "manual_categories" + ManualItemTable = "manual_items" +) + +var manualColumns = []string{"id", "owner_user_id", "name", "description", "access_mode", "status", "cover_item_id", "revision", "client_request_id", "request_hash", "create_time", "update_time"} +var manualItemColumns = []string{"id", "manual_id", "kind", "title", "text", "url", "storage_key", "thumbnail_key", "content_type", "original_name", "size_bytes", "sha256", "position", "preview_status", "client_request_id", "request_hash", "deleted_at", "create_time"} + +type ManualListFilter struct { + ViewerUserID int64 + Mine bool + Cursor int64 + Limit int + Query string + Category string + CategorySet bool + RequireActiveOwner bool + ActiveOwnerIDs []int64 +} + +type ManualItemCount struct { + ManualID int64 `gorm:"column:manual_id"` + Count int `gorm:"column:item_count"` +} + +type ManualItemPreview struct { + ID int64 `gorm:"column:id"` + ManualID int64 `gorm:"column:manual_id"` + Kind model.ManualItemKind `gorm:"column:kind"` + Title string `gorm:"column:title"` + TextExcerpt string `gorm:"column:text_excerpt"` + URL string `gorm:"column:url"` + ThumbnailKey *string `gorm:"column:thumbnail_key"` + PreviewStatus model.ManualPreviewStatus `gorm:"column:preview_status"` + Position int `gorm:"column:position"` +} + +func InsertManual(tx *gorm.DB, manual *model.Manual) error { + return tx.Table(ManualTable).Create(manual).Error +} + +func InsertManualCategories(tx *gorm.DB, manualID int64, categories []string) error { + if len(categories) == 0 { + return nil + } + rows := make([]*model.ManualCategory, 0, len(categories)) + for _, category := range categories { + rows = append(rows, &model.ManualCategory{ManualID: manualID, Category: category}) + } + return tx.Table(ManualCategoryTable).Select("manual_id", "category").Create(&rows).Error +} + +func ReplaceManualCategories(tx *gorm.DB, manualID int64, categories []string) error { + if err := tx.Table(ManualCategoryTable).Where("manual_id = ?", manualID).Delete(&model.ManualCategory{}).Error; err != nil { + return err + } + return InsertManualCategories(tx, manualID, categories) +} + +func FindManual(database *gorm.DB, id int64, lock bool) (*model.Manual, error) { + query := database.Table(ManualTable).Select(manualColumns).Where("id = ?", id) + if lock { + query = query.Clauses(clause.Locking{Strength: "UPDATE"}) + } + var manual model.Manual + err := query.Take(&manual).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return &manual, err +} + +func FindManualByRequestID(database *gorm.DB, ownerID int64, requestID string, lock bool) (*model.Manual, error) { + query := database.Table(ManualTable).Select(manualColumns).Where("owner_user_id = ? AND client_request_id = ?", ownerID, requestID) + if lock { + query = query.Clauses(clause.Locking{Strength: "UPDATE"}) + } + var manual model.Manual + err := query.Take(&manual).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return &manual, err +} + +func ListManuals(database *gorm.DB, filter ManualListFilter) ([]*model.Manual, error) { + query := visibleManualQuery(database, filter).Select(prefixedColumns("m", manualColumns)) + if filter.Cursor > 0 { + query = query.Where("m.id < ?", filter.Cursor) + } + if filter.CategorySet { + if filter.Category == "" { + query = query.Where("NOT EXISTS (SELECT 1 FROM " + ManualCategoryTable + " AS filtered_category WHERE filtered_category.manual_id = m.id)") + } else { + query = query.Where("EXISTS (SELECT 1 FROM "+ManualCategoryTable+" AS filtered_category WHERE filtered_category.manual_id = m.id AND filtered_category.category = ?)", filter.Category) + } + } + if filter.Query != "" { + query = query.Where("m.name LIKE ? ESCAPE '!'", ManualNameContainsPattern(filter.Query)) + } + var manuals []*model.Manual + err := query.Order("m.id DESC").Limit(filter.Limit).Find(&manuals).Error + return manuals, err +} + +func ListManualCategories(database *gorm.DB, filter ManualListFilter, limit int) ([]string, error) { + return ListManualCategoriesAfter(database, filter, "", limit) +} + +func ListManualCategoriesAfter(database *gorm.DB, filter ManualListFilter, cursor string, limit int) ([]string, error) { + var categories []string + query := visibleManualQuery(database, filter).Joins("JOIN " + ManualCategoryTable + " AS mc ON mc.manual_id = m.id") + if cursor != "" { + query = query.Where("mc.category > ?", cursor) + } + err := query.Distinct("mc.category").Order("mc.category ASC").Limit(limit).Pluck("mc.category", &categories).Error + return categories, err +} + +func ListManualCategoriesByManualIDs(database *gorm.DB, manualIDs []int64) ([]*model.ManualCategory, error) { + if len(manualIDs) == 0 { + return []*model.ManualCategory{}, nil + } + var categories []*model.ManualCategory + err := database.Table(ManualCategoryTable).Select("manual_id", "category").Where("manual_id IN ?", manualIDs).Order("manual_id ASC, category ASC").Find(&categories).Error + return categories, err +} + +func visibleManualQuery(database *gorm.DB, filter ManualListFilter) *gorm.DB { + query := database.Table(ManualTable + " AS m") + if filter.RequireActiveOwner { + query = query.Joins("JOIN "+AccountTable+" AS owner_account ON owner_account.id = m.owner_user_id AND owner_account.status = ? AND owner_account.must_change_password = ?", model.AccountActive, false) + } else if filter.ActiveOwnerIDs != nil { + query = query.Where("m.owner_user_id IN ?", filter.ActiveOwnerIDs) + } + if filter.Mine { + return query.Where("m.owner_user_id = ? AND m.status <> ?", filter.ViewerUserID, model.ManualStatusDeleted) + } + if filter.ViewerUserID > 0 { + return query.Where("(m.owner_user_id = ? AND m.status <> ?) OR (m.status = ? AND m.access_mode IN ?)", filter.ViewerUserID, model.ManualStatusDeleted, model.ManualStatusActive, []model.ManualAccess{model.ManualAccessAuthenticated, model.ManualAccessPublic}) + } + return query.Where("m.status = ? AND m.access_mode = ?", model.ManualStatusActive, model.ManualAccessPublic) +} + +func ManualNameContainsPattern(value string) string { + replacer := strings.NewReplacer("!", "!!", "%", "!%", "_", "!_") + return "%" + replacer.Replace(value) + "%" +} + +func ListManualItemCounts(database *gorm.DB, manualIDs []int64) ([]ManualItemCount, error) { + if len(manualIDs) == 0 { + return []ManualItemCount{}, nil + } + var counts []ManualItemCount + err := database.Table(ManualItemTable).Select("manual_id, COUNT(*) AS item_count").Where("manual_id IN ? AND deleted_at IS NULL", manualIDs).Group("manual_id").Find(&counts).Error + return counts, err +} + +func ListManualItemsByIDs(database *gorm.DB, itemIDs []int64) ([]*model.ManualItem, error) { + if len(itemIDs) == 0 { + return []*model.ManualItem{}, nil + } + var items []*model.ManualItem + err := database.Table(ManualItemTable).Select(manualItemColumns).Where("id IN ? AND deleted_at IS NULL", itemIDs).Find(&items).Error + return items, err +} + +func ListManualItemPreviewsByIDs(database *gorm.DB, itemIDs []int64) ([]*ManualItemPreview, error) { + if len(itemIDs) == 0 { + return []*ManualItemPreview{}, nil + } + var items []*ManualItemPreview + err := database.Table(ManualItemTable).Select("id", "manual_id", "kind", "title", "LEFT(text, 160) AS text_excerpt", "url", "thumbnail_key", "preview_status", "position").Where("id IN ? AND deleted_at IS NULL", itemIDs).Find(&items).Error + return items, err +} + +// ListAutomaticManualCovers returns one bounded preview per manual. The window +// ranks ready image/PDF thumbnails first, then preserves the user's item order. +func ListAutomaticManualCovers(database *gorm.DB, manualIDs []int64) ([]*ManualItemPreview, error) { + if len(manualIDs) == 0 { + return []*ManualItemPreview{}, nil + } + selectSQL := "id, manual_id, kind, title, LEFT(text, 160) AS text_excerpt, url, thumbnail_key, preview_status, position, " + + "ROW_NUMBER() OVER (PARTITION BY manual_id ORDER BY CASE WHEN kind IN (?, ?) AND preview_status = ? THEN 0 ELSE 1 END, position ASC, id ASC) AS cover_rank" + ranked := database.Table(ManualItemTable).Select(selectSQL, model.ManualItemImage, model.ManualItemPDF, model.ManualPreviewReady).Where("manual_id IN ? AND deleted_at IS NULL", manualIDs) + var items []*ManualItemPreview + err := database.Table("(?) AS ranked_manual_items", ranked).Select("id", "manual_id", "kind", "title", "text_excerpt", "url", "thumbnail_key", "preview_status", "position").Where("cover_rank = 1").Find(&items).Error + return items, err +} + +func ListManualItems(database *gorm.DB, manualID int64, lock bool) ([]*model.ManualItem, error) { + query := database.Table(ManualItemTable).Select(manualItemColumns).Where("manual_id = ? AND deleted_at IS NULL", manualID).Order("position ASC, id ASC") + if lock { + query = query.Clauses(clause.Locking{Strength: "UPDATE"}) + } + var items []*model.ManualItem + err := query.Find(&items).Error + return items, err +} + +func FindManualItem(database *gorm.DB, manualID, itemID int64, lock bool) (*model.ManualItem, error) { + query := database.Table(ManualItemTable).Select(manualItemColumns).Where("manual_id = ? AND id = ? AND deleted_at IS NULL", manualID, itemID) + if lock { + query = query.Clauses(clause.Locking{Strength: "UPDATE"}) + } + var item model.ManualItem + err := query.Take(&item).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return &item, err +} + +func FindManualItemByRequestID(database *gorm.DB, manualID int64, requestID string, lock bool) (*model.ManualItem, error) { + query := database.Table(ManualItemTable).Select(manualItemColumns).Where("manual_id = ? AND client_request_id = ?", manualID, requestID) + if lock { + query = query.Clauses(clause.Locking{Strength: "UPDATE"}) + } + var item model.ManualItem + err := query.Take(&item).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return &item, err +} + +func InsertManualItem(tx *gorm.DB, item *model.ManualItem) error { + return tx.Table(ManualItemTable).Create(item).Error +} + +func CountUserManuals(tx *gorm.DB, ownerID int64) (int64, error) { + var count int64 + err := tx.Table(ManualTable).Where("owner_user_id = ? AND status <> ?", ownerID, model.ManualStatusDeleted).Count(&count).Error + return count, err +} + +func CountActiveManualItems(tx *gorm.DB, manualID int64) (int64, error) { + var count int64 + err := tx.Table(ManualItemTable).Where("manual_id = ? AND deleted_at IS NULL", manualID).Count(&count).Error + return count, err +} + +func SumManualItemBytes(tx *gorm.DB, manualID int64) (int64, error) { + return sumItemBytes(tx.Table(ManualItemTable).Where("manual_id = ?", manualID), "size_bytes") +} + +func SumUserManualItemBytes(tx *gorm.DB, ownerID int64) (int64, error) { + query := tx.Table(ManualItemTable+" AS i").Joins("JOIN "+ManualTable+" AS m ON m.id = i.manual_id").Where("m.owner_user_id = ?", ownerID) + return sumItemBytes(query, "i.size_bytes") +} + +func sumItemBytes(query *gorm.DB, column string) (int64, error) { + var result struct { + Bytes int64 `gorm:"column:bytes"` + InvalidRows int64 `gorm:"column:invalid_rows"` + } + err := query.Select("COALESCE(SUM(" + column + "),0) AS bytes, COALESCE(SUM(CASE WHEN " + column + " < 0 THEN 1 ELSE 0 END),0) AS invalid_rows").Scan(&result).Error + if err != nil { + return 0, err + } + if result.Bytes < 0 || result.InvalidRows != 0 { + return 0, fmt.Errorf("invalid recorded manual item size") + } + return result.Bytes, nil +} + +func UpdateManualFieldsCAS(tx *gorm.DB, ownerID, manualID, revision int64, fields map[string]interface{}) (bool, error) { + result := tx.Table(ManualTable).Where("owner_user_id = ? AND id = ? AND revision = ?", ownerID, manualID, revision).Updates(fields) + return result.RowsAffected == 1, result.Error +} + +func UpdateManualFields(tx *gorm.DB, manualID int64, fields map[string]interface{}) error { + return tx.Table(ManualTable).Where("id = ?", manualID).Updates(fields).Error +} + +func UpdateManualItemPositions(tx *gorm.DB, manualID int64, positions map[int64]int) (bool, error) { + if len(positions) == 0 { + return true, nil + } + ids := make([]int64, 0, len(positions)) + for id := range positions { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + query := "UPDATE " + ManualItemTable + " SET position = CASE id" + arguments := make([]interface{}, 0, len(ids)*2+2) + for _, id := range ids { + query += " WHEN ? THEN ?" + arguments = append(arguments, id, positions[id]) + } + query += " ELSE position END WHERE manual_id = ? AND deleted_at IS NULL AND id IN ?" + arguments = append(arguments, manualID, ids) + result := tx.Exec(query, arguments...) + return result.RowsAffected == int64(len(ids)), result.Error +} + +func DeleteManualItem(tx *gorm.DB, manualID, itemID int64, deletedAt time.Time) error { + return tx.Table(ManualItemTable).Where("manual_id = ? AND id = ? AND deleted_at IS NULL", manualID, itemID).Updates(map[string]interface{}{"deleted_at": deletedAt}).Error +} + +func CompactManualItemPositions(tx *gorm.DB, manualID int64, removedPosition int) error { + return tx.Table(ManualItemTable).Where("manual_id = ? AND deleted_at IS NULL AND position > ?", manualID, removedPosition).Update("position", gorm.Expr("position - 1")).Error +} + +func prefixedColumns(alias string, columns []string) []string { + result := make([]string, 0, len(columns)) + for _, column := range columns { + result = append(result, alias+"."+column) + } + return result +} diff --git a/domain/dal/manual_integration_test.go b/domain/dal/manual_integration_test.go new file mode 100644 index 0000000..c73cc50 --- /dev/null +++ b/domain/dal/manual_integration_test.go @@ -0,0 +1,102 @@ +package dal + +import ( + "os" + "strings" + "testing" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/mcoder2014/home_server/domain/model" + "github.com/stretchr/testify/require" + gormmysql "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func manualTestDatabase(t *testing.T) *gorm.DB { + t.Helper() + dsn := os.Getenv("MANUALS_TEST_DSN") + if dsn == "" { + t.Skip("MANUALS_TEST_DSN is required for MariaDB manual tests") + } + parsed, err := mysql.ParseDSN(dsn) + require.NoError(t, err) + require.True(t, strings.HasPrefix(parsed.DBName, "home_server_manuals_test_"), "refuse non-fixture database") + database, err := gorm.Open(gormmysql.Open(dsn), &gorm.Config{Logger: logger.Discard}) + require.NoError(t, err) + connection, err := database.DB() + require.NoError(t, err) + t.Cleanup(func() { + _ = connection.Close() + }) + var actual string + require.NoError(t, database.Raw("SELECT DATABASE()").Scan(&actual).Error) + require.Equal(t, parsed.DBName, actual) + for _, table := range []string{ManualCategoryTable, ManualItemTable, ManualTable} { + require.NoError(t, database.Exec("DROP TABLE IF EXISTS "+table).Error) + } + ddl, err := os.ReadFile("migrations/20260919_manuals.sql") + require.NoError(t, err) + for _, statement := range strings.Split(string(ddl), ";") { + if strings.TrimSpace(statement) != "" { + require.NoError(t, database.Exec(statement).Error) + } + } + return database +} + +func TestManualMigrationAndLiteralSearchCategories(t *testing.T) { + database := manualTestDatabase(t) + now := time.Now() + manuals := []*model.Manual{ + {ID: 101, OwnerUserID: 1, Name: "维修100%指南", AccessMode: model.ManualAccessPublic, Status: model.ManualStatusActive, Revision: 1, ClientRequestID: "CaseKey", RequestHash: strings.Repeat("a", 64), CreateTime: now, UpdateTime: now}, + {ID: 102, OwnerUserID: 1, Name: "维修100x指南", AccessMode: model.ManualAccessPublic, Status: model.ManualStatusActive, Revision: 1, ClientRequestID: "request-102", RequestHash: strings.Repeat("b", 64), CreateTime: now, UpdateTime: now}, + {ID: 103, OwnerUserID: 1, Name: "A_B手册", AccessMode: model.ManualAccessPublic, Status: model.ManualStatusActive, Revision: 1, ClientRequestID: "request-103", RequestHash: strings.Repeat("c", 64), CreateTime: now, UpdateTime: now}, + {ID: 104, OwnerUserID: 1, Name: "AxB手册", AccessMode: model.ManualAccessPublic, Status: model.ManualStatusActive, Revision: 1, ClientRequestID: "request-104", RequestHash: strings.Repeat("d", 64), CreateTime: now, UpdateTime: now}, + {ID: 105, OwnerUserID: 1, Name: "A!B手册", AccessMode: model.ManualAccessPublic, Status: model.ManualStatusActive, Revision: 1, ClientRequestID: "request-105", RequestHash: strings.Repeat("e", 64), CreateTime: now, UpdateTime: now}, + {ID: 106, OwnerUserID: 1, Name: "冰箱ABC手册", AccessMode: model.ManualAccessPublic, Status: model.ManualStatusActive, Revision: 1, ClientRequestID: "request-106", RequestHash: strings.Repeat("f", 64), CreateTime: now, UpdateTime: now}, + {ID: 107, OwnerUserID: 1, Name: "大小写请求", AccessMode: model.ManualAccessPublic, Status: model.ManualStatusActive, Revision: 1, ClientRequestID: "casekey", RequestHash: strings.Repeat("0", 64), CreateTime: now, UpdateTime: now}, + } + for _, manual := range manuals { + require.NoError(t, InsertManual(database, manual)) + } + require.NoError(t, InsertManualCategories(database, 101, []string{"Kitchen", "厨房"})) + require.NoError(t, InsertManualCategories(database, 102, []string{"kitchen"})) + require.NoError(t, InsertManualCategories(database, 103, []string{"厨房"})) + + for query, expected := range map[string][]int64{ + "%": {101}, + "_": {103}, + "!": {105}, + "冰箱abc": {106}, + } { + rows, err := ListManuals(database, ManualListFilter{Query: query, Limit: 100}) + require.NoError(t, err) + require.Equal(t, expected, manualIDs(rows), query) + } + rows, err := ListManuals(database, ManualListFilter{CategorySet: true, Category: "", Limit: 100}) + require.NoError(t, err) + require.ElementsMatch(t, []int64{104, 105, 106, 107}, manualIDs(rows)) + rows, err = ListManuals(database, ManualListFilter{CategorySet: true, Category: "Kitchen", Limit: 100}) + require.NoError(t, err) + require.Equal(t, []int64{101}, manualIDs(rows)) + rows, err = ListManuals(database, ManualListFilter{CategorySet: true, Category: "kitchen", Limit: 100}) + require.NoError(t, err) + require.Equal(t, []int64{102}, manualIDs(rows)) + + categories, err := ListManualCategories(database, ManualListFilter{}, 100) + require.NoError(t, err) + require.Equal(t, []string{"Kitchen", "kitchen", "厨房"}, categories) + categoryRows, err := ListManualCategoriesByManualIDs(database, []int64{101, 104}) + require.NoError(t, err) + require.Equal(t, []*model.ManualCategory{{ManualID: 101, Category: "Kitchen"}, {ManualID: 101, Category: "厨房"}}, categoryRows) +} + +func manualIDs(rows []*model.Manual) []int64 { + ids := make([]int64, 0, len(rows)) + for _, row := range rows { + ids = append(ids, row.ID) + } + return ids +} diff --git a/domain/dal/manual_test.go b/domain/dal/manual_test.go new file mode 100644 index 0000000..766d00b --- /dev/null +++ b/domain/dal/manual_test.go @@ -0,0 +1,12 @@ +package dal + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestManualNamePatternTreatsWildcardsLiterally(t *testing.T) { + require.Equal(t, `%100!%!_done%`, ManualNameContainsPattern(`100%_done`)) + require.Equal(t, `%a!!b%`, ManualNameContainsPattern(`a!b`)) +} diff --git a/domain/dal/migrations/20260919_manuals.sql b/domain/dal/migrations/20260919_manuals.sql new file mode 100644 index 0000000..377bb52 --- /dev/null +++ b/domain/dal/migrations/20260919_manuals.sql @@ -0,0 +1,53 @@ +CREATE TABLE manuals +( + id BIGINT NOT NULL PRIMARY KEY COMMENT 'manual id generated by the service', + owner_user_id BIGINT NOT NULL COMMENT 'creator and sole manager', + name VARCHAR(120) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'display name, at most 120 Unicode characters', + description TEXT NOT NULL COMMENT 'plain text description', + access_mode TINYINT UNSIGNED NOT NULL COMMENT '1 owner, 2 authenticated, 3 public', + status TINYINT UNSIGNED NOT NULL COMMENT '1 draft, 2 active, 3 deleted', + cover_item_id BIGINT NULL COMMENT 'explicitly selected non-deleted item, NULL selects the automatic cover', + revision BIGINT NOT NULL DEFAULT 1 COMMENT 'optimistic lock revision', + client_request_id VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT 'create idempotency key', + request_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT 'canonical create input SHA-256', + create_time DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + update_time DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + UNIQUE KEY uk_manuals_owner_request (owner_user_id, client_request_id), + KEY idx_manuals_owner_status_id (owner_user_id, status, id), + KEY idx_manuals_access_status_id (access_mode, status, id) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT 'private and shared equipment manuals'; + +CREATE TABLE manual_categories +( + manual_id BIGINT NOT NULL, + category VARCHAR(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT 'trimmed owner-defined category', + create_time DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (manual_id, category), + KEY idx_manual_categories_category_manual (category, manual_id) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT 'manual to category relationships'; + +CREATE TABLE manual_items +( + id BIGINT NOT NULL PRIMARY KEY COMMENT 'manual item id generated by the service', + manual_id BIGINT NOT NULL, + kind TINYINT UNSIGNED NOT NULL COMMENT '1 image, 2 pdf, 3 text, 4 url', + title VARCHAR(200) NOT NULL, + text MEDIUMTEXT NOT NULL COMMENT 'plain text item body', + url VARCHAR(2048) NOT NULL COMMENT 'validated http or https URL', + storage_key VARCHAR(512) NULL COMMENT 'service-generated original path relative to manuals root', + thumbnail_key VARCHAR(512) NULL COMMENT 'service-generated JPEG thumbnail path', + content_type VARCHAR(128) NOT NULL, + original_name VARCHAR(512) NOT NULL, + size_bytes BIGINT NOT NULL DEFAULT 0 COMMENT 'original bytes, retained rows keep counting toward quota', + sha256 CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + position INT UNSIGNED NOT NULL, + preview_status TINYINT UNSIGNED NOT NULL COMMENT '1 none, 2 ready, 3 unavailable', + client_request_id VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT 'append idempotency key', + request_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT 'canonical item input SHA-256', + deleted_at DATETIME(6) NULL COMMENT 'logical deletion, original remains allocated', + create_time DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + UNIQUE KEY uk_manual_items_request (manual_id, client_request_id), + UNIQUE KEY uk_manual_items_storage (storage_key), + KEY idx_manual_items_manual_active_position (manual_id, deleted_at, position, id), + KEY idx_manual_items_manual_id (manual_id, id) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT 'ordered manual image, pdf, text and URL items'; diff --git a/domain/dal/token.go b/domain/dal/token.go index ee3c6fc..5678098 100644 --- a/domain/dal/token.go +++ b/domain/dal/token.go @@ -7,6 +7,7 @@ import ( "github.com/mcoder2014/home_server/domain/model" myErrors "github.com/mcoder2014/home_server/errors" "gorm.io/gorm" + "gorm.io/gorm/clause" "gorm.io/gorm/logger" ) @@ -41,3 +42,21 @@ func QueryByToken(token string) (*model.UserToken, error) { } return &res, nil } + +func QueryByTokenTx(tx *gorm.DB, token string, lock bool) (*model.UserToken, error) { + query := tx.Session(&gorm.Session{Logger: logger.Discard}).Table(TableUserToken). + Select("id, user_id, token, is_expired, create_time, update_time, expire_time"). + Where("token = ?", token) + if lock { + query = query.Clauses(clause.Locking{Strength: "UPDATE"}) + } + var result model.UserToken + err := query.Take(&result).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, myErrors.Wrap(err, myErrors.ErrorCodeDbError) + } + return &result, nil +} diff --git a/domain/model/manual.go b/domain/model/manual.go new file mode 100644 index 0000000..8d6a00c --- /dev/null +++ b/domain/model/manual.go @@ -0,0 +1,173 @@ +package model + +import "time" + +type ManualAccess uint8 + +const ( + ManualAccessOwner ManualAccess = 1 + ManualAccessAuthenticated ManualAccess = 2 + ManualAccessPublic ManualAccess = 3 +) + +func (value ManualAccess) String() string { + switch value { + case ManualAccessOwner: + return "owner" + case ManualAccessAuthenticated: + return "authenticated" + case ManualAccessPublic: + return "public" + default: + return "" + } +} + +func ParseManualAccess(value string) (ManualAccess, bool) { + switch value { + case "owner": + return ManualAccessOwner, true + case "authenticated": + return ManualAccessAuthenticated, true + case "public": + return ManualAccessPublic, true + default: + return 0, false + } +} + +type ManualStatus uint8 + +const ( + ManualStatusDraft ManualStatus = 1 + ManualStatusActive ManualStatus = 2 + ManualStatusDeleted ManualStatus = 3 +) + +func (value ManualStatus) String() string { + switch value { + case ManualStatusDraft: + return "draft" + case ManualStatusActive: + return "active" + case ManualStatusDeleted: + return "deleted" + default: + return "" + } +} + +func ParseManualStatus(value string) (ManualStatus, bool) { + switch value { + case "draft": + return ManualStatusDraft, true + case "active": + return ManualStatusActive, true + case "deleted": + return ManualStatusDeleted, true + default: + return 0, false + } +} + +type ManualItemKind uint8 + +const ( + ManualItemImage ManualItemKind = 1 + ManualItemPDF ManualItemKind = 2 + ManualItemText ManualItemKind = 3 + ManualItemURL ManualItemKind = 4 +) + +func (value ManualItemKind) String() string { + switch value { + case ManualItemImage: + return "image" + case ManualItemPDF: + return "pdf" + case ManualItemText: + return "text" + case ManualItemURL: + return "url" + default: + return "" + } +} + +func ParseManualItemKind(value string) (ManualItemKind, bool) { + switch value { + case "image": + return ManualItemImage, true + case "pdf": + return ManualItemPDF, true + case "text": + return ManualItemText, true + case "url": + return ManualItemURL, true + default: + return 0, false + } +} + +type ManualPreviewStatus uint8 + +const ( + ManualPreviewNone ManualPreviewStatus = 1 + ManualPreviewReady ManualPreviewStatus = 2 + ManualPreviewUnavailable ManualPreviewStatus = 3 +) + +func (value ManualPreviewStatus) String() string { + switch value { + case ManualPreviewNone: + return "none" + case ManualPreviewReady: + return "ready" + case ManualPreviewUnavailable: + return "unavailable" + default: + return "" + } +} + +type Manual struct { + ID int64 `gorm:"column:id"` + OwnerUserID int64 `gorm:"column:owner_user_id"` + Name string `gorm:"column:name"` + Description string `gorm:"column:description"` + AccessMode ManualAccess `gorm:"column:access_mode"` + Status ManualStatus `gorm:"column:status"` + CoverItemID *int64 `gorm:"column:cover_item_id"` + Revision int64 `gorm:"column:revision"` + ClientRequestID string `gorm:"column:client_request_id"` + RequestHash string `gorm:"column:request_hash"` + CreateTime time.Time `gorm:"column:create_time"` + UpdateTime time.Time `gorm:"column:update_time"` +} + +type ManualCategory struct { + ManualID int64 `gorm:"column:manual_id"` + Category string `gorm:"column:category"` + CreateTime time.Time `gorm:"column:create_time"` +} + +type ManualItem struct { + ID int64 `gorm:"column:id"` + ManualID int64 `gorm:"column:manual_id"` + Kind ManualItemKind `gorm:"column:kind"` + Title string `gorm:"column:title"` + Text string `gorm:"column:text"` + URL string `gorm:"column:url"` + StorageKey *string `gorm:"column:storage_key"` + ThumbnailKey *string `gorm:"column:thumbnail_key"` + ContentType string `gorm:"column:content_type"` + OriginalName string `gorm:"column:original_name"` + SizeBytes int64 `gorm:"column:size_bytes"` + SHA256 string `gorm:"column:sha256"` + Position int `gorm:"column:position"` + PreviewStatus ManualPreviewStatus `gorm:"column:preview_status"` + ClientRequestID string `gorm:"column:client_request_id"` + RequestHash string `gorm:"column:request_hash"` + DeletedAt *time.Time `gorm:"column:deleted_at"` + CreateTime time.Time `gorm:"column:create_time"` +} diff --git a/domain/service/accounts/manuals_test.go b/domain/service/accounts/manuals_test.go new file mode 100644 index 0000000..54b2e89 --- /dev/null +++ b/domain/service/accounts/manuals_test.go @@ -0,0 +1,27 @@ +package accounts + +import ( + "context" + "testing" + + "github.com/mcoder2014/home_server/config" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/stretchr/testify/require" +) + +func TestManualsModuleUsesStaticConfigurationWithoutDatabaseNamespace(t *testing.T) { + before := config.Global() + t.Cleanup(func() { + config.SetGlobalConfig(before) + }) + config.SetGlobalConfig(config.Config{ConfigSource: "database", Manuals: config.ManualsConfig{Enabled: true}}) + + enabled, err := ModuleEnabled(context.Background(), "manuals") + require.NoError(t, err) + require.True(t, enabled) + enabled, err = EnabledTx(nil, "manuals", "enabled", true) + require.NoError(t, err) + require.True(t, enabled) + _, err = EnabledTx(nil, "manuals", "unknown", false) + require.ErrorIs(t, err, apperrors.ErrDependency) +} diff --git a/domain/service/accounts/service.go b/domain/service/accounts/service.go index 067dc70..eaff635 100644 --- a/domain/service/accounts/service.go +++ b/domain/service/accounts/service.go @@ -311,6 +311,12 @@ func UserIdentity(user *model.UserAccount) *model.UserIdentity { // EnabledTx 从配置文件或事务内的配置行读取布尔开关；数据库配置需通过版本、摘要和字段校验。 func EnabledTx(tx *gorm.DB, namespace, key string, lock bool) (bool, error) { conf := config.Global() + if namespace == "manuals" { + if key != "enabled" { + return false, apperrors.ErrDependency + } + return conf.Manuals.Enabled, nil + } if conf.ConfigSource != "database" { value, ok := config.DefaultRuntimeValues(conf)[namespace][key].(bool) if !ok { @@ -342,6 +348,9 @@ func EnabledTx(tx *gorm.DB, namespace, key string, lock bool) (bool, error) { } func ModuleEnabled(ctx context.Context, namespace string) (bool, error) { + if namespace == "manuals" { + return config.Global().Manuals.Enabled, nil + } key := "enabled" if namespace == "auth" { key = "applications_enabled" diff --git a/domain/service/accounts/sessions.go b/domain/service/accounts/sessions.go index 3030f92..2e188a9 100644 --- a/domain/service/accounts/sessions.go +++ b/domain/service/accounts/sessions.go @@ -21,6 +21,7 @@ import ( "github.com/mileusna/useragent" "gorm.io/gorm" "gorm.io/gorm/clause" + "gorm.io/gorm/logger" ) const SessionWarningThreshold = 10 @@ -158,6 +159,45 @@ func actingSessionTx(tx *gorm.DB, token string, lock, admin bool) (*model.UserAc return user, session, now, nil } +// RequireUserSessionTx revalidates and locks the exact acting user session for +// a downstream write transaction. The token user must match the resource owner. +func RequireUserSessionTx(tx *gorm.DB, token string, userID int64) (*model.UserAccount, error) { + if tx == nil { + return nil, apperrors.ErrDependency + } + if userID <= 0 || len(token) < 16 || len(token) > 256 { + return nil, apperrors.ErrUnauthorized + } + digest := sha256.Sum256([]byte(token)) + // The resource owner is already known. Lock it before any consistent read so + // quota queries after this check observe writes committed by the prior owner. + user, err := dal.QueryAccount(tx, userID, true) + if err != nil { + return nil, err + } + if user == nil { + return nil, apperrors.ErrUnauthorized + } + var session model.AccountSession + err = tx.Session(&gorm.Session{Logger: logger.Discard}).Table(dal.TableUserToken). + Select(dal.AccountSessionColumns). + Where("user_id = ? AND token_digest = ?", userID, digest[:]). + Clauses(clause.Locking{Strength: "UPDATE"}).Take(&session).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, apperrors.ErrUnauthorized + } + if err != nil { + return nil, err + } + if !sessionUsableAt(user, &session, time.Now()) { + return nil, apperrors.ErrUnauthorized + } + if user.MustChangePassword || session.Purpose != model.SessionUser { + return nil, apperrors.ErrForbidden + } + return user, nil +} + type SessionFilter struct { Cursor string Limit int diff --git a/domain/service/applications/service.go b/domain/service/applications/service.go index 6c55729..c8fb7ef 100644 --- a/domain/service/applications/service.go +++ b/domain/service/applications/service.go @@ -34,6 +34,8 @@ const ( ScopeLibraryWrite = "library:write" ScopeWebDAVRead = "webdav:read" ScopeWebDAVWrite = "webdav:write" + ScopeManualsRead = "manuals:read" + ScopeManualsWrite = "manuals:write" StatusEnabledName = "enabled" StatusDisabledName = "disabled" @@ -54,6 +56,8 @@ var orderedScopes = []string{ ScopeLibraryWrite, ScopeWebDAVRead, ScopeWebDAVWrite, + ScopeManualsRead, + ScopeManualsWrite, } var readForWrite = map[string]string{ @@ -61,6 +65,7 @@ var readForWrite = map[string]string{ ScopeWebProjectsWrite: ScopeWebProjectsRead, ScopeLibraryWrite: ScopeLibraryRead, ScopeWebDAVWrite: ScopeWebDAVRead, + ScopeManualsWrite: ScopeManualsRead, } type Repository interface { diff --git a/domain/service/applications/service_test.go b/domain/service/applications/service_test.go index 39fff4f..13b0ac3 100644 --- a/domain/service/applications/service_test.go +++ b/domain/service/applications/service_test.go @@ -35,6 +35,12 @@ func TestCreateGeneratesOpaqueCredentialsAndStoresOnlyDigest(t *testing.T) { require.Empty(t, stored.LastIssuedAt) } +func TestManualsWriteScopeIncludesReadScope(t *testing.T) { + scopes, err := normalizeScopes([]string{ScopeManualsWrite}, false) + require.NoError(t, err) + require.Equal(t, []string{ScopeManualsRead, ScopeManualsWrite}, scopes) +} + func TestOwnerIsolationAndRevisionProtection(t *testing.T) { service, repo, now := testService() application, _, err := service.Create(context.Background(), 101, CreateInput{Name: "owner app", Scopes: []string{ScopeLibraryRead}}) diff --git a/domain/service/manuals/pdf_preview_boundary_test.go b/domain/service/manuals/pdf_preview_boundary_test.go new file mode 100644 index 0000000..29b0b2d --- /dev/null +++ b/domain/service/manuals/pdf_preview_boundary_test.go @@ -0,0 +1,64 @@ +package manuals + +import ( + "bytes" + "fmt" + "image" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/mcoder2014/home_server/config" + "github.com/mcoder2014/home_server/domain/model" + "github.com/stretchr/testify/require" +) + +func TestPDFPreviewStrictlyBoundsDecodedJPEG(t *testing.T) { + pdftoppm, err := exec.LookPath("pdftoppm") + if err != nil { + t.Skip("pdftoppm is required for the real PDF preview boundary test") + } + prlimit, err := exec.LookPath("prlimit") + if err != nil { + t.Skip("prlimit is required for the real PDF preview boundary test") + } + root := filepath.Join(t.TempDir(), "manuals") + conf := config.ManualsConfig{Enabled: true, StorageRoot: root, PDFToPPMPath: pdftoppm, PRLimitPath: prlimit} + require.NoError(t, Init(&conf, "", "")) + + stored, err := StoreFile(&conf, 9, 10, 11, "boundary.pdf", bytes.NewReader(pdfPreviewBoundaryFixture())) + require.NoError(t, err) + require.Equal(t, model.ManualItemPDF, stored.Kind) + require.Equal(t, model.ManualPreviewReady, stored.PreviewStatus) + preview, err := os.Open(filepath.Join(root, filepath.FromSlash(stored.ThumbnailKey))) + require.NoError(t, err) + defer preview.Close() + decoded, format, err := image.Decode(preview) + require.NoError(t, err) + require.Equal(t, "jpeg", format) + require.LessOrEqual(t, decoded.Bounds().Dx(), ThumbnailMaxEdge) + require.LessOrEqual(t, decoded.Bounds().Dy(), ThumbnailMaxEdge) +} + +func pdfPreviewBoundaryFixture() []byte { + var document bytes.Buffer + document.WriteString("%PDF-1.4\n") + offsets := make([]int, 5) + writeObject := func(id int, body string) { + offsets[id] = document.Len() + fmt.Fprintf(&document, "%d 0 obj\n%s\nendobj\n", id, body) + } + writeObject(1, "<< /Type /Catalog /Pages 2 0 R >>") + writeObject(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>") + writeObject(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R /Resources << >> >>") + stream := "0.2 0.6 0.9 rg\n20 20 160 160 re f\n" + writeObject(4, fmt.Sprintf("<< /Length %d >>\nstream\n%sendstream", len(stream), stream)) + xref := document.Len() + document.WriteString("xref\n0 5\n0000000000 65535 f \n") + for id := 1; id <= 4; id++ { + fmt.Fprintf(&document, "%010d 00000 n \n", offsets[id]) + } + fmt.Fprintf(&document, "trailer\n<< /Size 5 /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", xref) + return document.Bytes() +} diff --git a/domain/service/manuals/service.go b/domain/service/manuals/service.go new file mode 100644 index 0000000..8bbf93b --- /dev/null +++ b/domain/service/manuals/service.go @@ -0,0 +1,279 @@ +package manuals + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "unicode/utf8" + + "github.com/mcoder2014/home_server/config" + "github.com/mcoder2014/home_server/domain/model" + apperrors "github.com/mcoder2014/home_server/errors" +) + +const ( + MaxNameRunes = 120 + MaxCategoryRunes = 100 + MaxCategories = 20 + MaxDescriptionRunes = 2000 + MaxTitleRunes = 200 + MaxTextRunes = 100000 + MaxURLBytes = 2048 + MaxRequestIDBytes = 128 + MaxOriginalNameRunes = 255 + MaxImagePixels = 40_000_000 + ThumbnailMaxEdge = 480 +) + +type InlineItemInput struct { + Kind string `json:"kind"` + Title string `json:"title"` + Text string `json:"text"` + URL string `json:"url"` + ClientRequestID string `json:"client_request_id"` +} + +type ValidatedInlineItem struct { + Kind model.ManualItemKind + Title string + Text string + URL string + ClientRequestID string +} + +// Init applies fixed defaults and prepares the private storage root only when +// the statically configured module is enabled. +func Init(conf *config.ManualsConfig, webDAVRoot, webProjectsRoot string) error { + if conf == nil { + return fmt.Errorf("manuals configuration is missing") + } + applyDefaults(conf) + if err := validateLimits(conf); err != nil { + return err + } + configurePreviewGate(conf.MaxConcurrentPDFPreviews) + if !conf.Enabled { + return nil + } + if !filepath.IsAbs(conf.StorageRoot) { + return fmt.Errorf("manuals.storage_root must be an absolute path") + } + if err := os.MkdirAll(conf.StorageRoot, 0700); err != nil { + return fmt.Errorf("create manuals storage root: %w", err) + } + if err := rejectRootSymlink(conf.StorageRoot); err != nil { + return err + } + if err := validateStorageIsolation(conf.StorageRoot, webDAVRoot, "webdav.share_path"); err != nil { + return err + } + if err := validateStorageIsolation(conf.StorageRoot, webProjectsRoot, "web_projects.storage_root"); err != nil { + return err + } + if err := os.Chmod(conf.StorageRoot, 0700); err != nil { + return fmt.Errorf("secure manuals storage root: %w", err) + } + return os.MkdirAll(filepath.Join(conf.StorageRoot, ".staging"), 0700) +} + +func applyDefaults(conf *config.ManualsConfig) { + if conf.MaxFileBytes <= 0 { + conf.MaxFileBytes = 50 << 20 + } + if conf.MaxItemsPerManual <= 0 { + conf.MaxItemsPerManual = 100 + } + if conf.MaxManualBytes <= 0 { + conf.MaxManualBytes = 500 << 20 + } + if conf.MaxManualsPerUser <= 0 { + conf.MaxManualsPerUser = 1000 + } + if conf.MaxUserBytes <= 0 { + conf.MaxUserBytes = 10 << 30 + } + if conf.MinFreeDiskBytes <= 0 { + conf.MinFreeDiskBytes = 2 << 30 + } + if conf.PDFToPPMPath == "" { + conf.PDFToPPMPath = "/usr/bin/pdftoppm" + } + if conf.PRLimitPath == "" { + conf.PRLimitPath = "/usr/bin/prlimit" + } + if conf.PDFPreviewTimeoutSeconds <= 0 { + conf.PDFPreviewTimeoutSeconds = 10 + } + if conf.MaxConcurrentPDFPreviews <= 0 { + conf.MaxConcurrentPDFPreviews = 1 + } + if conf.PDFPreviewMemoryLimitBytes <= 0 { + conf.PDFPreviewMemoryLimitBytes = 512 << 20 + } + if conf.PDFPreviewCPUSeconds <= 0 { + conf.PDFPreviewCPUSeconds = 10 + } + if conf.PDFPreviewOutputLimitBytes <= 0 { + conf.PDFPreviewOutputLimitBytes = 16 << 20 + } + if conf.PDFPreviewOpenFilesLimit <= 0 { + conf.PDFPreviewOpenFilesLimit = 32 + } +} + +func validateLimits(conf *config.ManualsConfig) error { + if conf.MaxFileBytes <= 0 || conf.MaxItemsPerManual < 1 || conf.MaxItemsPerManual > 1000 || conf.MaxManualBytes < conf.MaxFileBytes || conf.MaxUserBytes < conf.MaxManualBytes || conf.MaxManualsPerUser < 1 || conf.MinFreeDiskBytes < 0 { + return fmt.Errorf("invalid manuals storage limits") + } + if conf.PDFPreviewTimeoutSeconds < 1 || conf.PDFPreviewTimeoutSeconds > 60 || conf.MaxConcurrentPDFPreviews < 1 || conf.MaxConcurrentPDFPreviews > 8 || conf.PDFPreviewMemoryLimitBytes < 64<<20 || conf.PDFPreviewCPUSeconds < 1 || conf.PDFPreviewCPUSeconds > 60 || conf.PDFPreviewOutputLimitBytes < 1<<20 || conf.PDFPreviewOpenFilesLimit < 8 { + return fmt.Errorf("invalid manuals PDF preview limits") + } + if !filepath.IsAbs(conf.PDFToPPMPath) || !filepath.IsAbs(conf.PRLimitPath) { + return fmt.Errorf("manuals PDF tools must use absolute paths") + } + return nil +} + +func rejectRootSymlink(root string) error { + info, err := os.Lstat(root) + if err != nil { + return fmt.Errorf("inspect manuals storage root: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("manuals.storage_root must be a real directory, not a symlink") + } + return nil +} + +func validateStorageIsolation(root, other, otherName string) error { + if strings.TrimSpace(other) == "" { + return nil + } + manualPath, err := canonicalPath(root) + if err != nil { + return fmt.Errorf("resolve manuals storage: %w", err) + } + otherPath, err := canonicalPath(other) + if err != nil { + return fmt.Errorf("resolve %s: %w", otherName, err) + } + separator := string(os.PathSeparator) + if manualPath == otherPath || strings.HasPrefix(manualPath, otherPath+separator) || strings.HasPrefix(otherPath, manualPath+separator) { + return fmt.Errorf("manuals.storage_root must not overlap %s", otherName) + } + return nil +} + +func canonicalPath(value string) (string, error) { + absolute, err := filepath.Abs(value) + if err != nil { + return "", err + } + resolved, err := filepath.EvalSymlinks(absolute) + if err == nil { + return filepath.Clean(resolved), nil + } + parent, parentErr := filepath.EvalSymlinks(filepath.Dir(absolute)) + if parentErr != nil { + return "", err + } + return filepath.Join(parent, filepath.Base(absolute)), nil +} + +func ValidateManualFields(name, description, accessMode string) (string, string, model.ManualAccess, error) { + name = strings.TrimSpace(name) + if accessMode == "" { + accessMode = "owner" + } + access, ok := model.ParseManualAccess(accessMode) + if name == "" || !ok || !validRunes(name, MaxNameRunes) || !validRunes(description, MaxDescriptionRunes) { + return "", "", 0, apperrors.ErrInvalid + } + return name, description, access, nil +} + +func ValidateCategories(values []string) ([]string, error) { + if len(values) > MaxCategories { + return nil, apperrors.ErrInvalid + } + categories := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + category := strings.TrimSpace(value) + if category == "" || !validRunes(category, MaxCategoryRunes) { + return nil, apperrors.ErrInvalid + } + if _, exists := seen[category]; exists { + continue + } + seen[category] = struct{}{} + categories = append(categories, category) + } + sort.Strings(categories) + return categories, nil +} + +func ValidateInlineItem(input InlineItemInput) (*ValidatedInlineItem, error) { + kind, ok := model.ParseManualItemKind(input.Kind) + input.Title = strings.TrimSpace(input.Title) + if !ok || (kind != model.ManualItemText && kind != model.ManualItemURL) || !validRunes(input.Title, MaxTitleRunes) || !ValidRequestID(input.ClientRequestID) { + return nil, apperrors.ErrInvalid + } + result := &ValidatedInlineItem{Kind: kind, Title: input.Title, ClientRequestID: input.ClientRequestID} + if kind == model.ManualItemText { + if input.Text == "" || !validRunes(input.Text, MaxTextRunes) || input.URL != "" { + return nil, apperrors.ErrInvalid + } + result.Text = input.Text + return result, nil + } + if input.Text != "" || len(input.URL) == 0 || len(input.URL) > MaxURLBytes || strings.TrimSpace(input.URL) != input.URL { + return nil, apperrors.ErrInvalid + } + parsed, err := url.Parse(input.URL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" || parsed.User != nil { + return nil, apperrors.ErrInvalid + } + result.URL = parsed.String() + return result, nil +} + +func ValidRequestID(value string) bool { + if value == "" || len(value) > MaxRequestIDBytes { + return false + } + for _, character := range value { + if (character < 'a' || character > 'z') && (character < 'A' || character > 'Z') && (character < '0' || character > '9') && !strings.ContainsRune("._:-", character) { + return false + } + } + return true +} + +func ValidateFileMetadata(title, requestID string) (string, error) { + title = strings.TrimSpace(title) + if !validRunes(title, MaxTitleRunes) || !ValidRequestID(requestID) { + return "", apperrors.ErrInvalid + } + return title, nil +} + +func CanReadManual(access model.ManualAccess, status model.ManualStatus, ownerUserID, viewerUserID int64) bool { + if status == model.ManualStatusDeleted { + return false + } + if viewerUserID > 0 && viewerUserID == ownerUserID { + return true + } + if status != model.ManualStatusActive { + return false + } + return access == model.ManualAccessPublic || (access == model.ManualAccessAuthenticated && viewerUserID > 0) +} + +func validRunes(value string, maximum int) bool { + return utf8.ValidString(value) && utf8.RuneCountInString(value) <= maximum +} diff --git a/domain/service/manuals/service_test.go b/domain/service/manuals/service_test.go new file mode 100644 index 0000000..938bcb3 --- /dev/null +++ b/domain/service/manuals/service_test.go @@ -0,0 +1,170 @@ +package manuals + +import ( + "bytes" + "encoding/binary" + "hash/crc32" + "image" + "image/color" + "image/draw" + "image/jpeg" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mcoder2014/home_server/config" + "github.com/mcoder2014/home_server/domain/model" + apperrors "github.com/mcoder2014/home_server/errors" + "github.com/stretchr/testify/require" +) + +func TestValidateManualFieldsUsesDocumentedCharacterLimits(t *testing.T) { + name, description, access, err := ValidateManualFields(strings.Repeat("说", 120), strings.Repeat("明", 2000), "") + require.NoError(t, err) + require.Equal(t, model.ManualAccessOwner, access) + require.Len(t, []rune(name), 120) + require.Len(t, []rune(description), 2000) + + _, _, _, err = ValidateManualFields(strings.Repeat("说", 121), "", "owner") + require.ErrorIs(t, err, apperrors.ErrInvalid) + _, _, _, err = ValidateManualFields("name", "", "unknown") + require.ErrorIs(t, err, apperrors.ErrInvalid) +} + +func TestValidateCategoriesTrimsDeduplicatesAndPreservesCase(t *testing.T) { + categories, err := ValidateCategories([]string{" 厨房 ", "厨房", "kitchen", "Kitchen"}) + require.NoError(t, err) + require.Equal(t, []string{"Kitchen", "kitchen", "厨房"}, categories) + + _, err = ValidateCategories([]string{""}) + require.ErrorIs(t, err, apperrors.ErrInvalid) + _, err = ValidateCategories([]string{strings.Repeat("类", MaxCategoryRunes+1)}) + require.ErrorIs(t, err, apperrors.ErrInvalid) + _, err = ValidateCategories(make([]string, MaxCategories+1)) + require.ErrorIs(t, err, apperrors.ErrInvalid) +} + +func TestValidateInlineItemRejectsUnsafeURLAndAcceptsLiteralText(t *testing.T) { + _, err := ValidateInlineItem(InlineItemInput{Kind: "url", URL: "https://user:secret@example.com/manual", ClientRequestID: "unsafe-userinfo"}) + require.ErrorIs(t, err, apperrors.ErrInvalid) + _, err = ValidateInlineItem(InlineItemInput{Kind: "url", URL: "javascript:alert(1)", ClientRequestID: "unsafe-scheme"}) + require.ErrorIs(t, err, apperrors.ErrInvalid) + validated, err := ValidateInlineItem(InlineItemInput{Kind: "url", Title: "官网", URL: "https://example.com/manual?q=1", ClientRequestID: "valid-url"}) + require.NoError(t, err) + require.Equal(t, "https://example.com/manual?q=1", validated.URL) + + validated, err = ValidateInlineItem(InlineItemInput{Kind: "text", Text: "第一行\n第二行", ClientRequestID: "valid-text"}) + require.NoError(t, err) + require.Equal(t, "第一行\n第二行", validated.Text) +} + +func TestCanReadManualCoversOwnerAuthenticatedAndPublicModes(t *testing.T) { + tests := []struct { + name string + mode model.ManualAccess + status model.ManualStatus + ownerID int64 + viewerID int64 + want bool + }{ + {name: "owner draft", mode: model.ManualAccessPublic, status: model.ManualStatusDraft, ownerID: 1, viewerID: 1, want: true}, + {name: "other draft", mode: model.ManualAccessPublic, status: model.ManualStatusDraft, ownerID: 1, viewerID: 2}, + {name: "anonymous public active", mode: model.ManualAccessPublic, status: model.ManualStatusActive, ownerID: 1, want: true}, + {name: "anonymous authenticated active", mode: model.ManualAccessAuthenticated, status: model.ManualStatusActive, ownerID: 1}, + {name: "other authenticated active", mode: model.ManualAccessAuthenticated, status: model.ManualStatusActive, ownerID: 1, viewerID: 2, want: true}, + {name: "other owner-only active", mode: model.ManualAccessOwner, status: model.ManualStatusActive, ownerID: 1, viewerID: 2}, + {name: "deleted owner", mode: model.ManualAccessOwner, status: model.ManualStatusDeleted, ownerID: 1, viewerID: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, CanReadManual(tt.mode, tt.status, tt.ownerID, tt.viewerID)) + }) + } +} + +func TestImageThumbnailAppliesJPEGEXIFOrientation(t *testing.T) { + source := image.NewRGBA(image.Rect(0, 0, 60, 90)) + draw.Draw(source, image.Rect(0, 0, 30, 45), image.NewUniform(color.RGBA{R: 255, A: 255}), image.Point{}, draw.Src) + draw.Draw(source, image.Rect(30, 0, 60, 45), image.NewUniform(color.RGBA{G: 255, A: 255}), image.Point{}, draw.Src) + draw.Draw(source, image.Rect(0, 45, 30, 90), image.NewUniform(color.RGBA{B: 255, A: 255}), image.Point{}, draw.Src) + draw.Draw(source, image.Rect(30, 45, 60, 90), image.NewUniform(color.RGBA{R: 255, G: 255, A: 255}), image.Point{}, draw.Src) + var encoded bytes.Buffer + require.NoError(t, jpeg.Encode(&encoded, source, &jpeg.Options{Quality: 100})) + oriented := insertEXIFOrientation(t, encoded.Bytes(), 6) + + destination := filepath.Join(t.TempDir(), "thumbnail.jpg") + contentType, err := CreateImageThumbnail(bytes.NewReader(oriented), destination) + require.NoError(t, err) + require.Equal(t, "image/jpeg", contentType) + file, err := os.Open(destination) + require.NoError(t, err) + defer file.Close() + thumbnail, _, err := image.Decode(file) + require.NoError(t, err) + require.Equal(t, 90, thumbnail.Bounds().Dx()) + require.Equal(t, 60, thumbnail.Bounds().Dy()) + // Orientation 6 rotates the original clockwise: its blue bottom-left pixel + // block moves to the thumbnail's top-left. Sample well inside the block so + // JPEG chroma subsampling at color boundaries cannot decide the assertion. + r, g, b, _ := thumbnail.At(15, 15).RGBA() + require.Greater(t, b, r) + require.Greater(t, b, g) +} + +func TestImageThumbnailRejectsOversizedDimensionsBeforeDecode(t *testing.T) { + destination := filepath.Join(t.TempDir(), "thumbnail.jpg") + _, err := CreateImageThumbnail(bytes.NewReader(pngHeader(10000, 5000)), destination) + require.ErrorIs(t, err, apperrors.ErrTooLarge) + require.NoFileExists(t, destination) +} + +func TestInitRejectsManualStorageOverlapAndSymlinkAlias(t *testing.T) { + root := t.TempDir() + shared := filepath.Join(root, "shared") + require.NoError(t, os.MkdirAll(shared, 0700)) + conf := config.ManualsConfig{Enabled: true, StorageRoot: filepath.Join(shared, "manuals")} + require.Error(t, Init(&conf, shared, "")) + + private := filepath.Join(root, "private") + require.NoError(t, os.MkdirAll(private, 0700)) + alias := filepath.Join(root, "manual-link") + require.NoError(t, os.Symlink(private, alias)) + conf.StorageRoot = alias + require.Error(t, Init(&conf, "", private)) +} + +func insertEXIFOrientation(t *testing.T, jpegData []byte, orientation uint16) []byte { + t.Helper() + require.GreaterOrEqual(t, len(jpegData), 2) + payload := make([]byte, 32) + copy(payload, []byte("Exif\x00\x00II")) + binary.LittleEndian.PutUint16(payload[8:10], 42) + binary.LittleEndian.PutUint32(payload[10:14], 8) + binary.LittleEndian.PutUint16(payload[14:16], 1) + binary.LittleEndian.PutUint16(payload[16:18], 0x0112) + binary.LittleEndian.PutUint16(payload[18:20], 3) + binary.LittleEndian.PutUint32(payload[20:24], 1) + binary.LittleEndian.PutUint16(payload[24:26], orientation) + segment := []byte{0xff, 0xe1, 0, byte(len(payload) + 2)} + segment = append(segment, payload...) + result := append([]byte{}, jpegData[:2]...) + result = append(result, segment...) + return append(result, jpegData[2:]...) +} + +func pngHeader(width, height uint32) []byte { + data := make([]byte, 13) + binary.BigEndian.PutUint32(data[0:4], width) + binary.BigEndian.PutUint32(data[4:8], height) + data[8], data[9] = 8, 2 + chunk := append([]byte("IHDR"), data...) + result := append([]byte{}, []byte("\x89PNG\r\n\x1a\n")...) + length := make([]byte, 4) + binary.BigEndian.PutUint32(length, uint32(len(data))) + result = append(result, length...) + result = append(result, chunk...) + checksum := make([]byte, 4) + binary.BigEndian.PutUint32(checksum, crc32.ChecksumIEEE(chunk)) + return append(result, checksum...) +} diff --git a/domain/service/manuals/storage.go b/domain/service/manuals/storage.go new file mode 100644 index 0000000..f2d4ebc --- /dev/null +++ b/domain/service/manuals/storage.go @@ -0,0 +1,626 @@ +package manuals + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "image" + "image/color" + _ "image/gif" + "image/jpeg" + _ "image/png" + "io" + "math" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + "unicode/utf8" + + "github.com/mcoder2014/home_server/config" + "github.com/mcoder2014/home_server/domain/model" + apperrors "github.com/mcoder2014/home_server/errors" + "golang.org/x/image/draw" + _ "golang.org/x/image/webp" +) + +type StoredFile struct { + Kind model.ManualItemKind + Text string + StorageKey string + ThumbnailKey string + ContentType string + OriginalName string + SizeBytes int64 + SHA256 string + PreviewStatus model.ManualPreviewStatus +} + +var previewState = struct { + sync.RWMutex + gate chan struct{} +}{gate: make(chan struct{}, 1)} + +func configurePreviewGate(limit int) { + previewState.Lock() + previewState.gate = make(chan struct{}, limit) + previewState.Unlock() +} + +// StoreFile streams an original into a same-filesystem staging directory, +// validates actual bytes, creates a stripped thumbnail, then atomically moves +// the complete item directory into its service-generated final location. +func StoreFile(conf *config.ManualsConfig, ownerID, manualID, itemID int64, originalName string, source io.Reader) (*StoredFile, error) { + if conf == nil || !conf.Enabled || ownerID <= 0 || manualID <= 0 || itemID <= 0 { + return nil, apperrors.ErrDependency + } + name, err := validateOriginalName(originalName) + if err != nil { + return nil, err + } + staging, err := os.MkdirTemp(filepath.Join(conf.StorageRoot, ".staging"), "item-") + if err != nil { + return nil, apperrors.ErrDependency + } + defer os.RemoveAll(staging) + originalPath := filepath.Join(staging, "original") + size, digest, err := writeOriginal(originalPath, source, conf.MaxFileBytes) + if err != nil { + return nil, err + } + stored, err := inspectStoredFile(conf, originalPath, filepath.Join(staging, "thumbnail.jpg"), name) + if err != nil { + return nil, err + } + stored.OriginalName, stored.SizeBytes, stored.SHA256 = name, size, digest + if stored.Kind == model.ManualItemText { + return stored, nil + } + itemKey := path.Join(strconv.FormatInt(ownerID, 10), strconv.FormatInt(manualID, 10), strconv.FormatInt(itemID, 10)) + finalParent, err := ensureManualDirectory(conf.StorageRoot, strconv.FormatInt(ownerID, 10), strconv.FormatInt(manualID, 10)) + if err != nil { + return nil, apperrors.ErrDependency + } + finalDirectory := filepath.Join(finalParent, strconv.FormatInt(itemID, 10)) + if _, err := os.Lstat(finalDirectory); err == nil { + return nil, apperrors.ErrConflict + } else if !errors.Is(err, os.ErrNotExist) { + return nil, apperrors.ErrDependency + } + if err := os.Rename(staging, finalDirectory); err != nil { + return nil, apperrors.ErrDependency + } + if err := syncDirectory(finalDirectory); err != nil { + _ = os.RemoveAll(finalDirectory) + return nil, apperrors.ErrDependency + } + if err := syncDirectory(finalParent); err != nil { + _ = os.RemoveAll(finalDirectory) + return nil, apperrors.ErrDependency + } + stored.StorageKey = path.Join(itemKey, "original") + if stored.PreviewStatus == model.ManualPreviewReady { + stored.ThumbnailKey = path.Join(itemKey, "thumbnail.jpg") + } + return stored, nil +} + +func validateOriginalName(name string) (string, error) { + name = strings.TrimSpace(name) + if name == "" || !validRunes(name, MaxOriginalNameRunes) || strings.ContainsAny(name, "\x00\r\n") { + return "", apperrors.ErrInvalid + } + name = filepath.Base(strings.ReplaceAll(name, "\\", "/")) + if name == "." || name == "/" || name == "" { + return "", apperrors.ErrInvalid + } + return name, nil +} + +func writeOriginal(target string, source io.Reader, limit int64) (int64, string, error) { + file, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return 0, "", apperrors.ErrDependency + } + hash := sha256.New() + written, copyErr := io.Copy(io.MultiWriter(file, hash), io.LimitReader(source, limit+1)) + syncErr := file.Sync() + closeErr := file.Close() + if copyErr != nil || syncErr != nil || closeErr != nil { + return 0, "", apperrors.ErrDependency + } + if written == 0 { + return 0, "", apperrors.ErrUnprocessable + } + if written > limit { + return 0, "", apperrors.ErrTooLarge + } + return written, hex.EncodeToString(hash.Sum(nil)), nil +} + +func inspectStoredFile(conf *config.ManualsConfig, originalPath, thumbnailPath, originalName string) (*StoredFile, error) { + header, err := filePrefix(originalPath, 512) + if err != nil { + return nil, apperrors.ErrDependency + } + if bytes.HasPrefix(header, []byte("%PDF-")) { + status := model.ManualPreviewUnavailable + if renderPDFPreview(conf, originalPath, thumbnailPath) == nil { + status = model.ManualPreviewReady + } + return &StoredFile{Kind: model.ManualItemPDF, ContentType: "application/pdf", PreviewStatus: status}, nil + } + file, err := os.Open(originalPath) + if err != nil { + return nil, apperrors.ErrDependency + } + _, format, imageErr := image.DecodeConfig(file) + _ = file.Close() + if imageErr == nil && imageContentType(format) != "" { + contentType, thumbErr := createImageThumbnailFile(originalPath, thumbnailPath) + if thumbErr != nil { + return nil, thumbErr + } + return &StoredFile{Kind: model.ManualItemImage, ContentType: contentType, PreviewStatus: model.ManualPreviewReady}, nil + } + if strings.HasPrefix(http.DetectContentType(header), "image/") { + return nil, apperrors.ErrUnprocessable + } + if strings.ToLower(filepath.Ext(originalName)) != ".txt" { + return nil, apperrors.ErrUnsupported + } + text, textErr := readPlainText(originalPath) + if textErr == nil { + return &StoredFile{Kind: model.ManualItemText, Text: text, ContentType: "text/plain; charset=utf-8", PreviewStatus: model.ManualPreviewNone}, nil + } + return nil, textErr +} + +func filePrefix(name string, limit int64) ([]byte, error) { + file, err := os.Open(name) + if err != nil { + return nil, err + } + defer file.Close() + return io.ReadAll(io.LimitReader(file, limit)) +} + +func readPlainText(name string) (string, error) { + file, err := os.Open(name) + if err != nil { + return "", apperrors.ErrDependency + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, MaxTextRunes*4+1)) + if err != nil { + return "", apperrors.ErrDependency + } + if len(data) > MaxTextRunes*4 || !utf8.Valid(data) || utf8.RuneCount(data) > MaxTextRunes || bytes.IndexByte(data, 0) >= 0 { + return "", apperrors.ErrUnsupported + } + return string(data), nil +} + +func CreateImageThumbnail(source io.Reader, destination string) (string, error) { + temporary, err := os.CreateTemp(filepath.Dir(destination), "image-source-") + if err != nil { + return "", apperrors.ErrDependency + } + name := temporary.Name() + defer os.Remove(name) + if _, err = io.Copy(temporary, source); err != nil || temporary.Close() != nil { + return "", apperrors.ErrDependency + } + return createImageThumbnailFile(name, destination) +} + +func createImageThumbnailFile(source, destination string) (string, error) { + previewState.RLock() + gate := previewState.gate + previewState.RUnlock() + gate <- struct{}{} + defer func() { + <-gate + }() + return writeImageThumbnailFile(source, destination) +} + +// writeImageThumbnailFile decodes, bounds and re-encodes an image while its +// caller holds the shared preview gate. +func writeImageThumbnailFile(source, destination string) (string, error) { + file, err := os.Open(source) + if err != nil { + return "", apperrors.ErrDependency + } + reader := bufio.NewReader(file) + orientation := jpegOrientation(reader) + if _, err = file.Seek(0, io.SeekStart); err != nil { + _ = file.Close() + return "", apperrors.ErrDependency + } + imageConfig, format, err := image.DecodeConfig(file) + contentType := imageContentType(format) + if err != nil || contentType == "" { + _ = file.Close() + return "", apperrors.ErrUnprocessable + } + width, height := int64(imageConfig.Width), int64(imageConfig.Height) + if width <= 0 || height <= 0 || width > MaxImagePixels/height { + _ = file.Close() + return "", apperrors.WithMessage(apperrors.ErrTooLarge, "image pixel count exceeds limit") + } + if _, err = file.Seek(0, io.SeekStart); err != nil { + _ = file.Close() + return "", apperrors.ErrDependency + } + decoded, decodedFormat, err := image.Decode(file) + _ = file.Close() + if err != nil || decodedFormat != format { + return "", apperrors.ErrUnprocessable + } + bounds := decoded.Bounds() + width, height = int64(bounds.Dx()), int64(bounds.Dy()) + if width <= 0 || height <= 0 || width > MaxImagePixels/height { + return "", apperrors.WithMessage(apperrors.ErrTooLarge, "image pixel count exceeds limit") + } + oriented := orientImage(decoded, orientation) + targetWidth, targetHeight := thumbnailSize(oriented.Bounds().Dx(), oriented.Bounds().Dy()) + thumbnail := image.NewRGBA(image.Rect(0, 0, targetWidth, targetHeight)) + draw.CatmullRom.Scale(thumbnail, thumbnail.Bounds(), oriented, oriented.Bounds(), draw.Over, nil) + output, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return "", apperrors.ErrDependency + } + encodeErr := jpeg.Encode(output, thumbnail, &jpeg.Options{Quality: 88}) + syncErr := output.Sync() + closeErr := output.Close() + if encodeErr != nil || syncErr != nil || closeErr != nil { + _ = os.Remove(destination) + return "", apperrors.ErrDependency + } + return contentType, nil +} + +func imageContentType(format string) string { + switch format { + case "jpeg": + return "image/jpeg" + case "png": + return "image/png" + case "gif": + return "image/gif" + case "webp": + return "image/webp" + default: + return "" + } +} + +func thumbnailSize(width, height int) (int, int) { + if width <= ThumbnailMaxEdge && height <= ThumbnailMaxEdge { + return width, height + } + if width >= height { + return ThumbnailMaxEdge, maxInt(1, height*ThumbnailMaxEdge/width) + } + return maxInt(1, width*ThumbnailMaxEdge/height), ThumbnailMaxEdge +} + +func maxInt(left, right int) int { + if left > right { + return left + } + return right +} + +type oriented struct { + image.Image + orientation int + bounds image.Rectangle +} + +func orientImage(source image.Image, orientation int) image.Image { + bounds := source.Bounds() + width, height := bounds.Dx(), bounds.Dy() + if orientation < 2 || orientation > 8 { + return source + } + if orientation >= 5 { + width, height = height, width + } + return &oriented{Image: source, orientation: orientation, bounds: image.Rect(0, 0, width, height)} +} + +func (value *oriented) Bounds() image.Rectangle { + return value.bounds +} + +func (value *oriented) At(x, y int) color.Color { + source := value.Image.Bounds() + w, h := source.Dx(), source.Dy() + var sourceX, sourceY int + switch value.orientation { + case 2: + sourceX, sourceY = w-1-x, y + case 3: + sourceX, sourceY = w-1-x, h-1-y + case 4: + sourceX, sourceY = x, h-1-y + case 5: + sourceX, sourceY = y, x + case 6: + sourceX, sourceY = y, h-1-x + case 7: + sourceX, sourceY = w-1-y, h-1-x + case 8: + sourceX, sourceY = w-1-y, x + default: + sourceX, sourceY = x, y + } + return value.Image.At(source.Min.X+sourceX, source.Min.Y+sourceY) +} + +func jpegOrientation(reader *bufio.Reader) int { + const maxMetadataBytes = 1 << 20 + var signature [2]byte + if _, err := io.ReadFull(reader, signature[:]); err != nil || signature != [2]byte{0xff, 0xd8} { + return 1 + } + scanned := len(signature) + for scanned < maxMetadataBytes { + prefix, err := reader.ReadByte() + scanned++ + if err != nil || prefix != 0xff { + break + } + marker, err := reader.ReadByte() + scanned++ + if err != nil { + break + } + for marker == 0xff && scanned < maxMetadataBytes { + marker, err = reader.ReadByte() + scanned++ + if err != nil { + return 1 + } + } + if marker == 0x00 { + break + } + if marker == 0xda || marker == 0xd9 { + break + } + if marker == 0xd8 || marker == 0x01 || marker >= 0xd0 && marker <= 0xd7 { + continue + } + var lengthBytes [2]byte + if _, err = io.ReadFull(reader, lengthBytes[:]); err != nil { + break + } + scanned += len(lengthBytes) + length := int(binary.BigEndian.Uint16(lengthBytes[:])) + if length < 2 { + break + } + payloadLength := length - 2 + if payloadLength > maxMetadataBytes-scanned { + break + } + if marker == 0xe1 { + payload := make([]byte, payloadLength) + if _, err = io.ReadFull(reader, payload); err != nil { + break + } + scanned += payloadLength + if orientation := parseEXIFOrientation(payload); orientation != 1 { + return orientation + } + continue + } + if _, err = io.CopyN(io.Discard, reader, int64(payloadLength)); err != nil { + break + } + scanned += payloadLength + } + return 1 +} + +func parseEXIFOrientation(data []byte) int { + if len(data) < 14 || !bytes.Equal(data[:6], []byte("Exif\x00\x00")) { + return 1 + } + tiff := data[6:] + var order binary.ByteOrder + switch string(tiff[:2]) { + case "II": + order = binary.LittleEndian + case "MM": + order = binary.BigEndian + default: + return 1 + } + if order.Uint16(tiff[2:4]) != 42 { + return 1 + } + offset := int(order.Uint32(tiff[4:8])) + if offset < 8 || offset+2 > len(tiff) { + return 1 + } + count := int(order.Uint16(tiff[offset : offset+2])) + for index := 0; index < count; index++ { + entry := offset + 2 + index*12 + if entry+12 > len(tiff) { + return 1 + } + if order.Uint16(tiff[entry:entry+2]) == 0x0112 && order.Uint16(tiff[entry+2:entry+4]) == 3 && order.Uint32(tiff[entry+4:entry+8]) == 1 { + orientation := int(order.Uint16(tiff[entry+8 : entry+10])) + if orientation >= 1 && orientation <= 8 { + return orientation + } + } + } + return 1 +} + +func renderPDFPreview(conf *config.ManualsConfig, source, destination string) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.PDFPreviewTimeoutSeconds)*time.Second) + defer cancel() + previewState.RLock() + gate := previewState.gate + previewState.RUnlock() + select { + case gate <- struct{}{}: + defer func() { + <-gate + }() + case <-ctx.Done(): + return ctx.Err() + } + prefix := strings.TrimSuffix(destination, filepath.Ext(destination)) + args := []string{ + "--as=" + strconv.FormatInt(conf.PDFPreviewMemoryLimitBytes, 10), + "--cpu=" + strconv.Itoa(conf.PDFPreviewCPUSeconds), + "--fsize=" + strconv.FormatInt(conf.PDFPreviewOutputLimitBytes, 10), + "--nofile=" + strconv.Itoa(conf.PDFPreviewOpenFilesLimit), "--", conf.PDFToPPMPath, + "-f", "1", "-singlefile", "-scale-to", strconv.Itoa(ThumbnailMaxEdge), "-jpeg", "-jpegopt", "quality=88", source, prefix, + } + if err := exec.CommandContext(ctx, conf.PRLimitPath, args...).Run(); err != nil { + _ = os.Remove(destination) + return err + } + info, err := os.Stat(destination) + if err != nil || !info.Mode().IsRegular() || info.Size() <= 0 || info.Size() > conf.PDFPreviewOutputLimitBytes { + _ = os.Remove(destination) + return fmt.Errorf("invalid PDF preview output") + } + normalized := destination + ".normalized" + defer os.Remove(normalized) + contentType, err := writeImageThumbnailFile(destination, normalized) + if err != nil || contentType != "image/jpeg" { + _ = os.Remove(destination) + return fmt.Errorf("invalid PDF preview image") + } + info, err = os.Stat(normalized) + if err != nil || !info.Mode().IsRegular() || info.Size() <= 0 || info.Size() > conf.PDFPreviewOutputLimitBytes { + _ = os.Remove(destination) + return fmt.Errorf("invalid normalized PDF preview") + } + if err = os.Rename(normalized, destination); err != nil { + _ = os.Remove(destination) + return err + } + return nil +} + +func ensureManualDirectory(root string, components ...string) (string, error) { + current := root + for _, component := range components { + current = filepath.Join(current, component) + mkdirErr := os.Mkdir(current, 0700) + if mkdirErr != nil && !errors.Is(mkdirErr, os.ErrExist) { + return "", mkdirErr + } + info, err := os.Lstat(current) + if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return "", fmt.Errorf("manual storage ancestry is invalid") + } + if mkdirErr == nil { + if err := syncDirectory(filepath.Dir(current)); err != nil { + return "", err + } + } + } + return current, nil +} + +func syncDirectory(directoryPath string) error { + directory, err := os.Open(directoryPath) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} + +func RemoveStoredFile(conf *config.ManualsConfig, ownerID, manualID, itemID int64) error { + if conf == nil || ownerID <= 0 || manualID <= 0 || itemID <= 0 { + return apperrors.ErrInvalid + } + target := filepath.Join(conf.StorageRoot, strconv.FormatInt(ownerID, 10), strconv.FormatInt(manualID, 10), strconv.FormatInt(itemID, 10)) + if err := os.RemoveAll(target); err != nil { + return err + } + return syncDirectory(filepath.Dir(target)) +} + +func OpenStoredFile(conf *config.ManualsConfig, key string) (*os.File, os.FileInfo, error) { + if conf == nil || !validStorageKey(key) { + return nil, nil, apperrors.ErrNotFound + } + root, err := filepath.EvalSymlinks(conf.StorageRoot) + if err != nil { + return nil, nil, apperrors.ErrDependency + } + target := filepath.Join(conf.StorageRoot, filepath.FromSlash(key)) + resolved, err := filepath.EvalSymlinks(target) + if err != nil || (resolved != root && !strings.HasPrefix(resolved, root+string(os.PathSeparator))) { + return nil, nil, apperrors.ErrNotFound + } + file, err := os.Open(resolved) + if err != nil { + return nil, nil, apperrors.ErrNotFound + } + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() { + _ = file.Close() + return nil, nil, apperrors.ErrNotFound + } + return file, info, nil +} + +func validStorageKey(key string) bool { + if key == "" || path.Clean(key) != key || strings.Contains(key, "\\") || strings.HasPrefix(key, "/") { + return false + } + parts := strings.Split(key, "/") + if len(parts) != 4 || (parts[3] != "original" && parts[3] != "thumbnail.jpg") { + return false + } + for _, value := range parts[:3] { + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil || parsed <= 0 || strconv.FormatInt(parsed, 10) != value { + return false + } + } + return true +} + +func DiskFreeBytes(root string) (uint64, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(root, &stat); err != nil { + return 0, err + } + if stat.Bsize <= 0 || uint64(stat.Bavail) > math.MaxUint64/uint64(stat.Bsize) { + return 0, fmt.Errorf("invalid filesystem free-space result") + } + return uint64(stat.Bavail) * uint64(stat.Bsize), nil +} + +func minInt(left, right int) int { + if left < right { + return left + } + return right +} diff --git a/domain/service/manuals/storage_large_exif_test.go b/domain/service/manuals/storage_large_exif_test.go new file mode 100644 index 0000000..b8f8f6f --- /dev/null +++ b/domain/service/manuals/storage_large_exif_test.go @@ -0,0 +1,28 @@ +package manuals + +import ( + "bufio" + "bytes" + "encoding/binary" + "testing" +) + +func TestJPEGOrientationReadsLargeAPP1Segment(t *testing.T) { + payload := make([]byte, 8<<10) + copy(payload, []byte("Exif\x00\x00II")) + binary.LittleEndian.PutUint16(payload[8:10], 42) + binary.LittleEndian.PutUint32(payload[10:14], 8) + binary.LittleEndian.PutUint16(payload[14:16], 1) + binary.LittleEndian.PutUint16(payload[16:18], 0x0112) + binary.LittleEndian.PutUint16(payload[18:20], 3) + binary.LittleEndian.PutUint32(payload[20:24], 1) + binary.LittleEndian.PutUint16(payload[24:26], 6) + segmentLength := len(payload) + 2 + jpeg := []byte{0xff, 0xd8, 0xff, 0xe1, byte(segmentLength >> 8), byte(segmentLength)} + jpeg = append(jpeg, payload...) + jpeg = append(jpeg, 0xff, 0xd9) + + if orientation := jpegOrientation(bufio.NewReader(bytes.NewReader(jpeg))); orientation != 6 { + t.Fatalf("large EXIF APP1 orientation = %d, want 6", orientation) + } +} diff --git a/domain/service/manuals/types.go b/domain/service/manuals/types.go new file mode 100644 index 0000000..7b56d58 --- /dev/null +++ b/domain/service/manuals/types.go @@ -0,0 +1,110 @@ +package manuals + +import ( + "bytes" + "encoding/json" + "time" +) + +type OptionalID struct { + Set bool + Value *string +} + +func (value *OptionalID) UnmarshalJSON(data []byte) error { + value.Set = true + if bytes.Equal(bytes.TrimSpace(data), []byte("null")) { + value.Value = nil + return nil + } + var decoded string + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + value.Value = &decoded + return nil +} + +type CreateManualInput struct { + Name string `json:"name"` + Description string `json:"description"` + Categories []string `json:"categories"` + AccessMode string `json:"access_mode"` + ClientRequestID string `json:"client_request_id"` +} + +type UpdateManualInput struct { + Revision int64 `json:"revision"` + Name *string `json:"name"` + Description *string `json:"description"` + Categories *[]string `json:"categories"` + AccessMode *string `json:"access_mode"` + Status *string `json:"status"` + CoverItemID OptionalID `json:"cover_item_id"` + ItemIDs *[]string `json:"item_ids"` +} + +type CoverView struct { + ID string `json:"id"` + Kind string `json:"kind"` + Title string `json:"title"` + ThumbnailURL string `json:"thumbnail_url"` + TextExcerpt string `json:"text_excerpt"` + URLHost string `json:"url_host"` + PreviewStatus string `json:"preview_status"` +} + +type ItemView struct { + ID string `json:"id"` + Kind string `json:"kind"` + Title string `json:"title"` + Text string `json:"text"` + URL string `json:"url"` + OriginalName string `json:"original_name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + Position int `json:"position"` + ContentURL string `json:"content_url"` + ThumbnailURL string `json:"thumbnail_url"` + PreviewStatus string `json:"preview_status"` +} + +type ManualView struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Categories []string `json:"categories"` + AccessMode string `json:"access_mode"` + Status string `json:"status"` + Revision int64 `json:"revision"` + CoverItemID *string `json:"cover_item_id"` + CoverURL string `json:"cover_url"` + Cover *CoverView `json:"cover"` + ItemCount int `json:"item_count"` + CanEdit bool `json:"can_edit"` + CreateTime time.Time `json:"create_time"` + UpdateTime time.Time `json:"update_time"` + Items []*ItemView `json:"items,omitempty"` +} + +type ManualPage struct { + Items []*ManualView `json:"items"` + HasMore bool `json:"has_more"` + NextCursor string `json:"next_cursor"` +} + +type CategoryPage struct { + Items []string `json:"items"` + HasMore bool `json:"has_more"` + NextCursor string `json:"next_cursor"` +} + +type ItemMutationResult struct { + Item *ItemView `json:"item"` + Revision int64 `json:"revision"` +} + +type DeleteItemResult struct { + Revision int64 `json:"revision"` + CoverItemID *string `json:"cover_item_id"` +} diff --git a/domain/service/passport/token.go b/domain/service/passport/token.go index a4e83da..72dd9f6 100644 --- a/domain/service/passport/token.go +++ b/domain/service/passport/token.go @@ -11,6 +11,7 @@ import ( "github.com/mcoder2014/home_server/domain/service/accounts" myErrors "github.com/mcoder2014/home_server/errors" "github.com/mcoder2014/home_server/utils" + "gorm.io/gorm" ) const ( @@ -93,6 +94,32 @@ func DeleteToken(ctx context.Context, token string) error { return dal.ExpireToken(tokenEntity.ID) } +// RequireConfigSessionTx locks and revalidates the exact legacy session in the +// caller's write transaction. Database-backed identities use accounts instead. +func RequireConfigSessionTx(ctx context.Context, tx *gorm.DB, token string, userID, authVersion int64) error { + if token == "" { + return myErrors.ErrUnauthorized + } + if tx == nil { + return myErrors.ErrDependency + } + session, err := dal.QueryByTokenTx(tx, token, true) + if err != nil { + return myErrors.ErrDependency + } + if session == nil || session.UserID != userID || session.IsExpired == model.UserTokenExpired || !session.ExpireTime.After(time.Now()) { + return myErrors.ErrUnauthorized + } + identity, err := GetByID(ctx, userID) + if err != nil { + return myErrors.ErrDependency + } + if identity == nil || identity.ID != userID || identity.AuthVersion != authVersion { + return myErrors.ErrUnauthorized + } + return nil +} + func BuildUserToken(userID int64, expireTime time.Duration) *model.UserToken { if expireTime <= 0 { expireTime = TokenExpireTime diff --git a/domain/service/service.go b/domain/service/service.go index 9ea584b..4d7f9f4 100644 --- a/domain/service/service.go +++ b/domain/service/service.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/mcoder2014/home_server/config" "github.com/mcoder2014/home_server/domain/service/applications" + "github.com/mcoder2014/home_server/domain/service/manuals" "github.com/mcoder2014/home_server/domain/service/passport" "github.com/mcoder2014/home_server/domain/service/webdav" "github.com/mcoder2014/home_server/domain/service/webprojects" @@ -20,6 +21,9 @@ func Init(conf *config.Config) error { if err := webprojects.Init(&conf.WebProjects); err != nil { return err } + if err := manuals.Init(&conf.Manuals, conf.WebDAV.SharePath, conf.WebProjects.StorageRoot); err != nil { + return err + } // Database configuration can enable web projects after startup. Storage // boundaries must already be valid even when the bootstrap switch is off. if conf.WebProjects.Enabled || conf.ConfigSource == "database" { diff --git a/front_vue/package.json b/front_vue/package.json index 68c62b3..af4c5ba 100644 --- a/front_vue/package.json +++ b/front_vue/package.json @@ -8,6 +8,7 @@ "lint": "vue-cli-service lint", "test:frontend": "node --test test/*.test.cjs", "test:account-center-browser": "node --test test/account-center-browser.cjs", + "test:manuals-browser": "node --test test/manuals-browser.cjs", "test:web-projects": "node --test test/web-projects-*.test.cjs" }, "dependencies": { diff --git a/front_vue/src/api/manuals.cjs b/front_vue/src/api/manuals.cjs new file mode 100644 index 0000000..a9c4e4b --- /dev/null +++ b/front_vue/src/api/manuals.cjs @@ -0,0 +1,113 @@ +const axios = require('axios') +const {browserHeaders, requestError} = require('./browser_client.cjs') + +function envelopeError(response) { + const body = response && response.data ? response.data : {} + const error = new Error(body.message || body.msg || '请求失败') + error.code = body.code + error.status = response && response.status + return error +} + +function manualRequestError(error, sentCSRF, upload) { + if (upload && error && error.response && error.response.status === 413) { + const normalized = new Error('文件上传失败：说明书单个文件不能超过 50 MiB') + normalized.code = error.response.data && error.response.data.code + normalized.status = 413 + return normalized + } + return requestError(error, sentCSRF) +} + +// 说明书接口统一使用同源 Cookie/CSRF；revision 放在 JSON 中，文件上传保留独立的 50 MiB 错误提示。 +function createManualsApi(transport, getCSRF) { + const client = transport || axios.create({baseURL: '/', withCredentials: true}) + function headers() { + return browserHeaders({}, getCSRF) + } + async function request(config, upload = false) { + try { + const response = await client.request(config) + if (!response.data || response.data.code !== 0) throw envelopeError(response) + return response.data.data + } catch (error) { + throw manualRequestError(error, config.headers && config.headers['X-CSRF-Token'] || '', upload) + } + } + function manualURL(manualID, suffix = '') { + return `/api/manuals/${encodeURIComponent(String(manualID))}${suffix}` + } + async function listAllCategories(params = {}) { + const baseParams = {...params} + const startCursor = baseParams.cursor + delete baseParams.cursor + delete baseParams.limit + let cursor = String(startCursor || '') + let hasMore = true + const seenCursors = new Set([cursor]) + const items = [] + while (hasMore) { + const query = {...baseParams, limit: 1000} + if (cursor) query.cursor = cursor + const data = await request({method: 'get', url: '/api/manuals/categories', params: query, headers: headers()}) + items.push(...(data.items || [])) + hasMore = data.has_more === true + if (!hasMore) break + const nextCursor = String(data.next_cursor || '') + if (!nextCursor || seenCursors.has(nextCursor)) throw new Error('分类列表分页游标未前进') + seenCursors.add(nextCursor) + cursor = nextCursor + } + return {items, has_more: false, next_cursor: ''} + } + + return { + listManuals(params) { + return request({method: 'get', url: '/api/manuals', params, headers: headers()}) + }, + listCategories(params) { + return request({method: 'get', url: '/api/manuals/categories', params, headers: headers()}) + }, + listAllCategories, + createManual(data) { + return request({method: 'post', url: '/api/manuals', data, headers: headers()}) + }, + getManual(manualID) { + return request({method: 'get', url: manualURL(manualID), headers: headers()}) + }, + updateManual(manualID, data) { + return request({method: 'patch', url: manualURL(manualID), data, headers: headers()}) + }, + deleteManual(manualID, revision) { + return request({method: 'delete', url: manualURL(manualID), data: {revision}, headers: headers()}) + }, + addItem(manualID, data) { + return request({method: 'post', url: manualURL(manualID, '/items'), data, headers: headers()}) + }, + uploadFile(manualID, file, title, clientRequestID, onUploadProgress) { + const data = new FormData() + data.append('file', file) + data.append('title', title || '') + data.append('client_request_id', clientRequestID) + return request({ + method: 'post', + url: manualURL(manualID, '/files'), + data, + headers: headers(), + onUploadProgress, + }, true) + }, + deleteItem(manualID, itemID, revision) { + return request({ + method: 'delete', + url: manualURL(manualID, `/items/${encodeURIComponent(String(itemID))}`), + data: {revision}, + headers: headers(), + }) + }, + } +} + +const manualsApi = createManualsApi() + +module.exports = {createManualsApi, manualsApi} diff --git a/front_vue/src/components/MyHeader.vue b/front_vue/src/components/MyHeader.vue index c2fd7cf..272b00e 100644 --- a/front_vue/src/components/MyHeader.vue +++ b/front_vue/src/components/MyHeader.vue @@ -4,6 +4,7 @@