diff --git a/README.md b/README.md index dc19bf2..94fcbb3 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,10 @@ gobl correct -i ./invoice.json --credit gobl convert ./invoice.json ./invoice.xml --to ubl+en16931-v2017 gobl convert -i ./invoice.xml +# Embed a PDF version of the invoice, and save embedded files when importing +gobl convert ./invoice.json ./invoice.xml --to ubl+peppol+bis-v3 --attach ./invoice.pdf +gobl convert ./invoice.xml ./invoice.json --attachments ./files + # List the available conversion formats gobl convert --list -i @@ -127,8 +131,8 @@ the `/v0` prefix: | `POST` | `/v0/verify` | Verify an envelope's signatures | | `POST` | `/v0/correct` | Generate a corrective document (credit/debit note) | | `POST` | `/v0/replicate` | Clone a document with a new UUID | -| `POST` | `/v0/import` | Detect the format of the raw body (e.g. UBL XML) and import it into a GOBL envelope; `?format=` limits detection | -| `POST` | `/v0/export?format={key}` | Export a GOBL document or envelope into the first accepting format; the key used is returned in the `GOBL-Format` header | +| `POST` | `/v0/import` | Detect the format of the raw body (e.g. UBL XML) and import it into a GOBL envelope; `?format=` limits detection, and `?attachments=true` also returns the embedded files (see [Attachments](#attachments)) | +| `POST` | `/v0/export?format={key}` | Export a GOBL document or envelope into the first accepting format; the key used is returned in the `GOBL-Format` header. A `multipart/form-data` body also embeds files (see [Attachments](#attachments)) | | `POST` | `/v0/keygen` | Generate an ES256 key pair | | `GET` | `/v0/schemas` | List registered JSON schemas | | `GET` | `/v0/schemas/{path}` | Fetch a specific schema (add `?bundle` for bundled) | @@ -183,6 +187,41 @@ the addons of its specification, and explains a refusal with rules faults. `POST /v0/formats` reports, for each format that exports a document's schema, whether it accepts the document and why not. +### Attachments + +Formats with `attachments: true` in `/v0/formats`, such as UBL and most CII +formats, can embed files in exported documents, such as a PDF version of the +invoice. Formats that cannot embed them are skipped when attachments are given. + +- `gobl convert --to ... --attach file.pdf` embeds a file, and may be repeated. + The MIME type is taken from the file's extension. +- `POST /v0/export` takes a `multipart/form-data` body with the GOBL document + in a `document` field or file, and each file to embed in an `attachment` + file part: + + ```bash + curl -F document=@invoice.json -F attachment=@invoice.pdf \ + "https://gobl.dev/v0/export?format=ubl+peppol+bis-v3" + ``` + +Imports provide the files embedded in UBL and CII documents separately from +the envelope: + +- `gobl convert --attachments DIR` saves them to the directory, replacing + existing files. Only the base of each file name is used, and names repeated + in the document are numbered. +- `POST /v0/import` returns the envelope alone, as before, with the number of + embedded files in the `GOBL-Attachments` header. With `?attachments=true`, + it returns `{"format", "envelope", "attachments"}` instead, each attachment + with its `id`, `name`, `description`, `mime` and base64 `data`. + +Request bodies are limited in line with Peppol, whose access points must +accept messages of up to 100 MB including embedded files. `/v0/import` +accepts up to 100 MB, and `/v0/export` up to 75 MB including attachments, as +embedded files grow by a third when encoded in base64, so the exported +document stays within 100 MB. Larger requests are refused with +`413 Request Entity Too Large`. The CLI has no limits. + ## Project structure ``` diff --git a/api/convert.go b/api/convert.go index d28a9e4..ecf8ca0 100644 --- a/api/convert.go +++ b/api/convert.go @@ -1,17 +1,50 @@ package api import ( + "bytes" + "encoding/json" + "errors" + "io" + "mime" + "mime/multipart" "net/http" + "strconv" "strings" + "github.com/invopop/gobl" "github.com/invopop/gobl.dev/internal/ops" "github.com/invopop/gobl/cbc" + "github.com/invopop/gobl/convert" ) // FormatHeader is the response header that names the key of the format an // export was converted into, or an import was detected as. const FormatHeader = "GOBL-Format" +// AttachmentsHeader is the response header that gives the number of files +// embedded in an imported document. +const AttachmentsHeader = "GOBL-Attachments" + +// Request body limits for conversions, in line with Peppol, whose access +// points must accept messages of up to 100 MB. Exports are limited further, +// as embedded files grow by a third when encoded in base64, so the exported +// document stays within 100 MB. +var ( + importBodyLimit int64 = 100 << 20 + exportBodyLimit int64 = 75 << 20 +) + +// multipartMemory is the size of a multipart export request kept in memory. +// Larger files are stored temporarily on disk while the request is handled. +const multipartMemory = 32 << 20 + +// importResponse is the import response when attachments are requested. +type importResponse struct { + Format cbc.Key `json:"format"` + Envelope *gobl.Envelope `json:"envelope"` + Attachments []*convert.Attachment `json:"attachments"` +} + func handleFormatList(w http.ResponseWriter, _ *http.Request) { WriteJSON(w, map[string]any{"formats": ops.Formats()}) } @@ -29,28 +62,70 @@ func handleFormatCheck(w http.ResponseWriter, r *http.Request) { WriteJSON(w, map[string]any{"formats": checks}) } +// handleImport converts the document in the body into a GOBL envelope. With +// the "attachments" query parameter, the response also includes the files +// embedded in the document. func handleImport(w http.ResponseWriter, r *http.Request) { + body := limitBody(w, r, importBodyLimit) + withAttachments := false + if v := r.URL.Query().Get("attachments"); v != "" { + b, err := strconv.ParseBool(v) + if err != nil { + WriteError(w, gobl.ErrInput.WithReason("invalid attachments parameter: %s", v)) + return + } + withAttachments = b + } res, err := ops.Import(r.Context(), &ops.ImportOptions{ Input: r.Body, Formats: queryFormats(r), }) if err != nil { - WriteError(w, err) + body.writeError(w, err) return } w.Header().Set(FormatHeader, res.Format.Key.String()) - WriteJSON(w, res.Envelope) + w.Header().Set(AttachmentsHeader, strconv.Itoa(len(res.Attachments))) + if !withAttachments { + WriteJSON(w, res.Envelope) + return + } + atts := res.Attachments + if atts == nil { + atts = []*convert.Attachment{} + } + WriteJSON(w, &importResponse{ + Format: res.Format.Key, + Envelope: res.Envelope, + Attachments: atts, + }) } +// handleExport converts the GOBL document in the body into the first of the +// requested formats that accepts it. A multipart body provides the document +// in its "document" part and files to embed in "attachment" parts. func handleExport(w http.ResponseWriter, r *http.Request) { - out, err := ops.Export(r.Context(), &ops.ExportOptions{ + body := limitBody(w, r, exportBodyLimit) + opts := &ops.ExportOptions{ ParseOptions: &ops.ParseOptions{ Input: r.Body, }, Formats: queryFormats(r), - }) + } + if mt, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")); mt == "multipart/form-data" { + if err := r.ParseMultipartForm(multipartMemory); err != nil { + body.writeError(w, gobl.ErrInput.WithReason("invalid multipart body: %s", err)) + return + } + defer r.MultipartForm.RemoveAll() //nolint:errcheck + if err := readExportForm(r.MultipartForm, opts); err != nil { + WriteError(w, err) + return + } + } + out, err := ops.Export(r.Context(), opts) if err != nil { - WriteError(w, err) + body.writeError(w, err) return } mime := out.Format.MIME @@ -62,6 +137,82 @@ func handleExport(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(out.Data) //nolint:errcheck } +// limitedBody limits a request body, recording whether it was larger than +// the limit, as errors from reading it may not keep the cause. +type limitedBody struct { + io.ReadCloser + limit int64 + exceeded bool +} + +// limitBody limits the request's body to n bytes. +func limitBody(w http.ResponseWriter, r *http.Request, n int64) *limitedBody { + b := &limitedBody{ReadCloser: http.MaxBytesReader(w, r.Body, n), limit: n} + r.Body = b + return b +} + +func (b *limitedBody) Read(p []byte) (int, error) { + n, err := b.ReadCloser.Read(p) + var mbe *http.MaxBytesError + if errors.As(err, &mbe) { + b.exceeded = true + } + return n, err +} + +// writeError writes the error, or a 413 response when the body was larger +// than the limit. +func (b *limitedBody) writeError(w http.ResponseWriter, err error) { + if !b.exceeded { + WriteError(w, err) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusRequestEntityTooLarge) + ge := gobl.ErrInput.WithReason("request body is larger than the %d MB limit", b.limit>>20) + _ = json.NewEncoder(w).Encode(ge) //nolint:errcheck +} + +// readExportForm reads the document and the attachments of a multipart +// export request. The document may be a field or a file. +func readExportForm(form *multipart.Form, opts *ops.ExportOptions) error { + if v := form.Value["document"]; len(v) > 0 { + opts.Input = strings.NewReader(v[0]) + } else if f := form.File["document"]; len(f) > 0 { + data, err := readFormFile(f[0]) + if err != nil { + return err + } + opts.Input = bytes.NewReader(data) + } else { + return gobl.ErrInput.WithReason("multipart body has no document part") + } + for _, fh := range form.File["attachment"] { + data, err := readFormFile(fh) + if err != nil { + return err + } + opts.Attachments = append(opts.Attachments, + ops.NewAttachment(fh.Filename, fh.Header.Get("Content-Type"), data), + ) + } + return nil +} + +func readFormFile(fh *multipart.FileHeader) ([]byte, error) { + f, err := fh.Open() + if err != nil { + return nil, gobl.ErrInput.WithCause(err) + } + defer f.Close() //nolint:errcheck + data, err := io.ReadAll(f) + if err != nil { + return nil, gobl.ErrInput.WithCause(err) + } + return data, nil +} + // queryFormats reads the format keys from the "format" query parameter, which // may be repeated or hold a comma-separated list, preserving their order. // Keys contain "+", which query strings decode as a space, so unescaped keys diff --git a/api/convert_test.go b/api/convert_test.go index 5ba8083..7d1552f 100644 --- a/api/convert_test.go +++ b/api/convert_test.go @@ -1,8 +1,10 @@ package api_test import ( + "bytes" "encoding/json" "io" + "mime/multipart" "net/http" "net/http/httptest" "os" @@ -147,3 +149,175 @@ func TestFormatCheckEndpoint(t *testing.T) { defer resp.Body.Close() //nolint:errcheck assert.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode, "documents that do not build are rejected") } + +// multipartExport builds a multipart export body with the document as a +// field, or as a file when asFile is set, and the attachments as files. +func multipartExport(t *testing.T, doc []byte, asFile bool, files map[string][]byte) (*bytes.Buffer, string) { + t.Helper() + body := new(bytes.Buffer) + mw := multipart.NewWriter(body) + if asFile { + fw, err := mw.CreateFormFile("document", "invoice.json") + require.NoError(t, err) + _, err = fw.Write(doc) + require.NoError(t, err) + } else if doc != nil { + require.NoError(t, mw.WriteField("document", string(doc))) + } + for name, data := range files { + fw, err := mw.CreateFormFile("attachment", name) + require.NoError(t, err) + _, err = fw.Write(data) + require.NoError(t, err) + } + require.NoError(t, mw.Close()) + return body, mw.FormDataContentType() +} + +func TestExportImportAttachments(t *testing.T) { + srv := httptest.NewServer(api.NewHandler()) + defer srv.Close() + + doc, err := os.ReadFile("../internal/ops/testdata/convert-invoice.json") + require.NoError(t, err) + + for _, asFile := range []bool{false, true} { + body, ct := multipartExport(t, doc, asFile, map[string][]byte{"invoice.pdf": []byte("%PDF-1.7")}) + resp, err := http.Post(srv.URL+prefix+"/export?format=ubl+en16931-v2017", ct, body) + require.NoError(t, err) + defer resp.Body.Close() //nolint:errcheck + require.Equal(t, http.StatusOK, resp.StatusCode, "document as file: %v", asFile) + xml, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Contains(t, string(xml), `filename="invoice.pdf"`) + assert.Contains(t, string(xml), `mimeCode="application/pdf"`) + + // The envelope alone by default. + resp, err = http.Post(srv.URL+prefix+"/import", "application/xml", bytes.NewReader(xml)) + require.NoError(t, err) + defer resp.Body.Close() //nolint:errcheck + require.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "1", resp.Header.Get(api.AttachmentsHeader)) + env := new(gobl.Envelope) + require.NoError(t, json.NewDecoder(resp.Body).Decode(env)) + assert.NotNil(t, env.Extract()) + + // With the attachments when requested. + resp, err = http.Post(srv.URL+prefix+"/import?attachments=true", "application/xml", bytes.NewReader(xml)) + require.NoError(t, err) + defer resp.Body.Close() //nolint:errcheck + require.Equal(t, http.StatusOK, resp.StatusCode) + var res struct { + Format string `json:"format"` + Envelope *gobl.Envelope `json:"envelope"` + Attachments []json.RawMessage `json:"attachments"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&res)) + assert.Equal(t, "ubl+en16931-v2017", res.Format) + require.NotNil(t, res.Envelope) + _, ok := res.Envelope.Extract().(*bill.Invoice) + assert.True(t, ok) + require.Len(t, res.Attachments, 1) + var att struct { + Name string `json:"name"` + MIME string `json:"mime"` + Data []byte `json:"data"` + } + require.NoError(t, json.Unmarshal(res.Attachments[0], &att)) + assert.Equal(t, "invoice.pdf", att.Name) + assert.Equal(t, "application/pdf", att.MIME) + assert.Equal(t, []byte("%PDF-1.7"), att.Data) + } +} + +func TestImportAttachmentsEmpty(t *testing.T) { + srv := httptest.NewServer(api.NewHandler()) + defer srv.Close() + + doc, err := os.ReadFile("../internal/ops/testdata/convert-invoice.json") + require.NoError(t, err) + resp, err := http.Post(srv.URL+prefix+"/export?format=ubl+en16931-v2017", "application/json", bytes.NewReader(doc)) + require.NoError(t, err) + defer resp.Body.Close() //nolint:errcheck + xml, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + resp, err = http.Post(srv.URL+prefix+"/import?attachments=1", "application/xml", bytes.NewReader(xml)) + require.NoError(t, err) + defer resp.Body.Close() //nolint:errcheck + require.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "0", resp.Header.Get(api.AttachmentsHeader)) + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Contains(t, string(data), `"attachments":[]`) + + resp, err = http.Post(srv.URL+prefix+"/import?attachments=maybe", "application/xml", bytes.NewReader(xml)) + require.NoError(t, err) + defer resp.Body.Close() //nolint:errcheck + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) +} + +func TestExportMultipartErrors(t *testing.T) { + srv := httptest.NewServer(api.NewHandler()) + defer srv.Close() + + t.Run("no document", func(t *testing.T) { + body, ct := multipartExport(t, nil, false, map[string][]byte{"invoice.pdf": []byte("%PDF")}) + resp, err := http.Post(srv.URL+prefix+"/export?format=ubl+en16931-v2017", ct, body) + require.NoError(t, err) + defer resp.Body.Close() //nolint:errcheck + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Contains(t, string(data), "no document part") + }) + t.Run("malformed", func(t *testing.T) { + resp, err := http.Post(srv.URL+prefix+"/export?format=ubl+en16931-v2017", "multipart/form-data; boundary=x", strings.NewReader("garbage")) + require.NoError(t, err) + defer resp.Body.Close() //nolint:errcheck + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) + }) +} + +func TestConvertBodyLimits(t *testing.T) { + srv := httptest.NewServer(api.NewHandler()) + defer srv.Close() + defer api.SetBodyLimits(1<<20, 1<<20)() + + doc, err := os.ReadFile("../internal/ops/testdata/convert-invoice.json") + require.NoError(t, err) + large := bytes.Repeat([]byte("x"), 2<<20) + + assertTooLarge := func(t *testing.T, resp *http.Response) { + t.Helper() + defer resp.Body.Close() //nolint:errcheck + assert.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode) + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Contains(t, string(data), "larger than the 1 MB limit") + } + + t.Run("import", func(t *testing.T) { + resp, err := http.Post(srv.URL+prefix+"/import", "application/xml", bytes.NewReader(large)) + require.NoError(t, err) + assertTooLarge(t, resp) + }) + t.Run("export", func(t *testing.T) { + resp, err := http.Post(srv.URL+prefix+"/export?format=ubl+en16931-v2017", "application/json", bytes.NewReader(large)) + require.NoError(t, err) + assertTooLarge(t, resp) + }) + t.Run("export with attachments", func(t *testing.T) { + body, ct := multipartExport(t, doc, false, map[string][]byte{"large.pdf": large}) + resp, err := http.Post(srv.URL+prefix+"/export?format=ubl+en16931-v2017", ct, body) + require.NoError(t, err) + assertTooLarge(t, resp) + }) + t.Run("within the limit", func(t *testing.T) { + body, ct := multipartExport(t, doc, false, map[string][]byte{"small.pdf": []byte("%PDF")}) + resp, err := http.Post(srv.URL+prefix+"/export?format=ubl+en16931-v2017", ct, body) + require.NoError(t, err) + defer resp.Body.Close() //nolint:errcheck + assert.Equal(t, http.StatusOK, resp.StatusCode) + }) +} diff --git a/api/export_test.go b/api/export_test.go new file mode 100644 index 0000000..929fb86 --- /dev/null +++ b/api/export_test.go @@ -0,0 +1,11 @@ +package api + +// SetBodyLimits sets the import and export body limits for a test, and +// returns a function that restores them. +func SetBodyLimits(importLimit, exportLimit int64) func() { + i, e := importBodyLimit, exportBodyLimit + importBodyLimit, exportBodyLimit = importLimit, exportLimit + return func() { + importBodyLimit, exportBodyLimit = i, e + } +} diff --git a/api/openapi.json b/api/openapi.json index 18da000..345a0a8 100644 --- a/api/openapi.json +++ b/api/openapi.json @@ -219,7 +219,7 @@ "post": { "operationId": "import", "summary": "Import a document", - "description": "Detects the format of a document in an external format, such as UBL or CII, and converts it into a GOBL envelope.", + "description": "Detects the format of a document in an external format, such as UBL or CII, and converts it into a GOBL envelope. Files embedded in the document are counted in the GOBL-Attachments header, and returned with the envelope when requested. The body is limited to 100 MB.", "tags": ["Convert"], "parameters": [ { @@ -228,6 +228,13 @@ "description": "Format keys to limit detection to, comma-separated or repeated.", "schema": { "type": "string" }, "example": "ubl+en16931-v2017" + }, + { + "name": "attachments", + "in": "query", + "description": "When true, the response is an object with the format, the envelope and the files embedded in the document, instead of the envelope alone.", + "schema": { "type": "boolean" }, + "example": true } ], "requestBody": { @@ -240,20 +247,30 @@ }, "responses": { "200": { - "description": "Imported GOBL envelope", + "description": "Imported GOBL envelope, or with attachments=true, the import result", "headers": { "GOBL-Format": { "description": "Key of the format the document was detected as.", "schema": { "type": "string" } + }, + "GOBL-Attachments": { + "description": "Number of files embedded in the document.", + "schema": { "type": "integer" } } }, "content": { "application/json": { - "schema": {} + "schema": { + "oneOf": [ + { "description": "GOBL envelope" }, + { "$ref": "#/components/schemas/ImportResult" } + ] + } } } }, "400": { "$ref": "#/components/responses/Error" }, + "413": { "$ref": "#/components/responses/Error" }, "422": { "$ref": "#/components/responses/Error" } } } @@ -262,7 +279,7 @@ "post": { "operationId": "export", "summary": "Export a document", - "description": "Converts a GOBL document or envelope into the first of the requested formats that accepts it. Documents and unsigned envelopes are calculated and validated first.", + "description": "Converts a GOBL document or envelope into the first of the requested formats that accepts it. Documents and unsigned envelopes are calculated and validated first. A multipart body also provides files to embed, such as a PDF version of the invoice; formats that cannot embed them are skipped. The body is limited to 75 MB, so the exported document stays within 100 MB.", "tags": ["Convert"], "parameters": [ { @@ -279,6 +296,23 @@ "content": { "application/json": { "schema": {} + }, + "multipart/form-data": { + "schema": { + "type": "object", + "required": ["document"], + "properties": { + "document": { + "type": "string", + "description": "GOBL document or envelope, as a field or a file." + }, + "attachment": { + "type": "array", + "description": "Files to embed in the exported document.", + "items": { "type": "string", "format": "binary" } + } + } + } } } }, @@ -298,6 +332,7 @@ } }, "400": { "$ref": "#/components/responses/Error" }, + "413": { "$ref": "#/components/responses/Error" }, "422": { "$ref": "#/components/responses/Error" } } } @@ -725,6 +760,27 @@ } } }, + "Attachment": { + "type": "object", + "properties": { + "id": { "type": "string", "description": "Identifier of the file in the document." }, + "name": { "type": "string", "example": "invoice.pdf" }, + "description": { "type": "string" }, + "mime": { "type": "string", "example": "application/pdf" }, + "data": { "type": "string", "format": "byte", "description": "File contents, encoded in base64." } + } + }, + "ImportResult": { + "type": "object", + "properties": { + "format": { "type": "string", "example": "ubl+en16931-v2017" }, + "envelope": { "description": "Imported GOBL envelope." }, + "attachments": { + "type": "array", + "items": { "$ref": "#/components/schemas/Attachment" } + } + } + }, "FormatCheck": { "allOf": [ { "$ref": "#/components/schemas/Format" }, diff --git a/cmd/gobl/convert.go b/cmd/gobl/convert.go index 7297c9d..93aaba5 100644 --- a/cmd/gobl/convert.go +++ b/cmd/gobl/convert.go @@ -3,17 +3,25 @@ package main import ( "encoding/json" "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" "github.com/invopop/gobl.dev/internal/ops" "github.com/invopop/gobl/cbc" + gconvert "github.com/invopop/gobl/convert" "github.com/spf13/cobra" ) type convertOpts struct { *rootOpts - to []string // formats to export into, in order of preference - from []string // formats to limit import detection to - list bool + to []string // formats to export into, in order of preference + from []string // formats to limit import detection to + list bool + attach []string // files to embed on export + attachments string // directory to save embedded files to on import } func convert(root *rootOpts) *convertOpts { @@ -33,9 +41,15 @@ bundled converters, such as UBL and CII. With --to, the input GOBL document or envelope is exported into the first of the formats given that accepts it. Without --to, the format of the input is detected -and it is imported into a GOBL envelope. Use --list to see the available formats.`, +and it is imported into a GOBL envelope. Use --list to see the available formats. + +On export, --attach embeds files, such as a PDF version of the invoice, in +formats that support attachments. On import, --attachments saves the files +embedded in the document to a directory.`, Example: ` gobl convert invoice.json invoice.xml --to ubl+en16931-v2017 + gobl convert invoice.json invoice.xml --to ubl+peppol+bis-v3 --attach invoice.pdf gobl convert invoice.xml -i + gobl convert invoice.xml invoice.json --attachments ./files gobl convert --list -i`, } @@ -43,6 +57,8 @@ and it is imported into a GOBL envelope. Use --list to see the available formats f.StringSliceVarP(&o.to, "to", "t", nil, "Format keys to export into, in order of preference.") f.StringSliceVar(&o.from, "from", nil, "Format keys to limit import detection to.") f.BoolVarP(&o.list, "list", "l", false, "List the available formats.") + f.StringArrayVar(&o.attach, "attach", nil, "File to embed in the exported document. May be repeated.") + f.StringVar(&o.attachments, "attachments", "", "Directory to save the files embedded in an imported document to.") return cmd } @@ -53,6 +69,12 @@ func (o *convertOpts) runE(cmd *cobra.Command, args []string) error { if len(o.to) > 0 && len(o.from) > 0 { return errors.New("--to and --from cannot be used together") } + if len(o.attach) > 0 && len(o.to) == 0 { + return errors.New("--attach requires --to") + } + if o.attachments != "" && len(o.to) > 0 { + return errors.New("--attachments cannot be used with --to") + } if o.list { return o.encode(cmd, args, ops.Formats()) @@ -72,14 +94,24 @@ func (o *convertOpts) runE(cmd *cobra.Command, args []string) error { if err != nil { return err } + if o.attachments != "" { + if err := saveAttachments(o.attachments, res.Attachments); err != nil { + return err + } + } return o.encode(cmd, args, res.Envelope) } + atts, err := loadAttachments(o.attach) + if err != nil { + return err + } res, err := ops.Export(ctx, &ops.ExportOptions{ ParseOptions: &ops.ParseOptions{ Input: input, }, - Formats: formatKeys(o.to), + Formats: formatKeys(o.to), + Attachments: atts, }) if err != nil { return err @@ -116,3 +148,50 @@ func formatKeys(list []string) []cbc.Key { } return keys } + +// loadAttachments reads the files to embed in an exported document. +func loadAttachments(paths []string) ([]*gconvert.Attachment, error) { + atts := make([]*gconvert.Attachment, 0, len(paths)) + for _, p := range paths { + data, err := os.ReadFile(p) + if err != nil { + return nil, err + } + atts = append(atts, ops.NewAttachment(p, "", data)) + } + return atts, nil +} + +// saveAttachments writes the files embedded in an imported document to the +// directory. Only the base of each name is used, as names come from the +// document, and repeated names are numbered. +func saveAttachments(dir string, atts []*gconvert.Attachment) error { + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + used := make(map[string]bool) + for i, a := range atts { + name := attachmentFileName(a, i) + base, ext := strings.TrimSuffix(name, filepath.Ext(name)), filepath.Ext(name) + for n := 2; used[name]; n++ { + name = fmt.Sprintf("%s-%d%s", base, n, ext) + } + used[name] = true + if err := os.WriteFile(filepath.Join(dir, name), a.Data, 0o644); err != nil { + return err + } + } + return nil +} + +// attachmentFileName provides a file name for an attachment from its name or +// ID, without any directories. +func attachmentFileName(a *gconvert.Attachment, i int) string { + for _, n := range []string{a.Name, a.ID} { + n = path.Base(strings.ReplaceAll(n, `\`, "/")) + if n != "" && n != "." && n != ".." && n != "/" { + return n + } + } + return fmt.Sprintf("attachment-%d", i+1) +} diff --git a/cmd/gobl/convert_test.go b/cmd/gobl/convert_test.go index 39650ee..7f8105e 100644 --- a/cmd/gobl/convert_test.go +++ b/cmd/gobl/convert_test.go @@ -3,10 +3,13 @@ package main import ( "bytes" "encoding/json" + "os" + "path/filepath" "testing" "github.com/invopop/gobl" "github.com/invopop/gobl/bill" + gconvert "github.com/invopop/gobl/convert" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -72,4 +75,59 @@ func Test_convert(t *testing.T) { _, err := runConvert(t, nil, "--to", "a", "--from", "b") assert.EqualError(t, err, "--to and --from cannot be used together") }) + + t.Run("attachments", func(t *testing.T) { + dir := t.TempDir() + pdf := filepath.Join(dir, "invoice.pdf") + require.NoError(t, os.WriteFile(pdf, []byte("%PDF-1.7"), 0o644)) + + xml, err := runConvert(t, nil, "testdata/convert-invoice.json", "--to", "ubl+en16931-v2017", "--attach", pdf) + require.NoError(t, err) + assert.Contains(t, string(xml), `mimeCode="application/pdf"`) + assert.Contains(t, string(xml), `filename="invoice.pdf"`) + + out := filepath.Join(dir, "out") + _, err = runConvert(t, xml, "--attachments", out) + require.NoError(t, err) + data, err := os.ReadFile(filepath.Join(out, "invoice.pdf")) + require.NoError(t, err) + assert.Equal(t, []byte("%PDF-1.7"), data) + }) + + t.Run("attachment not found", func(t *testing.T) { + _, err := runConvert(t, nil, "testdata/convert-invoice.json", "--to", "ubl+en16931-v2017", "--attach", "missing.pdf") + assert.ErrorContains(t, err, "missing.pdf") + }) + + t.Run("attach without to", func(t *testing.T) { + _, err := runConvert(t, nil, "--attach", "invoice.pdf") + assert.EqualError(t, err, "--attach requires --to") + }) + + t.Run("attachments with to", func(t *testing.T) { + _, err := runConvert(t, nil, "--to", "a", "--attachments", "files") + assert.EqualError(t, err, "--attachments cannot be used with --to") + }) +} + +func Test_saveAttachments(t *testing.T) { + dir := t.TempDir() + atts := []*gconvert.Attachment{ + {Name: "../../escape.pdf", Data: []byte("1")}, + {Name: `C:\files\escape.pdf`, Data: []byte("2")}, + {ID: "doc-3", Data: []byte("3")}, + {Name: "..", Data: []byte("4")}, + } + require.NoError(t, saveAttachments(dir, atts)) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + names := make([]string, len(entries)) + for i, e := range entries { + names[i] = e.Name() + } + assert.ElementsMatch(t, []string{"escape.pdf", "escape-2.pdf", "doc-3", "attachment-4"}, names) + data, err := os.ReadFile(filepath.Join(dir, "escape-2.pdf")) + require.NoError(t, err) + assert.Equal(t, []byte("2"), data) } diff --git a/internal/ops/convert.go b/internal/ops/convert.go index cf7a059..930def3 100644 --- a/internal/ops/convert.go +++ b/internal/ops/convert.go @@ -5,7 +5,11 @@ import ( "encoding/json" "errors" "io" + "mime" + "path" + "path/filepath" "slices" + "strings" "github.com/invopop/gobl" "github.com/invopop/gobl/cbc" @@ -27,6 +31,31 @@ type ExportOptions struct { // Formats lists the keys of the formats to export into, in order of // preference. The first one that accepts the document is used. Formats []cbc.Key + // Attachments are files to embed in the exported document. Formats that + // cannot embed them are skipped. + Attachments []*convert.Attachment +} + +// NewAttachment prepares a file to embed in an exported document. The MIME +// type is taken from the name's extension when not given, and directories +// are removed from the name. +func NewAttachment(name, mimeType string, data []byte) *convert.Attachment { + name = path.Base(strings.ReplaceAll(name, `\`, "/")) + if name == "." || name == "/" { + name = "" + } + if mimeType == "" || mimeType == "application/octet-stream" { + if t := mime.TypeByExtension(filepath.Ext(name)); t != "" { + mimeType = t + } + } + if t, _, err := mime.ParseMediaType(mimeType); err == nil { + mimeType = t + } + if mimeType == "" { + mimeType = "application/octet-stream" + } + return &convert.Attachment{Name: name, MIME: mimeType, Data: data} } // Formats lists every format registered with GOBL's convert package by the @@ -73,7 +102,10 @@ func Export(ctx context.Context, opts *ExportOptions) (*convert.ExportResult, er if err != nil { return nil, err } - out, err := convert.Export(env, opts.Formats[0], convert.WithAlternatives(opts.Formats[1:]...)) + out, err := convert.Export(env, opts.Formats[0], + convert.WithAlternatives(opts.Formats[1:]...), + convert.WithAttachments(opts.Attachments...), + ) if err != nil { return nil, gobl.ErrInternal.WithCause(err) } diff --git a/internal/ops/convert_test.go b/internal/ops/convert_test.go index 7a7bf48..8fa2950 100644 --- a/internal/ops/convert_test.go +++ b/internal/ops/convert_test.go @@ -115,3 +115,55 @@ func TestImport(t *testing.T) { assert.ErrorIs(t, err, convert.ErrUnknownFormat) }) } + +func TestExportAttachments(t *testing.T) { + ctx := context.Background() + pdf := NewAttachment("invoice.pdf", "", []byte("%PDF-1.7")) + + t.Run("embedded and imported again", func(t *testing.T) { + out, err := Export(ctx, &ExportOptions{ + ParseOptions: &ParseOptions{Input: testFileReader(t, "testdata/convert-invoice.json")}, + Formats: []cbc.Key{"cii+en16931-v2017"}, + Attachments: []*convert.Attachment{pdf}, + }) + require.NoError(t, err) + + res, err := Import(ctx, &ImportOptions{Input: bytes.NewReader(out.Data)}) + require.NoError(t, err) + require.Len(t, res.Attachments, 1) + assert.Equal(t, "invoice.pdf", res.Attachments[0].Name) + assert.Equal(t, "application/pdf", res.Attachments[0].MIME) + assert.Equal(t, []byte("%PDF-1.7"), res.Attachments[0].Data) + }) + + t.Run("refused by formats that cannot embed them", func(t *testing.T) { + _, err := Export(ctx, &ExportOptions{ + ParseOptions: &ParseOptions{Input: testFileReader(t, "testdata/convert-invoice.json")}, + Formats: []cbc.Key{"es-facturae-v3"}, + Attachments: []*convert.Attachment{pdf}, + }) + assert.ErrorIs(t, err, convert.ErrNotSupported) + assert.ErrorContains(t, err, "format es-facturae-v3 does not embed attachments") + }) +} + +func TestNewAttachment(t *testing.T) { + tests := []struct { + name, mime string + wantName, want string + }{ + {"invoice.pdf", "", "invoice.pdf", "application/pdf"}, + {"dir/invoice.pdf", "", "invoice.pdf", "application/pdf"}, + {`C:\files\invoice.pdf`, "", "invoice.pdf", "application/pdf"}, + {"invoice.pdf", "application/octet-stream", "invoice.pdf", "application/pdf"}, + {"data.bin", "text/csv; charset=utf-8", "data.bin", "text/csv"}, + {"data.unknownext", "", "data.unknownext", "application/octet-stream"}, + {"", "", "", "application/octet-stream"}, + } + for _, tt := range tests { + a := NewAttachment(tt.name, tt.mime, []byte("x")) + assert.Equal(t, tt.wantName, a.Name, tt.name) + assert.Equal(t, tt.want, a.MIME, tt.name) + assert.Equal(t, []byte("x"), a.Data) + } +}