diff --git a/.woodpecker.star b/.woodpecker.star index 978a1dfdfa..e5cd56303f 100644 --- a/.woodpecker.star +++ b/.woodpecker.star @@ -8,7 +8,7 @@ docker_repo_slug = "opencloudeu/opencloud" # images ALPINE_GIT = "alpine/git:latest" -APACHE_TIKA = "apache/tika:3.2.3.0-full" +APACHE_TIKA = "apache/tika:4.0.0-full" CHKO_DOCKER_PUSHRM = "chko/docker-pushrm:1" CODACY_COVERAGE_REPORTER = "codacy/codacy-coverage-reporter:14.1.3" COLLABORA_CODE = "collabora/code:24.04.5.1.1" @@ -3442,6 +3442,9 @@ def tikaService(): return [{ "name": "tika", "image": APACHE_TIKA, + # tika 4 discovers its plugins relative to the image working directory, + # the workspace default would leave the pipes fetchers empty + "directory": "/opt/tika-server", "detach": True, }, { "name": "wait-for-tika-service", diff --git a/services/search/README.md b/services/search/README.md index 026f4fc5ae..96bdc00c21 100644 --- a/services/search/README.md +++ b/services/search/README.md @@ -99,7 +99,7 @@ It does not do any further content analysis. The main difference is that this extractor is able to analyze and extract data from more advanced file types like PDF, DOCX, PPTX, etc. However, [Apache Tika](https://tika.apache.org/) is required for this task. -Read the [Getting Started with Apache Tika](https://tika.apache.org/2.6.0/gettingstarted.html) guide on how to install and run Tika or use a ready to run [Tika container](https://hub.docker.com/r/apache/tika). +Read the [Getting Started with Apache Tika](https://tika.apache.org/) guide on how to install and run Tika or use a ready to run [Tika container](https://hub.docker.com/r/apache/tika). See the [Tika container usage document](https://github.com/apache/tika-docker#usage) for a quickstart. As soon as Tika is installed and configured, the search service needs to be told to use it. diff --git a/services/search/pkg/content/extractor.go b/services/search/pkg/content/extractor.go index a91be6cb4e..3d4e457288 100644 --- a/services/search/pkg/content/extractor.go +++ b/services/search/pkg/content/extractor.go @@ -23,7 +23,7 @@ func getFirstValue(m map[string][]string, key string) (string, error) { return "", fmt.Errorf("unknown key: %v", key) } - if len(m) == 0 { + if len(v) == 0 { return "", fmt.Errorf("no values for: %v", key) } diff --git a/services/search/pkg/content/tika.go b/services/search/pkg/content/tika.go index 2b475efb17..619744fed6 100644 --- a/services/search/pkg/content/tika.go +++ b/services/search/pkg/content/tika.go @@ -3,15 +3,13 @@ package content import ( "context" "fmt" - "math" - "strconv" + "io" + "net/http" "strings" - "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/google/go-tika/tika" - libregraph "github.com/opencloud-eu/libre-graph-api-go" "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" "github.com/opencloud-eu/opencloud/pkg/log" @@ -24,6 +22,7 @@ type Tika struct { *Basic Retriever tika *tika.Client + tikaURL string ContentExtractionSizeLimit uint64 CleanStopWords bool } @@ -46,6 +45,7 @@ func NewTikaExtractor(gatewaySelector pool.Selectable[gateway.GatewayAPIClient], Basic: basic, Retriever: newCS3Retriever(gatewaySelector, logger, cfg.Extractor.CS3AllowInsecure), tika: tika.NewClient(nil, cfg.Extractor.Tika.TikaURL), + tikaURL: cfg.Extractor.Tika.TikaURL, ContentExtractionSizeLimit: cfg.ContentExtractionSizeLimit, CleanStopWords: cfg.Extractor.Tika.CleanStopWords, }, nil @@ -91,218 +91,54 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, doc.Title = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Title, title)) } - if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil { + // tika 4 renamed the meta prefix from X-TIKA: to tk: + if content, err := getFirstValue(meta, "tk:content"); err == nil { + doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content)) + } else if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil { doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content)) } - doc.Location = t.getLocation(meta) - doc.Image = t.getImage(meta) - doc.Photo = t.getPhoto(meta) - - if contentType, err := getFirstValue(meta, "Content-Type"); err == nil && strings.HasPrefix(contentType, "audio/") { - doc.Audio = t.getAudio(meta) - } - } - - if langCode, _ := t.tika.LanguageString(ctx, doc.Content); langCode != "" && t.CleanStopWords { - doc.Content = CleanString(doc.Content, langCode) - } - - return doc, nil -} - -func (t Tika) getImage(meta map[string][]string) *libregraph.Image { - var image *libregraph.Image - initImage := func() { - if image == nil { - image = libregraph.NewImage() - } - } - - if v, err := getFirstValue(meta, "tiff:ImageWidth"); err == nil { - if i, err := strconv.ParseInt(v, 0, 32); err == nil { - initImage() - image.SetWidth(int32(i)) - } - } - - if v, err := getFirstValue(meta, "tiff:ImageLength"); err == nil { - if i, err := strconv.ParseInt(v, 0, 32); err == nil { - initImage() - image.SetHeight(int32(i)) - } - } - - return image -} - -func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates { - var location *libregraph.GeoCoordinates - initLocation := func() { - if location == nil { - location = libregraph.NewGeoCoordinates() - } - } - - // TODO: location.Altitute: transform the following data to … feet above sea level. - // "GPS:GPS Altitude": []string{"227.4 metres"}, - // "GPS:GPS Altitude Ref": []string{"Sea level"}, - - if v, err := getFirstValue(meta, "geo:lat"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initLocation() - location.SetLatitude(i) - } - } - - if v, err := getFirstValue(meta, "geo:long"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initLocation() - location.SetLongitude(i) - } - } - - return location -} - -func (t Tika) getPhoto(meta map[string][]string) *libregraph.Photo { - var photo *libregraph.Photo - initPhoto := func() { - if photo == nil { - photo = libregraph.NewPhoto() - } - } - - if v, err := getFirstValue(meta, "tiff:Make"); err == nil { - initPhoto() - photo.SetCameraMake(v) - } - - if v, err := getFirstValue(meta, "tiff:Model"); err == nil { - initPhoto() - photo.SetCameraModel(v) - } - - if v, err := getFirstValue(meta, "exif:FNumber"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initPhoto() - photo.SetFNumber(i) - } - } - - if v, err := getFirstValue(meta, "exif:FocalLength"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initPhoto() - photo.SetFocalLength(i) + // keep facets from earlier entries, an embedded resource's meta + // (e.g. cover art) must not reset them + if v := t.getLocation(meta); v != nil { + doc.Location = v } - } - - if v, err := getFirstValue(meta, "Base ISO"); err == nil { - if i, err := strconv.ParseInt(v, 0, 32); err == nil { - initPhoto() - photo.SetIso(int32(i)) + if v := t.getImage(meta); v != nil { + doc.Image = v } - } - - if v, err := getFirstValue(meta, "tiff:Orientation"); err == nil { - if i, err := strconv.ParseInt(v, 0, 32); err == nil { - initPhoto() - photo.SetOrientation(int32(i)) + if v := t.getPhoto(meta); v != nil { + doc.Photo = v } - } - - if v, err := getFirstValue(meta, "exif:DateTimeOriginal"); err == nil { - layout := "2006-01-02T15:04:05" - if t, err := time.Parse(layout, v); err == nil { - initPhoto() - photo.SetTakenDateTime(t) + if v := t.getAudio(meta); v != nil { + doc.Audio = v } } - if v, err := getFirstValue(meta, "exif:ExposureTime"); err == nil { - if i, err := strconv.ParseFloat(v, 64); err == nil { - initPhoto() - photo.SetExposureNumerator(1) - photo.SetExposureDenominator(math.Round(1 / i)) - } + if langCode := t.detectLanguage(ctx, doc.Content); langCode != "" && t.CleanStopWords { + doc.Content = CleanString(doc.Content, langCode) } - return photo + return doc, nil } -func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { - var audio *libregraph.Audio - initAudio := func() { - if audio == nil { - audio = libregraph.NewAudio() - } - } - - if v, err := getFirstValue(meta, "xmpDM:album"); err == nil { - initAudio() - audio.SetAlbum(v) - } - - if v, err := getFirstValue(meta, "xmpDM:albumArtist"); err == nil { - initAudio() - audio.SetAlbumArtist(v) - } - - if v, err := getFirstValue(meta, "xmpDM:artist"); err == nil { - initAudio() - audio.SetArtist(v) - } - - // TODO: audio.Bitrate: not provided by tika - // TODO: audio.Composers: not provided by tika - // TODO: audio.Copyright: not provided by tika for audio files? - - if v, err := getFirstValue(meta, "xmpDM:discNumber"); err == nil { - if i, err := strconv.ParseInt(v, 10, 32); err == nil { - initAudio() - audio.SetDisc(int32(i)) - } - - } - - // TODO: audio.DiscCount: not provided by tika - - if v, err := getFirstValue(meta, "xmpDM:duration"); err == nil { - // Tika emits fractional seconds. - if f, err := strconv.ParseFloat(v, 64); err == nil { - initAudio() - audio.SetDuration(int64(math.Round(f * 1000))) +// detectLanguage asks tika for the language of content. Tika 4 moved the +// endpoint from /language/string to /language, so try the new path first and +// fall back for an older tika. +func (t Tika) detectLanguage(ctx context.Context, content string) string { + for _, path := range []string{"/language", "/language/string"} { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, t.tikaURL+path, strings.NewReader(content)) + if err != nil { + return "" } - } - - if v, err := getFirstValue(meta, "xmpDM:genre"); err == nil { - initAudio() - audio.SetGenre(v) - } - - // TODO: audio.HasDrm: not provided by tika - // TODO: audio.IsVariableBitrate: not provided by tika - - if v, err := getFirstValue(meta, "dc:title"); err == nil { - initAudio() - audio.SetTitle(v) - } - - if v, err := getFirstValue(meta, "xmpDM:trackNumber"); err == nil { - if i, err := strconv.ParseInt(v, 10, 32); err == nil { - initAudio() - audio.SetTrack(int32(i)) + res, err := http.DefaultClient.Do(req) + if err != nil { + return "" } - } - - // TODO: audio.TrackCount: not provided by tika - - if v, err := getFirstValue(meta, "xmpDM:releaseDate"); err == nil { - if i, err := strconv.ParseInt(v, 10, 32); err == nil { - initAudio() - audio.SetYear(int32(i)) + lang, err := io.ReadAll(res.Body) + _ = res.Body.Close() + if err == nil && res.StatusCode == http.StatusOK && len(lang) > 0 { + return string(lang) } } - - return audio + return "" } diff --git a/services/search/pkg/content/tika_audio.go b/services/search/pkg/content/tika_audio.go new file mode 100644 index 0000000000..6b6b088472 --- /dev/null +++ b/services/search/pkg/content/tika_audio.go @@ -0,0 +1,130 @@ +package content + +import ( + "math" + "strconv" + "strings" + "time" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio { + // generic keys like dc:title show up on any document, only audio + // content types carry the audio facet + if v, err := getFirstValue(meta, "Content-Type"); err != nil || !strings.HasPrefix(v, "audio/") { + return nil + } + + var audio *libregraph.Audio + initAudio := func() { + if audio == nil { + audio = libregraph.NewAudio() + } + } + + if v, err := getFirstValue(meta, "xmpDM:album"); err == nil { + initAudio() + audio.SetAlbum(v) + } + + if v, err := getFirstValue(meta, "xmpDM:albumArtist"); err == nil { + initAudio() + audio.SetAlbumArtist(v) + } + + if v, err := getFirstValue(meta, "xmpDM:artist"); err == nil { + initAudio() + audio.SetArtist(v) + } + + if v, err := getFirstValue(meta, "audio:bitrate"); err == nil { + // tika emits bits per second, graph wants kbps + if bps, err := strconv.ParseInt(v, 10, 64); err == nil && bps > 0 { + initAudio() + audio.SetBitrate(int64(math.Round(float64(bps) / 1000))) + } + } + + if v, err := getFirstValue(meta, "xmpDM:composer"); err == nil { + initAudio() + audio.SetComposers(v) + } + + if v, err := getFirstValue(meta, "xmpDM:copyright"); err == nil { + initAudio() + audio.SetCopyright(v) + } + + if v, err := getFirstValue(meta, "xmpDM:discNumber"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetDisc(int32(i)) + } + } + + if v, err := getFirstValue(meta, "audio:disc-count"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetDiscCount(int32(i)) + } + } + + if v, err := getFirstValue(meta, "xmpDM:duration"); err == nil { + // Tika emits fractional seconds. + if f, err := strconv.ParseFloat(v, 64); err == nil { + initAudio() + audio.SetDuration(int64(math.Round(f * 1000))) + } + } + + if v, err := getFirstValue(meta, "xmpDM:genre"); err == nil { + initAudio() + audio.SetGenre(v) + } + + if v, err := getFirstValue(meta, "audio:has-drm"); err == nil { + if b, err := strconv.ParseBool(v); err == nil { + initAudio() + audio.SetHasDrm(b) + } + } + + if v, err := getFirstValue(meta, "audio:is-variable-bitrate"); err == nil { + if b, err := strconv.ParseBool(v); err == nil { + initAudio() + audio.SetIsVariableBitrate(b) + } + } + + if v, err := getFirstValue(meta, "dc:title"); err == nil { + initAudio() + audio.SetTitle(v) + } + + if v, err := getFirstValue(meta, "xmpDM:trackNumber"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetTrack(int32(i)) + } + } + + if v, err := getFirstValue(meta, "audio:track-count"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetTrackCount(int32(i)) + } + } + + if v, err := getFirstValue(meta, "xmpDM:releaseDate"); err == nil { + if i, err := strconv.ParseInt(v, 10, 32); err == nil { + initAudio() + audio.SetYear(int32(i)) + } else if d, err := time.Parse(time.DateOnly, v); err == nil { + initAudio() + audio.SetYear(int32(d.Year())) + } + } + + return audio +} diff --git a/services/search/pkg/content/tika_audio_test.go b/services/search/pkg/content/tika_audio_test.go new file mode 100644 index 0000000000..02f85719f1 --- /dev/null +++ b/services/search/pkg/content/tika_audio_test.go @@ -0,0 +1,80 @@ +package content + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +var _ = Describe("getAudio", func() { + It("maps the audio metadata to the audio facet", func() { + meta := map[string][]string{ + "Content-Type": {"audio/mpeg"}, + "xmpDM:genre": {"Some Genre"}, + "xmpDM:album": {"Some Album"}, + "xmpDM:trackNumber": {"7"}, + "xmpDM:discNumber": {"4"}, + "xmpDM:releaseDate": {"2004"}, + "xmpDM:artist": {"Some Artist"}, + "xmpDM:albumArtist": {"Some AlbumArtist"}, + "dc:title": {"Some Title"}, + "xmpDM:duration": {"225.5"}, + "xmpDM:composer": {"Some Composers"}, + "xmpDM:copyright": {"Some Copyright"}, + "audio:bitrate": {"192000"}, + "audio:is-variable-bitrate": {"true"}, + "audio:has-drm": {"false"}, + "audio:track-count": {"9"}, + "audio:disc-count": {"5"}, + } + + audio := Tika{}.getAudio(meta) + Expect(audio).ToNot(BeNil()) + + Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album"))) + Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist"))) + Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist"))) + Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) + Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers"))) + Expect(audio.Copyright).To(Equal(libregraph.PtrString("Some Copyright"))) + Expect(audio.Disc).To(Equal(libregraph.PtrInt32(4))) + Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5))) + Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225500))) + Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre"))) + Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false))) + Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true))) + Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title"))) + Expect(audio.Track).To(Equal(libregraph.PtrInt32(7))) + Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(9))) + Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) + }) + + It("returns nil when no audio metadata is present", func() { + Expect(Tika{}.getAudio(map[string][]string{})).To(BeNil()) + }) + + It("returns nil for non-audio content types", func() { + Expect(Tika{}.getAudio(map[string][]string{ + "Content-Type": {"application/pdf"}, + "dc:title": {"quarterly report"}, + })).To(BeNil()) + }) + + It("takes the year from a full release date", func() { + audio := Tika{}.getAudio(map[string][]string{ + "Content-Type": {"audio/mpeg"}, + "xmpDM:releaseDate": {"2004-06-01"}, + }) + Expect(audio).ToNot(BeNil()) + Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) + }) + + It("rounds the bitrate to the nearest kbps", func() { + audio := Tika{}.getAudio(map[string][]string{ + "Content-Type": {"audio/mpeg"}, + "audio:bitrate": {"191999"}, + }) + Expect(audio).ToNot(BeNil()) + Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) + }) +}) diff --git a/services/search/pkg/content/tika_image.go b/services/search/pkg/content/tika_image.go new file mode 100644 index 0000000000..b5ce2a4037 --- /dev/null +++ b/services/search/pkg/content/tika_image.go @@ -0,0 +1,32 @@ +package content + +import ( + "strconv" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +func (t Tika) getImage(meta map[string][]string) *libregraph.Image { + var image *libregraph.Image + initImage := func() { + if image == nil { + image = libregraph.NewImage() + } + } + + if v, err := getFirstValue(meta, "tiff:ImageWidth"); err == nil { + if i, err := strconv.ParseInt(v, 0, 32); err == nil { + initImage() + image.SetWidth(int32(i)) + } + } + + if v, err := getFirstValue(meta, "tiff:ImageLength"); err == nil { + if i, err := strconv.ParseInt(v, 0, 32); err == nil { + initImage() + image.SetHeight(int32(i)) + } + } + + return image +} diff --git a/services/search/pkg/content/tika_image_test.go b/services/search/pkg/content/tika_image_test.go new file mode 100644 index 0000000000..91456ea97c --- /dev/null +++ b/services/search/pkg/content/tika_image_test.go @@ -0,0 +1,23 @@ +package content + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +var _ = Describe("getImage", func() { + It("maps the image dimensions to the image facet", func() { + image := Tika{}.getImage(map[string][]string{ + "tiff:ImageWidth": {"100"}, + "tiff:ImageLength": {"200"}, + }) + Expect(image).ToNot(BeNil()) + Expect(image.Width).To(Equal(libregraph.PtrInt32(100))) + Expect(image.Height).To(Equal(libregraph.PtrInt32(200))) + }) + + It("returns nil when no image metadata is present", func() { + Expect(Tika{}.getImage(map[string][]string{})).To(BeNil()) + }) +}) diff --git a/services/search/pkg/content/tika_location.go b/services/search/pkg/content/tika_location.go new file mode 100644 index 0000000000..586a4731c7 --- /dev/null +++ b/services/search/pkg/content/tika_location.go @@ -0,0 +1,52 @@ +package content + +import ( + "fmt" + "math" + "strconv" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +// graph geoCoordinates.altitude is in feet, exif GPS altitude (geo:alt) in metres. +const metresToFeet = 3.280839895 + +func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates { + // the facet needs a sane coordinate pair, an altitude alone is useless + lat, latErr := parseCoordinate(meta, "geo:lat", 90) + long, longErr := parseCoordinate(meta, "geo:long", 180) + if latErr != nil || longErr != nil { + return nil + } + + location := libregraph.NewGeoCoordinates() + location.SetLatitude(lat) + location.SetLongitude(long) + + // tika emits metres (negative below sea level), graph wants feet + if v, err := getFirstValue(meta, "geo:alt"); err == nil { + if metres, err := strconv.ParseFloat(v, 64); err == nil { + location.SetAltitude(metres * metresToFeet) + } + } + + return location +} + +func parseCoordinate(meta map[string][]string, key string, limit float64) (float64, error) { + v, err := getFirstValue(meta, key) + if err != nil { + return 0, err + } + + f, err := strconv.ParseFloat(v, 64) + if err != nil { + return 0, err + } + // ParseFloat accepts "NaN", which json cannot marshal + if math.IsNaN(f) || math.Abs(f) > limit { + return 0, fmt.Errorf("%s out of range: %v", key, f) + } + + return f, nil +} diff --git a/services/search/pkg/content/tika_location_test.go b/services/search/pkg/content/tika_location_test.go new file mode 100644 index 0000000000..413dae0c73 --- /dev/null +++ b/services/search/pkg/content/tika_location_test.go @@ -0,0 +1,48 @@ +package content + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +var _ = Describe("getLocation", func() { + It("maps lat/long and converts altitude from metres to feet", func() { + metres := 227.4 + location := Tika{}.getLocation(map[string][]string{ + "geo:lat": {"49.48675890884328"}, + "geo:long": {"11.103870357204285"}, + "geo:alt": {"227.4"}, + }) + Expect(location).ToNot(BeNil()) + Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328))) + Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285))) + Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(metres * metresToFeet))) + }) + + It("keeps below-sea-level altitude negative", func() { + metres := -227.4 + location := Tika{}.getLocation(map[string][]string{ + "geo:lat": {"31.5"}, + "geo:long": {"35.47"}, + "geo:alt": {"-227.4"}, + }) + Expect(location).ToNot(BeNil()) + Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(metres * metresToFeet))) + }) + + It("returns nil for an incomplete coordinate pair", func() { + Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"49.48"}})).To(BeNil()) + Expect(Tika{}.getLocation(map[string][]string{"geo:alt": {"227.4"}})).To(BeNil()) + }) + + It("returns nil for out-of-range coordinates", func() { + Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"91"}, "geo:long": {"11.1"}})).To(BeNil()) + Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"49.48"}, "geo:long": {"-180.5"}})).To(BeNil()) + Expect(Tika{}.getLocation(map[string][]string{"geo:lat": {"NaN"}, "geo:long": {"11.1"}})).To(BeNil()) + }) + + It("returns nil when no location metadata is present", func() { + Expect(Tika{}.getLocation(map[string][]string{})).To(BeNil()) + }) +}) diff --git a/services/search/pkg/content/tika_photo.go b/services/search/pkg/content/tika_photo.go new file mode 100644 index 0000000000..2ee961a300 --- /dev/null +++ b/services/search/pkg/content/tika_photo.go @@ -0,0 +1,74 @@ +package content + +import ( + "math" + "strconv" + "time" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +func (t Tika) getPhoto(meta map[string][]string) *libregraph.Photo { + var photo *libregraph.Photo + initPhoto := func() { + if photo == nil { + photo = libregraph.NewPhoto() + } + } + + if v, err := getFirstValue(meta, "tiff:Make"); err == nil { + initPhoto() + photo.SetCameraMake(v) + } + + if v, err := getFirstValue(meta, "tiff:Model"); err == nil { + initPhoto() + photo.SetCameraModel(v) + } + + if v, err := getFirstValue(meta, "exif:FNumber"); err == nil { + if i, err := strconv.ParseFloat(v, 64); err == nil { + initPhoto() + photo.SetFNumber(i) + } + } + + if v, err := getFirstValue(meta, "exif:FocalLength"); err == nil { + if i, err := strconv.ParseFloat(v, 64); err == nil { + initPhoto() + photo.SetFocalLength(i) + } + } + + if v, err := getFirstValue(meta, "Base ISO"); err == nil { + if i, err := strconv.ParseInt(v, 0, 32); err == nil { + initPhoto() + photo.SetIso(int32(i)) + } + } + + if v, err := getFirstValue(meta, "tiff:Orientation"); err == nil { + if i, err := strconv.ParseInt(v, 0, 32); err == nil { + initPhoto() + photo.SetOrientation(int32(i)) + } + } + + if v, err := getFirstValue(meta, "exif:DateTimeOriginal"); err == nil { + layout := "2006-01-02T15:04:05" + if t, err := time.Parse(layout, v); err == nil { + initPhoto() + photo.SetTakenDateTime(t) + } + } + + if v, err := getFirstValue(meta, "exif:ExposureTime"); err == nil { + if i, err := strconv.ParseFloat(v, 64); err == nil && i > 0 { + initPhoto() + photo.SetExposureNumerator(1) + photo.SetExposureDenominator(math.Round(1 / i)) + } + } + + return photo +} diff --git a/services/search/pkg/content/tika_photo_test.go b/services/search/pkg/content/tika_photo_test.go new file mode 100644 index 0000000000..9995d59587 --- /dev/null +++ b/services/search/pkg/content/tika_photo_test.go @@ -0,0 +1,38 @@ +package content + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" +) + +var _ = Describe("getPhoto", func() { + It("maps the exif metadata to the photo facet", func() { + photo := Tika{}.getPhoto(map[string][]string{ + "tiff:Make": {"Canon"}, + "tiff:Model": {"Canon EOS 5D"}, + "exif:ExposureTime": {"0.001"}, + "exif:FNumber": {"1.8"}, + "exif:FocalLength": {"50"}, + "Base ISO": {"100"}, + "tiff:Orientation": {"1"}, + "exif:DateTimeOriginal": {"2018-01-01T12:34:56"}, + }) + Expect(photo).ToNot(BeNil()) + Expect(photo.CameraMake).To(Equal(libregraph.PtrString("Canon"))) + Expect(photo.CameraModel).To(Equal(libregraph.PtrString("Canon EOS 5D"))) + Expect(photo.ExposureNumerator).To(Equal(libregraph.PtrFloat64(1))) + Expect(photo.ExposureDenominator).To(Equal(libregraph.PtrFloat64(1000))) + Expect(photo.FNumber).To(Equal(libregraph.PtrFloat64(1.8))) + Expect(photo.FocalLength).To(Equal(libregraph.PtrFloat64(50))) + Expect(photo.Iso).To(Equal(libregraph.PtrInt32(100))) + Expect(photo.Orientation).To(Equal(libregraph.PtrInt32(1))) + Expect(photo.TakenDateTime).To(Equal(libregraph.PtrTime(time.Date(2018, 1, 1, 12, 34, 56, 0, time.UTC)))) + }) + + It("returns nil when no photo metadata is present", func() { + Expect(Tika{}.getPhoto(map[string][]string{})).To(BeNil()) + }) +}) diff --git a/services/search/pkg/content/tika_test.go b/services/search/pkg/content/tika_test.go index 286ffa8b9c..772ed185b3 100644 --- a/services/search/pkg/content/tika_test.go +++ b/services/search/pkg/content/tika_test.go @@ -7,7 +7,6 @@ import ( "net/http" "net/http/httptest" "strings" - "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" . "github.com/onsi/ginkgo/v2" @@ -24,17 +23,19 @@ import ( var _ = Describe("Tika", func() { Describe("extract", func() { var ( - body string - fullResponse string - language string - version string - srv *httptest.Server - tika *content.Tika + body string + fullResponse string + language string + tika4Language bool + version string + srv *httptest.Server + tika *content.Tika ) BeforeEach(func() { body = "" language = "" + tika4Language = false version = "" fullResponse = "" srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -42,6 +43,13 @@ var _ = Describe("Tika", func() { switch req.URL.Path { case "/version": out = version + case "/language": + if tika4Language { + out = language + } else { + w.WriteHeader(http.StatusNotFound) + return + } case "/language/string": out = language case "/rmeta/text": @@ -102,6 +110,17 @@ var _ = Describe("Tika", func() { Expect(doc.Title).To(Equal("quarterly report")) }) + It("adds the content of a tika 4", func() { + fullResponse = `[{"tk:content": "some data"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Content).To(Equal("some data")) + }) + It("adds the title of an older tika", func() { fullResponse = `[{"title": "quarterly report"}]` @@ -113,141 +132,76 @@ var _ = Describe("Tika", func() { Expect(doc.Title).To(Equal("quarterly report")) }) - It("adds audio content", func() { - fullResponse = `[ - { - "xmpDM:genre": "Some Genre", - "xmpDM:album": "Some Album", - "xmpDM:trackNumber": "7", - "xmpDM:discNumber": "4", - "xmpDM:releaseDate": "2004", - "xmpDM:artist": "Some Artist", - "xmpDM:albumArtist": "Some AlbumArtist", - "xmpDM:audioCompressor": "MP3", - "xmpDM:audioChannelType": "Stereo", - "version": "MPEG 3 Layer III Version 1", - "xmpDM:logComment": "some comment", - "xmpDM:audioSampleRate": "44100", - "channels": "2", - "dc:title": "Some Title", - "xmpDM:duration": "225.5", - "Content-Type": "audio/mpeg", - "samplerate": "44100" - } - ]` + It("removes stop words with a tika 4 language endpoint", func() { + body = "body to test stop words!!! against almost everyone" + language = "en" + tika4Language = true + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ Type: provider.ResourceType_RESOURCE_TYPE_FILE, Size: 1, }) Expect(err).ToNot(HaveOccurred()) - - audio := doc.Audio - Expect(audio).ToNot(BeNil()) - - Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album"))) - Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist"))) - Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist"))) - // Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) - // Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers"))) - // Expect(audio.Copyright).To(Equal(libregraph.PtrString("Some Copyright"))) - Expect(audio.Disc).To(Equal(libregraph.PtrInt32(4))) - // Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5))) - Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225500))) - Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre"))) - // Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false))) - // Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true))) - Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title"))) - Expect(audio.Track).To(Equal(libregraph.PtrInt32(7))) - // Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(9))) - Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) - + Expect(doc.Content).To(Equal("body test stop words!!!")) }) - It("adds location content", func() { - fullResponse = `[ - { - "geo:lat": "49.48675890884328", - "geo:long": "11.103870357204285" - } - ]` + It("removes stop words", func() { + body = "body to test stop words!!! against almost everyone" + language = "en" + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ Type: provider.ResourceType_RESOURCE_TYPE_FILE, Size: 1, }) Expect(err).ToNot(HaveOccurred()) - - location := doc.Location - Expect(location).ToNot(BeNil()) - - // TODO: Altitude is not supported right now - Expect(location.Altitude).To(BeNil()) - Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328))) - Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285))) + Expect(doc.Content).To(Equal("body test stop words!!!")) }) - It("adds image content", func() { - fullResponse = `[ - { - "tiff:ImageWidth": "100", - "tiff:ImageLength": "100" - } - ]` + It("keeps the audio facet when an embedded resource follows", func() { + fullResponse = `[{"Content-Type": "audio/mpeg", "dc:title": "Sucker", "tk:content": "lyrics"}, {"Content-Type": "image/jpeg", "tiff:ImageWidth": "500"}]` + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ Type: provider.ResourceType_RESOURCE_TYPE_FILE, Size: 1, }) Expect(err).ToNot(HaveOccurred()) + Expect(doc.Audio).ToNot(BeNil()) + Expect(doc.Audio.Title).To(Equal(libregraph.PtrString("Sucker"))) + Expect(doc.Image).ToNot(BeNil()) + }) - image := doc.Image - Expect(image).ToNot(BeNil()) + It("adds no audio facet to non-audio documents", func() { + fullResponse = `[{"Content-Type": "application/pdf", "dc:title": "quarterly report"}]` - Expect(image.Width).To(Equal(libregraph.PtrInt32(100))) - Expect(image.Height).To(Equal(libregraph.PtrInt32(100))) + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Audio).To(BeNil()) + Expect(doc.Title).To(Equal("quarterly report")) }) - It("adds photo content", func() { - fullResponse = `[ - { - "tiff:Make": "Canon", - "tiff:Model": "Canon EOS 5D", - "exif:ExposureTime": "0.001", - "exif:FNumber": "1.8", - "exif:FocalLength": "50", - "Base ISO": "100", - "tiff:Orientation": "1", - "exif:DateTimeOriginal": "2018-01-01T12:34:56" - } - ]` + It("prefers the tika 4 content key over the legacy one", func() { + fullResponse = `[{"tk:content": "new", "X-TIKA:content": "old"}]` + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ Type: provider.ResourceType_RESOURCE_TYPE_FILE, Size: 1, }) Expect(err).ToNot(HaveOccurred()) - - photo := doc.Photo - Expect(photo).ToNot(BeNil()) - - Expect(photo.CameraMake).To(Equal(libregraph.PtrString("Canon"))) - Expect(photo.CameraModel).To(Equal(libregraph.PtrString("Canon EOS 5D"))) - Expect(photo.ExposureNumerator).To(Equal(libregraph.PtrFloat64(1))) - Expect(photo.ExposureDenominator).To(Equal(libregraph.PtrFloat64(1000))) - Expect(photo.FNumber).To(Equal(libregraph.PtrFloat64(1.8))) - Expect(photo.FocalLength).To(Equal(libregraph.PtrFloat64(50))) - Expect(photo.Iso).To(Equal(libregraph.PtrInt32(100))) - Expect(photo.Orientation).To(Equal(libregraph.PtrInt32(1))) - Expect(photo.TakenDateTime).To(Equal(libregraph.PtrTime(time.Date(2018, 1, 1, 12, 34, 56, 0, time.UTC)))) + Expect(doc.Content).To(Equal("new")) }) - It("removes stop words", func() { - body = "body to test stop words!!! against almost everyone" - language = "en" + It("joins the content of all meta entries", func() { + fullResponse = `[{"tk:content": "one"}, {"X-TIKA:content": "two"}]` doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ Type: provider.ResourceType_RESOURCE_TYPE_FILE, Size: 1, }) Expect(err).ToNot(HaveOccurred()) - Expect(doc.Content).To(Equal("body test stop words!!!")) + Expect(doc.Content).To(Equal("one two")) }) It("keeps stop words", func() { diff --git a/tests/acceptance/docker/src/tika.yml b/tests/acceptance/docker/src/tika.yml index 4c9768a090..fe8eff86e8 100644 --- a/tests/acceptance/docker/src/tika.yml +++ b/tests/acceptance/docker/src/tika.yml @@ -5,4 +5,4 @@ services: - tika command: tika:9998 tika: - image: apache/tika:3.2.3.0-full + image: apache/tika:4.0.0-full