feat(search): extract more data from tika 4 (if available)

.woodpecker.star

@@ -8,7 +8,7 @@ docker_repo_slug = "opencloudeu/opencloud"
 
 # images
 ALPINE_GIT = "alpine/git:latest"
-APACHE_TIKA = "apache/tika:3.2.3.0-full"
+APACHE_TIKA = "apache/tika:4.0.0-full"
 CHKO_DOCKER_PUSHRM = "chko/docker-pushrm:1"
 CODACY_COVERAGE_REPORTER = "codacy/codacy-coverage-reporter:14.1.3"
 COLLABORA_CODE = "collabora/code:24.04.5.1.1"
@@ -3442,6 +3442,9 @@ def tikaService():
     return [{
         "name": "tika",
         "image": APACHE_TIKA,
+        # tika 4 discovers its plugins relative to the image working directory,
+        # the workspace default would leave the pipes fetchers empty
+        "directory": "/opt/tika-server",
         "detach": True,
     }, {
         "name": "wait-for-tika-service",

services/search/README.md

@@ -99,7 +99,7 @@ It does not do any further content analysis.
 
 The main difference is that this extractor is able to analyze and extract data from more advanced file types like PDF, DOCX, PPTX, etc.
 However, [Apache Tika](https://tika.apache.org/) is required for this task.
-Read the [Getting Started with Apache Tika](https://tika.apache.org/2.6.0/gettingstarted.html) guide on how to install and run Tika or use a ready to run [Tika container](https://hub.docker.com/r/apache/tika).
+Read the [Getting Started with Apache Tika](https://tika.apache.org/) guide on how to install and run Tika or use a ready to run [Tika container](https://hub.docker.com/r/apache/tika).
 See the [Tika container usage document](https://github.com/apache/tika-docker#usage) for a quickstart.
 
 As soon as Tika is installed and configured, the search service needs to be told to use it.

services/search/pkg/content/extractor.go

@@ -23,7 +23,7 @@ func getFirstValue(m map[string][]string, key string) (string, error) {
 		return "", fmt.Errorf("unknown key: %v", key)
 	}
 
-	if len(m) == 0 {
+	if len(v) == 0 {
 		return "", fmt.Errorf("no values for: %v", key)
 	}
 

services/search/pkg/content/tika.go

@@ -3,15 +3,13 @@ package content
 import (
 	"context"
 	"fmt"
-	"math"
-	"strconv"
+	"io"
+	"net/http"
 	"strings"
-	"time"
 
 	gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
 	"github.com/google/go-tika/tika"
-	libregraph "github.com/opencloud-eu/libre-graph-api-go"
 	"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
 
 	"github.com/opencloud-eu/opencloud/pkg/log"
@@ -24,6 +22,7 @@ type Tika struct {
 	*Basic
 	Retriever
 	tika                       *tika.Client
+	tikaURL                    string
 	ContentExtractionSizeLimit uint64
 	CleanStopWords             bool
 }
@@ -46,6 +45,7 @@ func NewTikaExtractor(gatewaySelector pool.Selectable[gateway.GatewayAPIClient],
 		Basic:                      basic,
 		Retriever:                  newCS3Retriever(gatewaySelector, logger, cfg.Extractor.CS3AllowInsecure),
 		tika:                       tika.NewClient(nil, cfg.Extractor.Tika.TikaURL),
+		tikaURL:                    cfg.Extractor.Tika.TikaURL,
 		ContentExtractionSizeLimit: cfg.ContentExtractionSizeLimit,
 		CleanStopWords:             cfg.Extractor.Tika.CleanStopWords,
 	}, nil
@@ -91,218 +91,54 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document,
 			doc.Title = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Title, title))
 		}
 
-		if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil {
+		// tika 4 renamed the meta prefix from X-TIKA: to tk:
+		if content, err := getFirstValue(meta, "tk:content"); err == nil {
+			doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content))
+		} else if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil {
 			doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content))
 		}
 
-		doc.Location = t.getLocation(meta)
-		doc.Image = t.getImage(meta)
-		doc.Photo = t.getPhoto(meta)
-
-		if contentType, err := getFirstValue(meta, "Content-Type"); err == nil && strings.HasPrefix(contentType, "audio/") {
-			doc.Audio = t.getAudio(meta)
-		}
-	}
-
-	if langCode, _ := t.tika.LanguageString(ctx, doc.Content); langCode != "" && t.CleanStopWords {
-		doc.Content = CleanString(doc.Content, langCode)
-	}
-
-	return doc, nil
-}
-
-func (t Tika) getImage(meta map[string][]string) *libregraph.Image {
-	var image *libregraph.Image
-	initImage := func() {
-		if image == nil {
-			image = libregraph.NewImage()
-		}
-	}
-
-	if v, err := getFirstValue(meta, "tiff:ImageWidth"); err == nil {
-		if i, err := strconv.ParseInt(v, 0, 32); err == nil {
-			initImage()
-			image.SetWidth(int32(i))
-		}
-	}
-
-	if v, err := getFirstValue(meta, "tiff:ImageLength"); err == nil {
-		if i, err := strconv.ParseInt(v, 0, 32); err == nil {
-			initImage()
-			image.SetHeight(int32(i))
-		}
-	}
-
-	return image
-}
-
-func (t Tika) getLocation(meta map[string][]string) *libregraph.GeoCoordinates {
-	var location *libregraph.GeoCoordinates
-	initLocation := func() {
-		if location == nil {
-			location = libregraph.NewGeoCoordinates()
-		}
-	}
-
-	// TODO: location.Altitute: transform the following data to … feet above sea level.
-	// "GPS:GPS Altitude":                          []string{"227.4 metres"},
-	// "GPS:GPS Altitude Ref":                      []string{"Sea level"},
-
-	if v, err := getFirstValue(meta, "geo:lat"); err == nil {
-		if i, err := strconv.ParseFloat(v, 64); err == nil {
-			initLocation()
-			location.SetLatitude(i)
-		}
-	}
-
-	if v, err := getFirstValue(meta, "geo:long"); err == nil {
-		if i, err := strconv.ParseFloat(v, 64); err == nil {
-			initLocation()
-			location.SetLongitude(i)
-		}
-	}
-
-	return location
-}
-
-func (t Tika) getPhoto(meta map[string][]string) *libregraph.Photo {
-	var photo *libregraph.Photo
-	initPhoto := func() {
-		if photo == nil {
-			photo = libregraph.NewPhoto()
-		}
-	}
-
-	if v, err := getFirstValue(meta, "tiff:Make"); err == nil {
-		initPhoto()
-		photo.SetCameraMake(v)
-	}
-
-	if v, err := getFirstValue(meta, "tiff:Model"); err == nil {
-		initPhoto()
-		photo.SetCameraModel(v)
-	}
-
-	if v, err := getFirstValue(meta, "exif:FNumber"); err == nil {
-		if i, err := strconv.ParseFloat(v, 64); err == nil {
-			initPhoto()
-			photo.SetFNumber(i)
-		}
-	}
-
-	if v, err := getFirstValue(meta, "exif:FocalLength"); err == nil {
-		if i, err := strconv.ParseFloat(v, 64); err == nil {
-			initPhoto()
-			photo.SetFocalLength(i)
+		// keep facets from earlier entries, an embedded resource's meta
+		// (e.g. cover art) must not reset them
+		if v := t.getLocation(meta); v != nil {
+			doc.Location = v
 		}
-	}
-
-	if v, err := getFirstValue(meta, "Base ISO"); err == nil {
-		if i, err := strconv.ParseInt(v, 0, 32); err == nil {
-			initPhoto()
-			photo.SetIso(int32(i))
+		if v := t.getImage(meta); v != nil {
+			doc.Image = v
 		}
-	}
-
-	if v, err := getFirstValue(meta, "tiff:Orientation"); err == nil {
-		if i, err := strconv.ParseInt(v, 0, 32); err == nil {
-			initPhoto()
-			photo.SetOrientation(int32(i))
+		if v := t.getPhoto(meta); v != nil {
+			doc.Photo = v
 		}
-	}
-
-	if v, err := getFirstValue(meta, "exif:DateTimeOriginal"); err == nil {
-		layout := "2006-01-02T15:04:05"
-		if t, err := time.Parse(layout, v); err == nil {
-			initPhoto()
-			photo.SetTakenDateTime(t)
+		if v := t.getAudio(meta); v != nil {
+			doc.Audio = v
 		}
 	}
 
-	if v, err := getFirstValue(meta, "exif:ExposureTime"); err == nil {
-		if i, err := strconv.ParseFloat(v, 64); err == nil {
-			initPhoto()
-			photo.SetExposureNumerator(1)
-			photo.SetExposureDenominator(math.Round(1 / i))
-		}
+	if langCode := t.detectLanguage(ctx, doc.Content); langCode != "" && t.CleanStopWords {
+		doc.Content = CleanString(doc.Content, langCode)
 	}
 
-	return photo
+	return doc, nil
 }
 
-func (t Tika) getAudio(meta map[string][]string) *libregraph.Audio {
-	var audio *libregraph.Audio
-	initAudio := func() {
-		if audio == nil {
-			audio = libregraph.NewAudio()
-		}
-	}
-
-	if v, err := getFirstValue(meta, "xmpDM:album"); err == nil {
-		initAudio()
-		audio.SetAlbum(v)
-	}
-
-	if v, err := getFirstValue(meta, "xmpDM:albumArtist"); err == nil {
-		initAudio()
-		audio.SetAlbumArtist(v)
-	}
-
-	if v, err := getFirstValue(meta, "xmpDM:artist"); err == nil {
-		initAudio()
-		audio.SetArtist(v)
-	}
-
-	// TODO: audio.Bitrate: not provided by tika
-	// TODO: audio.Composers: not provided by tika
-	// TODO: audio.Copyright: not provided by tika for audio files?
-
-	if v, err := getFirstValue(meta, "xmpDM:discNumber"); err == nil {
-		if i, err := strconv.ParseInt(v, 10, 32); err == nil {
-			initAudio()
-			audio.SetDisc(int32(i))
-		}
-
-	}
-
-	//  TODO: audio.DiscCount: not provided by tika
-
-	if v, err := getFirstValue(meta, "xmpDM:duration"); err == nil {
-		// Tika emits fractional seconds.
-		if f, err := strconv.ParseFloat(v, 64); err == nil {
-			initAudio()
-			audio.SetDuration(int64(math.Round(f * 1000)))
+// detectLanguage asks tika for the language of content. Tika 4 moved the
+// endpoint from /language/string to /language, so try the new path first and
+// fall back for an older tika.
+func (t Tika) detectLanguage(ctx context.Context, content string) string {
+	for _, path := range []string{"/language", "/language/string"} {
+		req, err := http.NewRequestWithContext(ctx, http.MethodPut, t.tikaURL+path, strings.NewReader(content))
+		if err != nil {
+			return ""
 		}
-	}
-
-	if v, err := getFirstValue(meta, "xmpDM:genre"); err == nil {
-		initAudio()
-		audio.SetGenre(v)
-	}
-
-	// TODO: audio.HasDrm: not provided by tika
-	// TODO: audio.IsVariableBitrate: not provided by tika
-
-	if v, err := getFirstValue(meta, "dc:title"); err == nil {
-		initAudio()
-		audio.SetTitle(v)
-	}
-
-	if v, err := getFirstValue(meta, "xmpDM:trackNumber"); err == nil {
-		if i, err := strconv.ParseInt(v, 10, 32); err == nil {
-			initAudio()
-			audio.SetTrack(int32(i))
+		res, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return ""
 		}
-	}
-
-	// TODO: audio.TrackCount: not provided by tika
-
-	if v, err := getFirstValue(meta, "xmpDM:releaseDate"); err == nil {
-		if i, err := strconv.ParseInt(v, 10, 32); err == nil {
-			initAudio()
-			audio.SetYear(int32(i))
+		lang, err := io.ReadAll(res.Body)
+		_ = res.Body.Close()
+		if err == nil && res.StatusCode == http.StatusOK && len(lang) > 0 {
+			return string(lang)
 		}
 	}
-
-	return audio
+	return ""
 }