diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java index eda531307911..230a14fc0bbf 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java @@ -3398,6 +3398,89 @@ void test_glossaryTermSearchIndexUpdatedWhenGlossaryReviewerChanges(TestNamespac }); } + @Test + void test_hierarchySearchKeepsParentOfEveryMatchOnThePage(TestNamespace ns) throws Exception { + OpenMetadataClient client = SdkClients.adminClient(); + ObjectMapper mapper = new ObjectMapper(); + String word = "pagecut" + UUID.randomUUID().toString().replace("-", "").substring(0, 10); + int matchCount = 30; + int pageSize = 25; + + Glossary glossary = + client + .glossaries() + .create( + new CreateGlossary() + .withName(ns.prefix("hier_page_glossary")) + .withDescription("Glossary for hierarchy search paging")); + GlossaryTerm parent = + createEntity( + new CreateGlossaryTerm() + .withName(ns.prefix("hier_page_parent")) + .withGlossary(glossary.getFullyQualifiedName()) + .withDescription("Parent that does not match the search word")); + for (int i = 0; i < matchCount; i++) { + createEntity( + new CreateGlossaryTerm() + .withName(ns.prefix("hier_page_child_" + i)) + .withGlossary(glossary.getFullyQualifiedName()) + .withParent(parent.getFullyQualifiedName()) + .withDescription("Child matching " + word)); + } + + Awaitility.await("all matching terms should be indexed") + .atMost(java.time.Duration.ofSeconds(60)) + .pollInterval(java.time.Duration.ofSeconds(1)) + .ignoreExceptions() + .untilAsserted( + () -> { + String response = + client + .search() + .query(word) + .index("glossary_term_search_index") + .size(matchCount) + .execute(); + assertEquals( + matchCount, + mapper.readTree(response).path("hits").path("total").path("value").asInt()); + }); + + // More matches than the page holds push the non-matching parent out of the ranked page + RequestOptions options = + RequestOptions.builder() + .queryParam("q", word) + .queryParam("index", "glossary_term_search_index") + .queryParam("from", "0") + .queryParam("size", String.valueOf(pageSize)) + .queryParam("deleted", "false") + .queryParam("getHierarchy", "true") + .build(); + JsonNode glossaries = + mapper.readTree( + client + .getHttpClient() + .executeForString(HttpMethod.GET, "/v1/search/query", null, options)); + + JsonNode glossaryNode = findByFqn(glossaries, glossary.getFullyQualifiedName()); + assertNotNull(glossaryNode, "Glossary of the matches should be returned"); + JsonNode parentNode = findByFqn(glossaryNode.path("children"), parent.getFullyQualifiedName()); + assertNotNull(parentNode, "Parent of the matches should be returned even though it ranks last"); + assertEquals( + pageSize, + parentNode.path("children").size(), + "Every match on the page should sit under its parent"); + } + + private static JsonNode findByFqn(JsonNode nodes, String fqn) { + for (JsonNode node : nodes) { + if (fqn.equals(node.path("fullyQualifiedName").asText())) { + return node; + } + } + return null; + } + private String getAssetCounts(OpenMetadataClient client, String parent) { RequestOptions.Builder optionsBuilder = RequestOptions.builder(); if (parent != null) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSearchManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSearchManager.java index 9365d0a8b8e8..ed5eadcd1032 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSearchManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSearchManager.java @@ -1792,7 +1792,12 @@ private ElasticSearchRequestBuilder buildHierarchyQuery( // Add the original query as a should clause to preserve high scores b.should(originalQuery); // Add parent term queries as should clauses to fetch hierarchy - parentTermQueries.forEach(b::should); + if (indexName.equalsIgnoreCase(glossaryTermIndex) + || indexName.equalsIgnoreCase(glossaryTermAlias)) { + b.should(withZeroScore(parentTermQueries)); + } else { + parentTermQueries.forEach(b::should); + } if (indexName.equalsIgnoreCase(glossaryTermIndex)) { b.must(m -> m.match(ma -> ma.field("entityStatus").query("Approved"))); } @@ -1829,7 +1834,7 @@ private List buildSearchHierarchy( if (indexName.equalsIgnoreCase(glossaryTermIndex) || indexName.equalsIgnoreCase(glossaryTermAlias)) { - response = buildGlossaryTermSearchHierarchy(searchResponse); + response = buildGlossaryTermSearchHierarchy(indexName, searchResponse); } else if (indexName.equalsIgnoreCase(domainIndex) || indexName.equalsIgnoreCase(domainAlias)) { response = buildDomainSearchHierarchy(searchResponse); } @@ -1837,7 +1842,7 @@ private List buildSearchHierarchy( } private List buildGlossaryTermSearchHierarchy( - SearchResponse searchResponse) { + String indexName, SearchResponse searchResponse) { Map termMap = new LinkedHashMap<>(); Map rootTerms = new LinkedHashMap<>(); @@ -1861,6 +1866,7 @@ private List buildGlossaryTermSearchHierarchy( termMap.putIfAbsent(term.getFullyQualifiedName(), term); } + addMissingParents(indexName, termMap); termMap.putAll(rootTerms); termMap @@ -1888,6 +1894,69 @@ private List buildGlossaryTermSearchHierarchy( return new ArrayList<>(rootTerms.values()); } + // Parents only draw the tree, so they must not take page slots from real matches (the ones that + // miss the page are fetched by addMissingParents) + private static Query withZeroScore(List parentTermQueries) { + return Query.of( + q -> + q.constantScore( + c -> + c.filter(f -> f.bool(b -> b.should(parentTermQueries).minimumShouldMatch("1"))) + .boost(0f))); + } + + // A hit whose parent ranked below the page cut would otherwise be dropped from the tree + private void addMissingParents(String indexName, Map termMap) { + List missing = + termMap.keySet().stream() + .flatMap(fqn -> parentTermFqns(fqn).stream()) + .filter(fqn -> !termMap.containsKey(fqn)) + .distinct() + .map(FieldValue::of) + .toList(); + if (!missing.isEmpty()) { + for (EntityHierarchy parent : fetchApprovedTermsByFqn(indexName, missing)) { + parent.setChildren(new ArrayList<>()); + termMap.putIfAbsent(parent.getFullyQualifiedName(), parent); + } + } + } + + private List fetchApprovedTermsByFqn(String indexName, List fqns) { + Query byFqn = + Query.of(q -> q.terms(t -> t.field("fullyQualifiedName").terms(tv -> tv.value(fqns)))); + Query approved = Query.of(q -> q.match(m -> m.field("entityStatus").query("Approved"))); + try { + return client + .search( + s -> + s.index(indexName) + .size(fqns.size()) + .query(q -> q.bool(b -> b.filter(List.of(byFqn, approved)))), + JsonData.class) + .hits() + .hits() + .stream() + .filter(hit -> hit.source() != null) + .map(hit -> JsonUtils.readValue(hit.source().toJson().toString(), EntityHierarchy.class)) + .toList(); + } catch (IOException | ElasticsearchException e) { + LOG.warn("Could not fetch parent glossary terms for the search hierarchy", e); + return List.of(); + } + } + + // Term-level parents only: the glossary (single-part FQN) is the root and comes from each hit + private static List parentTermFqns(String fqn) { + List parents = new ArrayList<>(); + for (String parent = getParentFQN(fqn); + parent != null && getParentFQN(parent) != null; + parent = getParentFQN(parent)) { + parents.add(parent); + } + return parents; + } + private List buildDomainSearchHierarchy( SearchResponse searchResponse) { Map entityHierarchyMap = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSearchManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSearchManager.java index c93b3f580b8d..f799e1505553 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSearchManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSearchManager.java @@ -1871,7 +1871,12 @@ private OpenSearchRequestBuilder buildHierarchyQuery( // Add the original query as a should clause to preserve high scores b.should(originalQuery); // Add parent term queries as should clauses to fetch hierarchy - parentTermQueries.forEach(b::should); + if (indexName.equalsIgnoreCase(glossaryTermIndex) + || indexName.equalsIgnoreCase(glossaryTermAlias)) { + b.should(withZeroScore(parentTermQueries)); + } else { + parentTermQueries.forEach(b::should); + } if (indexName.equalsIgnoreCase(glossaryTermIndex)) { b.must( m -> @@ -1924,7 +1929,7 @@ private List buildSearchHierarchy( if (indexName.equalsIgnoreCase(glossaryTermIndex) || indexName.equalsIgnoreCase(glossaryTermAlias)) { - response = buildGlossaryTermSearchHierarchy(searchResponse); + response = buildGlossaryTermSearchHierarchy(indexName, searchResponse); } else if (indexName.equalsIgnoreCase(domainIndex) || indexName.equalsIgnoreCase(domainAlias)) { response = buildDomainSearchHierarchy(searchResponse); } @@ -1932,7 +1937,7 @@ private List buildSearchHierarchy( } private List buildGlossaryTermSearchHierarchy( - SearchResponse searchResponse) { + String indexName, SearchResponse searchResponse) { Map termMap = new LinkedHashMap<>(); Map rootTerms = new LinkedHashMap<>(); @@ -1956,6 +1961,7 @@ private List buildGlossaryTermSearchHierarchy( termMap.putIfAbsent(term.getFullyQualifiedName(), term); } + addMissingParents(indexName, termMap); termMap.putAll(rootTerms); termMap @@ -1983,6 +1989,70 @@ private List buildGlossaryTermSearchHierarchy( return new ArrayList<>(rootTerms.values()); } + // Parents only draw the tree, so they must not take page slots from real matches (the ones that + // miss the page are fetched by addMissingParents) + private static Query withZeroScore(List parentTermQueries) { + return Query.of( + q -> + q.constantScore( + c -> + c.filter(f -> f.bool(b -> b.should(parentTermQueries).minimumShouldMatch("1"))) + .boost(0f))); + } + + // A hit whose parent ranked below the page cut would otherwise be dropped from the tree + private void addMissingParents(String indexName, Map termMap) { + List missing = + termMap.keySet().stream() + .flatMap(fqn -> parentTermFqns(fqn).stream()) + .filter(fqn -> !termMap.containsKey(fqn)) + .distinct() + .map(FieldValue::of) + .toList(); + if (!missing.isEmpty()) { + for (EntityHierarchy parent : fetchApprovedTermsByFqn(indexName, missing)) { + parent.setChildren(new ArrayList<>()); + termMap.putIfAbsent(parent.getFullyQualifiedName(), parent); + } + } + } + + private List fetchApprovedTermsByFqn(String indexName, List fqns) { + Query byFqn = + Query.of(q -> q.terms(t -> t.field("fullyQualifiedName").terms(tv -> tv.value(fqns)))); + Query approved = + Query.of(q -> q.match(m -> m.field("entityStatus").query(FieldValue.of("Approved")))); + try { + return client + .search( + s -> + s.index(indexName) + .size(fqns.size()) + .query(q -> q.bool(b -> b.filter(List.of(byFqn, approved)))), + JsonData.class) + .hits() + .hits() + .stream() + .filter(hit -> hit.source() != null) + .map(hit -> JsonUtils.readValue(hit.source().toJson().toString(), EntityHierarchy.class)) + .toList(); + } catch (IOException | OpenSearchException e) { + LOG.warn("Could not fetch parent glossary terms for the search hierarchy", e); + return List.of(); + } + } + + // Term-level parents only: the glossary (single-part FQN) is the root and comes from each hit + private static List parentTermFqns(String fqn) { + List parents = new ArrayList<>(); + for (String parent = getParentFQN(fqn); + parent != null && getParentFQN(parent) != null; + parent = getParentFQN(parent)) { + parents.add(parent); + } + return parents; + } + private List buildDomainSearchHierarchy( SearchResponse searchResponse) { Map entityHierarchyMap =