Skip to content

Fixes #34954: keep the parent of every glossary search match in the hierarchy #35221

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yan-3005 merged 1 commit into from
Oct 12, 2026
Merged

Fixes #34954: keep the parent of every glossary search match in the hierarchy #35221

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 83 additions & 0 deletions ...ata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3398,6 +3398,89 @@ void test_glossaryTermSearchIndexUpdatedWhenGlossaryReviewerChanges(TestNamespac
});
}

@Test
void test_hierarchySearchKeepsParentOfEveryMatchOnThePage(TestNamespace ns) throws Exception {
OpenMetadataClient client = SdkClients.adminClient();
ObjectMapper mapper = new ObjectMapper();
String word = "pagecut" + UUID.randomUUID().toString().replace("-", "").substring(0, 10);
int matchCount = 30;
int pageSize = 25;

Glossary glossary =
client
.glossaries()
.create(
new CreateGlossary()
.withName(ns.prefix("hier_page_glossary"))
.withDescription("Glossary for hierarchy search paging"));
GlossaryTerm parent =
createEntity(
new CreateGlossaryTerm()
.withName(ns.prefix("hier_page_parent"))
.withGlossary(glossary.getFullyQualifiedName())
.withDescription("Parent that does not match the search word"));
for (int i = 0; i < matchCount; i++) {
createEntity(
new CreateGlossaryTerm()
.withName(ns.prefix("hier_page_child_" + i))
.withGlossary(glossary.getFullyQualifiedName())
.withParent(parent.getFullyQualifiedName())
.withDescription("Child matching " + word));
}

Awaitility.await("all matching terms should be indexed")
.atMost(java.time.Duration.ofSeconds(60))
.pollInterval(java.time.Duration.ofSeconds(1))
.ignoreExceptions()
.untilAsserted(
() -> {
String response =
client
.search()
.query(word)
.index("glossary_term_search_index")
.size(matchCount)
.execute();
assertEquals(
matchCount,
mapper.readTree(response).path("hits").path("total").path("value").asInt());
});

// More matches than the page holds push the non-matching parent out of the ranked page
RequestOptions options =
RequestOptions.builder()
.queryParam("q", word)
.queryParam("index", "glossary_term_search_index")
.queryParam("from", "0")
.queryParam("size", String.valueOf(pageSize))
.queryParam("deleted", "false")
.queryParam("getHierarchy", "true")
.build();
JsonNode glossaries =
mapper.readTree(
client
.getHttpClient()
.executeForString(HttpMethod.GET, "/v1/search/query", null, options));

JsonNode glossaryNode = findByFqn(glossaries, glossary.getFullyQualifiedName());
assertNotNull(glossaryNode, "Glossary of the matches should be returned");
JsonNode parentNode = findByFqn(glossaryNode.path("children"), parent.getFullyQualifiedName());
assertNotNull(parentNode, "Parent of the matches should be returned even though it ranks last");
assertEquals(
pageSize,
parentNode.path("children").size(),
"Every match on the page should sit under its parent");
}

private static JsonNode findByFqn(JsonNode nodes, String fqn) {
for (JsonNode node : nodes) {
if (fqn.equals(node.path("fullyQualifiedName").asText())) {
return node;
}
}
return null;
}

private String getAssetCounts(OpenMetadataClient client, String parent) {
RequestOptions.Builder optionsBuilder = RequestOptions.builder();
if (parent != null) {
Expand Down
75 changes: 72 additions & 3 deletions ...c/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSearchManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1792,7 +1792,12 @@ private ElasticSearchRequestBuilder buildHierarchyQuery(
// Add the original query as a should clause to preserve high scores
b.should(originalQuery);
// Add parent term queries as should clauses to fetch hierarchy
parentTermQueries.forEach(b::should);
if (indexName.equalsIgnoreCase(glossaryTermIndex)
|| indexName.equalsIgnoreCase(glossaryTermAlias)) {
b.should(withZeroScore(parentTermQueries));
} else {
parentTermQueries.forEach(b::should);
}
if (indexName.equalsIgnoreCase(glossaryTermIndex)) {
b.must(m -> m.match(ma -> ma.field("entityStatus").query("Approved")));
}
Expand Down Expand Up @@ -1829,15 +1834,15 @@ private List<?> buildSearchHierarchy(

if (indexName.equalsIgnoreCase(glossaryTermIndex)
|| indexName.equalsIgnoreCase(glossaryTermAlias)) {
response = buildGlossaryTermSearchHierarchy(searchResponse);
response = buildGlossaryTermSearchHierarchy(indexName, searchResponse);
} else if (indexName.equalsIgnoreCase(domainIndex) || indexName.equalsIgnoreCase(domainAlias)) {
response = buildDomainSearchHierarchy(searchResponse);
}
return response;
}

private List<EntityHierarchy> buildGlossaryTermSearchHierarchy(
SearchResponse<JsonData> searchResponse) {
String indexName, SearchResponse<JsonData> searchResponse) {
Map<String, EntityHierarchy> termMap = new LinkedHashMap<>();
Map<String, EntityHierarchy> rootTerms = new LinkedHashMap<>();

Expand All @@ -1861,6 +1866,7 @@ private List<EntityHierarchy> buildGlossaryTermSearchHierarchy(
termMap.putIfAbsent(term.getFullyQualifiedName(), term);
}

addMissingParents(indexName, termMap);
termMap.putAll(rootTerms);

termMap
Expand Down Expand Up @@ -1888,6 +1894,69 @@ private List<EntityHierarchy> buildGlossaryTermSearchHierarchy(
return new ArrayList<>(rootTerms.values());
}

// Parents only draw the tree, so they must not take page slots from real matches (the ones that
// miss the page are fetched by addMissingParents)
private static Query withZeroScore(List<Query> parentTermQueries) {
return Query.of(
q ->
q.constantScore(
c ->
c.filter(f -> f.bool(b -> b.should(parentTermQueries).minimumShouldMatch("1")))
.boost(0f)));
}

// A hit whose parent ranked below the page cut would otherwise be dropped from the tree
private void addMissingParents(String indexName, Map<String, EntityHierarchy> termMap) {
List<FieldValue> missing =
termMap.keySet().stream()
.flatMap(fqn -> parentTermFqns(fqn).stream())
.filter(fqn -> !termMap.containsKey(fqn))
.distinct()
.map(FieldValue::of)
.toList();
if (!missing.isEmpty()) {
for (EntityHierarchy parent : fetchApprovedTermsByFqn(indexName, missing)) {
parent.setChildren(new ArrayList<>());
termMap.putIfAbsent(parent.getFullyQualifiedName(), parent);
}
}
}

private List<EntityHierarchy> fetchApprovedTermsByFqn(String indexName, List<FieldValue> fqns) {
Query byFqn =
Query.of(q -> q.terms(t -> t.field("fullyQualifiedName").terms(tv -> tv.value(fqns))));
Query approved = Query.of(q -> q.match(m -> m.field("entityStatus").query("Approved")));
try {
return client
.search(
s ->
s.index(indexName)
.size(fqns.size())
.query(q -> q.bool(b -> b.filter(List.of(byFqn, approved)))),
JsonData.class)
.hits()
.hits()
.stream()
.filter(hit -> hit.source() != null)
.map(hit -> JsonUtils.readValue(hit.source().toJson().toString(), EntityHierarchy.class))
.toList();
} catch (IOException | ElasticsearchException e) {
LOG.warn("Could not fetch parent glossary terms for the search hierarchy", e);
return List.of();
}
}

// Term-level parents only: the glossary (single-part FQN) is the root and comes from each hit
private static List<String> parentTermFqns(String fqn) {
List<String> parents = new ArrayList<>();
for (String parent = getParentFQN(fqn);
parent != null && getParentFQN(parent) != null;
parent = getParentFQN(parent)) {
parents.add(parent);
}
return parents;
}

private List<EntityHierarchy> buildDomainSearchHierarchy(
SearchResponse<JsonData> searchResponse) {
Map<String, EntityHierarchy> entityHierarchyMap =
Expand Down
76 changes: 73 additions & 3 deletions ...ice/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSearchManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1871,7 +1871,12 @@ private OpenSearchRequestBuilder buildHierarchyQuery(
// Add the original query as a should clause to preserve high scores
b.should(originalQuery);
// Add parent term queries as should clauses to fetch hierarchy
parentTermQueries.forEach(b::should);
if (indexName.equalsIgnoreCase(glossaryTermIndex)
|| indexName.equalsIgnoreCase(glossaryTermAlias)) {
b.should(withZeroScore(parentTermQueries));
} else {
parentTermQueries.forEach(b::should);
}
if (indexName.equalsIgnoreCase(glossaryTermIndex)) {
b.must(
m ->
Expand Down Expand Up @@ -1924,15 +1929,15 @@ private List<?> buildSearchHierarchy(

if (indexName.equalsIgnoreCase(glossaryTermIndex)
|| indexName.equalsIgnoreCase(glossaryTermAlias)) {
response = buildGlossaryTermSearchHierarchy(searchResponse);
response = buildGlossaryTermSearchHierarchy(indexName, searchResponse);
} else if (indexName.equalsIgnoreCase(domainIndex) || indexName.equalsIgnoreCase(domainAlias)) {
response = buildDomainSearchHierarchy(searchResponse);
}
return response;
}

private List<EntityHierarchy> buildGlossaryTermSearchHierarchy(
SearchResponse<JsonData> searchResponse) {
String indexName, SearchResponse<JsonData> searchResponse) {
Map<String, EntityHierarchy> termMap = new LinkedHashMap<>();
Map<String, EntityHierarchy> rootTerms = new LinkedHashMap<>();

Expand All @@ -1956,6 +1961,7 @@ private List<EntityHierarchy> buildGlossaryTermSearchHierarchy(
termMap.putIfAbsent(term.getFullyQualifiedName(), term);
}

addMissingParents(indexName, termMap);
termMap.putAll(rootTerms);

termMap
Expand Down Expand Up @@ -1983,6 +1989,70 @@ private List<EntityHierarchy> buildGlossaryTermSearchHierarchy(
return new ArrayList<>(rootTerms.values());
}

// Parents only draw the tree, so they must not take page slots from real matches (the ones that
// miss the page are fetched by addMissingParents)
private static Query withZeroScore(List<Query> parentTermQueries) {
return Query.of(
q ->
q.constantScore(
c ->
c.filter(f -> f.bool(b -> b.should(parentTermQueries).minimumShouldMatch("1")))
.boost(0f)));
}

// A hit whose parent ranked below the page cut would otherwise be dropped from the tree
private void addMissingParents(String indexName, Map<String, EntityHierarchy> termMap) {
List<FieldValue> missing =
termMap.keySet().stream()
.flatMap(fqn -> parentTermFqns(fqn).stream())
.filter(fqn -> !termMap.containsKey(fqn))
.distinct()
.map(FieldValue::of)
.toList();
if (!missing.isEmpty()) {
for (EntityHierarchy parent : fetchApprovedTermsByFqn(indexName, missing)) {
parent.setChildren(new ArrayList<>());
termMap.putIfAbsent(parent.getFullyQualifiedName(), parent);
}
}
}

private List<EntityHierarchy> fetchApprovedTermsByFqn(String indexName, List<FieldValue> fqns) {
Query byFqn =
Query.of(q -> q.terms(t -> t.field("fullyQualifiedName").terms(tv -> tv.value(fqns))));
Query approved =
Query.of(q -> q.match(m -> m.field("entityStatus").query(FieldValue.of("Approved"))));
try {
return client
.search(
s ->
s.index(indexName)
.size(fqns.size())
.query(q -> q.bool(b -> b.filter(List.of(byFqn, approved)))),
JsonData.class)
.hits()
.hits()
.stream()
.filter(hit -> hit.source() != null)
.map(hit -> JsonUtils.readValue(hit.source().toJson().toString(), EntityHierarchy.class))
.toList();
} catch (IOException | OpenSearchException e) {
LOG.warn("Could not fetch parent glossary terms for the search hierarchy", e);
return List.of();
}
}

// Term-level parents only: the glossary (single-part FQN) is the root and comes from each hit
private static List<String> parentTermFqns(String fqn) {
List<String> parents = new ArrayList<>();
for (String parent = getParentFQN(fqn);
parent != null && getParentFQN(parent) != null;
parent = getParentFQN(parent)) {
parents.add(parent);
}
return parents;
}

private List<EntityHierarchy> buildDomainSearchHierarchy(
SearchResponse<JsonData> searchResponse) {
Map<String, EntityHierarchy> entityHierarchyMap =
Expand Down
Loading