From bd685a8962d3be496f7ffbc6e741a0617f9df0d3 Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Wed, 8 Jul 2026 20:43:49 +0200 Subject: [PATCH 01/23] Add Semantic-Swingers taxonomy-discovery learner [Task C] --- CHANGELOG.md | 4 + ...ner_semanticswingers_taxonomy_discovery.py | 37 ++++ .../learner/taxonomy_discovery/__init__.py | 1 + .../taxonomy_discovery/semanticswingers.py | 165 ++++++++++++++++++ 4 files changed, 207 insertions(+) create mode 100644 examples/llm_learner_semanticswingers_taxonomy_discovery.py create mode 100644 ontolearner/learner/taxonomy_discovery/semanticswingers.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e1125451..5812958f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## Changelog +### Unreleased +- Add Semantic-Swingers taxonomy-discovery learner (LLMs4OL 2026, Task C): retrieval-first + taxonomy induction with a swappable parent selector (offline embedding baseline / OpenAI champion). + ### v1.5.1 (March 30, 2026) - Fix challenge learner - Update requirements. diff --git a/examples/llm_learner_semanticswingers_taxonomy_discovery.py b/examples/llm_learner_semanticswingers_taxonomy_discovery.py new file mode 100644 index 00000000..36c683f0 --- /dev/null +++ b/examples/llm_learner_semanticswingers_taxonomy_discovery.py @@ -0,0 +1,37 @@ +from ontolearner import Wine, train_test_split, LearnerPipeline +from ontolearner.learner.taxonomy_discovery import SemanticSwingersTaxonomyLearner + +# 1) Load & split +ontology = Wine() +ontology.load() +data = ontology.extract() +train_data, test_data = train_test_split(data, test_size=0.2, random_state=42) + +# 2) Configure the semantic-swingers taxonomy learner. +# Offline default (no API key): the "embedding" selector. +# Champion: selector="openai" + api_key=... for gpt-4.1-mini parent selection. +learner = SemanticSwingersTaxonomyLearner( + embedding_model="sentence-transformers/all-MiniLM-L6-v2", # champion: mxbai-embed-large-v1 + top_k=10, + selector="embedding", + device="cpu", +) + +# 3) Build pipeline (pass our learner as `llm`, raw ontology objects) +pipeline = LearnerPipeline( + llm=learner, + llm_id="semanticswingers-taxonomy", + ontologizer_data=False, +) + +# 4) Train (no-op) + predict + evaluate +outputs = pipeline( + train_data=train_data, + test_data=test_data, + task="taxonomy-discovery", + evaluate=True, + ontologizer_data=False, +) + +print("Metrics:", outputs.get("metrics")) +print("Elapsed time:", outputs["elapsed_time"]) diff --git a/ontolearner/learner/taxonomy_discovery/__init__.py b/ontolearner/learner/taxonomy_discovery/__init__.py index ec6f2f4f..22d173e5 100644 --- a/ontolearner/learner/taxonomy_discovery/__init__.py +++ b/ontolearner/learner/taxonomy_discovery/__init__.py @@ -15,4 +15,5 @@ from .alexbek import AlexbekCrossAttnLearner from .rwthdbis import RWTHDBISSFTLearner from .sbunlp import SBUNLPFewShotLearner +from .semanticswingers import SemanticSwingersTaxonomyLearner from .skhnlp import SKHNLPSequentialFTLearner, SKHNLPZSLearner diff --git a/ontolearner/learner/taxonomy_discovery/semanticswingers.py b/ontolearner/learner/taxonomy_discovery/semanticswingers.py new file mode 100644 index 00000000..f46e570f --- /dev/null +++ b/ontolearner/learner/taxonomy_discovery/semanticswingers.py @@ -0,0 +1,165 @@ +# Copyright (c) 2025 SciKnowOrg +# +# Licensed under the MIT License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/MIT +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Semantic-Swingers taxonomy-discovery learner (LLMs4OL 2026, Task C). + +A retrieval-first taxonomy inducer: a sentence-embedding encoder embeds the type +vocabulary, nearest neighbours become candidate parents, and a selection step picks +the parent for each child. Two selectors are provided: + +* ``"embedding"`` (default) — fully offline/deterministic; the most *general* candidate + (highest mean similarity to the vocabulary) is chosen as parent. No API key needed. +* ``"openai"`` — the competition champion; an OpenAI chat model picks the parent from + the retrieved candidates. Enabled only when an ``api_key`` is supplied (never hard-coded). + +The team's finding is that the *encoder* is the bottleneck for this task, so the default +encoder is ``mixedbread-ai/mxbai-embed-large-v1`` (a clean, no-fine-tuning +0.03 over MiniLM). +""" +from __future__ import annotations + +import os +from typing import Any, List, Optional + +import numpy as np +from sentence_transformers import SentenceTransformer + +from ...base import AutoLearner + + +class SemanticSwingersTaxonomyLearner(AutoLearner): + """Embedding-retrieval taxonomy induction with a swappable parent selector. + + Args: + embedding_model: SentenceTransformer id used to embed type labels. Defaults to + ``mixedbread-ai/mxbai-embed-large-v1`` (the team's champion encoder). + top_k: Number of candidate parents retrieved per child before selection. + selector: ``"embedding"`` (offline heuristic) or ``"openai"`` (LLM selection). + openai_model: Chat model id used when ``selector="openai"``. + api_key: OpenAI API key. If ``None``, falls back to the ``OPENAI_API_KEY`` env + var; if still unset, the learner silently degrades to the embedding selector. + device: Torch device for the encoder. + """ + + def __init__( + self, + embedding_model: str = "mixedbread-ai/mxbai-embed-large-v1", + top_k: int = 30, + selector: str = "embedding", + openai_model: str = "gpt-4.1-mini", + api_key: Optional[str] = None, + device: str = "cpu", + ) -> None: + """Initialise the learner and record configuration (no I/O yet).""" + super().__init__() + self.embedding_model = embedding_model + self.top_k = top_k + self.selector = selector + self.openai_model = openai_model + self.api_key = api_key or os.environ.get("OPENAI_API_KEY") + self.device = device + self._encoder: Optional[SentenceTransformer] = None + + def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: + """Load the sentence-embedding encoder. + + Called by ``LearnerPipeline`` as ``load(model_id=llm_id)``. A model id that looks + like a SentenceTransformer path (contains ``/``) overrides the default; a plain + bookkeeping label is ignored. + """ + if model_id and "/" in model_id: + self.embedding_model = model_id + self._encoder = SentenceTransformer(self.embedding_model, device=self.device) + + def tasks_data_former(self, data: Any, task: str, test: bool = False) -> Any: + """Pass the raw ``OntologyData`` through (run with ``ontologizer_data=False``).""" + return data + + @staticmethod + def _nodes(data: Any) -> List[str]: + """Return the deduped type vocabulary (declared types + edge endpoints).""" + tt = data.type_taxonomies + nodes: List[str] = list(tt.types) + for rel in tt.taxonomies: + nodes.extend([rel.parent, rel.child]) + seen: set = set() + out: List[str] = [] + for node in nodes: + if node not in seen: + seen.add(node) + out.append(node) + return out + + def _select_embedding(self, nodes: List[str], sim: np.ndarray) -> List[dict]: + """Offline selector: the most general retrieved candidate is the parent. + + Generality is proxied by mean similarity to the whole vocabulary (a central, + broadly-similar type tends to be a superclass). For each child we keep the top + candidate that is more general than the child itself. + """ + generality = sim.mean(axis=1) + preds: List[dict] = [] + for child_idx, child in enumerate(nodes): + candidates = np.argsort(-sim[child_idx])[: self.top_k] + for parent_idx in candidates: + if parent_idx == child_idx: + continue + if generality[parent_idx] >= generality[child_idx]: + preds.append({"parent": nodes[int(parent_idx)], "child": child}) + break + return preds + + def _select_openai(self, nodes: List[str], sim: np.ndarray) -> List[dict]: + """Champion selector: an OpenAI chat model picks the parent from candidates.""" + from openai import OpenAI + + client = OpenAI(api_key=self.api_key) + preds: List[dict] = [] + for child_idx, child in enumerate(nodes): + candidates = [ + nodes[int(i)] + for i in np.argsort(-sim[child_idx])[: self.top_k] + if int(i) != child_idx + ] + if not candidates: + continue + prompt = ( + f"Which of the following is the most likely direct parent (superclass) " + f"of '{child}'? Answer with exactly one option or 'NONE'.\n" + + "\n".join(f"- {c}" for c in candidates) + ) + resp = client.chat.completions.create( + model=self.openai_model, + messages=[{"role": "user", "content": prompt}], + temperature=0, + max_tokens=32, + ) + answer = (resp.choices[0].message.content or "").strip() + for cand in candidates: + if cand.lower() in answer.lower(): + preds.append({"parent": cand, "child": child}) + break + return preds + + def _taxonomy_discovery(self, data: Any, test: bool = False) -> Optional[Any]: + """Retrieval-only method: no training; induce edges at inference.""" + if not test: + return None + if self._encoder is None: + self.load() + nodes = self._nodes(data) + emb = np.asarray( + self._encoder.encode(nodes, normalize_embeddings=True, show_progress_bar=False) + ) + sim = emb @ emb.T + use_openai = self.selector == "openai" and self.api_key + return self._select_openai(nodes, sim) if use_openai else self._select_embedding(nodes, sim) From 4b0fcac5f2da858fc0684dd954f2357aade18027 Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Wed, 8 Jul 2026 21:54:59 +0200 Subject: [PATCH 02/23] Add local Ollama selector, configurable max_tokens, and docs for Semantic-Swingers taxonomy learner [Task C] - selector="ollama": free local reproduction of the champion selection pipeline via Ollama's OpenAI-compatible endpoint (no API key) - max_tokens constructor param (was hardcoded 32, which silently truncates thinking models into empty answers) - docs page (llms4ol_challenge/semanticswingers_learner.rst) + toctree entry - validated on Wine through LearnerPipeline: gpt-4.1-mini f1=0.21, llama3.1:8b f1=0.07, embedding baseline f1=0.07 --- CHANGELOG.md | 3 +- docs/source/learners/llms4ol.rst | 1 + .../semanticswingers_learner.rst | 106 ++++++++++++++++++ ...ner_semanticswingers_taxonomy_discovery.py | 1 + .../taxonomy_discovery/semanticswingers.py | 53 ++++++--- 5 files changed, 149 insertions(+), 15 deletions(-) create mode 100644 docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst diff --git a/CHANGELOG.md b/CHANGELOG.md index 5812958f..9e3eb17a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ### Unreleased - Add Semantic-Swingers taxonomy-discovery learner (LLMs4OL 2026, Task C): retrieval-first - taxonomy induction with a swappable parent selector (offline embedding baseline / OpenAI champion). + taxonomy induction with a swappable parent selector (offline embedding baseline / OpenAI + champion / local Ollama reproduction), plus documentation page and example. ### v1.5.1 (March 30, 2026) - Fix challenge learner diff --git a/docs/source/learners/llms4ol.rst b/docs/source/learners/llms4ol.rst index 820171a8..3e1d0363 100644 --- a/docs/source/learners/llms4ol.rst +++ b/docs/source/learners/llms4ol.rst @@ -79,3 +79,4 @@ LLMs4OL is a community development initiative collocated with the International llms4ol_challenge/skhnlp_learner llms4ol_challenge/alexbek_learner llms4ol_challenge/sbunlp_learner + llms4ol_challenge/semanticswingers_learner diff --git a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst new file mode 100644 index 00000000..6581e132 --- /dev/null +++ b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst @@ -0,0 +1,106 @@ +Semantic-Swingers Learner +========================== + + +.. sidebar:: Semantic-Swingers Learner Examples + + * Taxonomy Discovery: `llm_learner_semanticswingers_taxonomy_discovery.py `_ + +The Semantic-Swingers team participated in the LLMs4OL 2026 Shared Task. This page documents the +taxonomy-discovery learner (Task C): a retrieval-first taxonomy inducer with a swappable +parent-selection step. + + +Overview +--------------------------------- + +The learner treats taxonomy discovery as *retrieve-then-select*: + +1. **Retrieve** — a sentence-embedding encoder embeds the type vocabulary; for every child + type, the ``top_k`` nearest neighbours become candidate parents. The team's finding is + that the *encoder* is the main lever for this stage, so the default encoder is + ``mixedbread-ai/mxbai-embed-large-v1``. +2. **Select** — a selection step picks the parent for each child from its candidates. + Three selectors are provided: + + - ``"embedding"`` (default) — fully offline and deterministic. The most *general* + candidate (highest mean similarity to the whole vocabulary) is chosen as parent. + No API key or LLM required; intended as a fast, reproducible baseline. + - ``"openai"`` — the competition champion. An OpenAI chat model (default + ``gpt-4.1-mini``) picks the parent from the retrieved candidates. Requires an + API key (via the ``api_key`` argument or the ``OPENAI_API_KEY`` environment + variable — never hard-coded); without a key the learner silently degrades to + the embedding selector. + - ``"ollama"`` — a free, local reproduction of the champion *pipeline*. The same + selection prompt is served by a local `Ollama `_ model + (default ``llama3.1:8b``) through its OpenAI-compatible endpoint. No API key + required. Prefer direct-answering models here: thinking models (e.g. Qwen3.5) + spend their completion budget on reasoning tokens and need ``max_tokens=1024`` + or more to produce an answer at all. + +The learner requires no training: ``fit`` is a no-op and all edges are induced at +inference time, so it works on unseen ontologies without any target-vocabulary +assumptions. + +Loading Ontological Data +---------------------------- + +.. code-block:: python + + from ontolearner import Wine, train_test_split + + ontology = Wine() + ontology.load() + data = ontology.extract() + + train_data, test_data = train_test_split(data, test_size=0.2, random_state=42) + +Initialize Learner +----------------------------- + +.. code-block:: python + + from ontolearner.learner.taxonomy_discovery import SemanticSwingersTaxonomyLearner + + # Offline baseline (no API key, deterministic) + learner = SemanticSwingersTaxonomyLearner( + embedding_model="mixedbread-ai/mxbai-embed-large-v1", + top_k=30, + selector="embedding", + device="cpu", + ) + + # Champion configuration (OpenAI LLM selection) + # learner = SemanticSwingersTaxonomyLearner( + # top_k=30, selector="openai", api_key="", + # ) + + # Local champion-reproduction (no API key; requires a running Ollama server) + # learner = SemanticSwingersTaxonomyLearner( + # top_k=30, selector="ollama", + # ) + +Run the Pipeline +----------------------------- + +The learner runs on raw ontology objects, so pass ``ontologizer_data=False``. + +.. code-block:: python + + from ontolearner import LearnerPipeline + + pipeline = LearnerPipeline( + llm=learner, + llm_id="semanticswingers-taxonomy", + ontologizer_data=False, + ) + + outputs = pipeline( + train_data=train_data, + test_data=test_data, + task="taxonomy-discovery", + evaluate=True, + ontologizer_data=False, + ) + + print(outputs["metrics"]) diff --git a/examples/llm_learner_semanticswingers_taxonomy_discovery.py b/examples/llm_learner_semanticswingers_taxonomy_discovery.py index 36c683f0..2ea7bb92 100644 --- a/examples/llm_learner_semanticswingers_taxonomy_discovery.py +++ b/examples/llm_learner_semanticswingers_taxonomy_discovery.py @@ -10,6 +10,7 @@ # 2) Configure the semantic-swingers taxonomy learner. # Offline default (no API key): the "embedding" selector. # Champion: selector="openai" + api_key=... for gpt-4.1-mini parent selection. +# Local champion-reproduction (no API key): selector="ollama" (local Ollama server). learner = SemanticSwingersTaxonomyLearner( embedding_model="sentence-transformers/all-MiniLM-L6-v2", # champion: mxbai-embed-large-v1 top_k=10, diff --git a/ontolearner/learner/taxonomy_discovery/semanticswingers.py b/ontolearner/learner/taxonomy_discovery/semanticswingers.py index f46e570f..d0517bb6 100644 --- a/ontolearner/learner/taxonomy_discovery/semanticswingers.py +++ b/ontolearner/learner/taxonomy_discovery/semanticswingers.py @@ -15,12 +15,14 @@ A retrieval-first taxonomy inducer: a sentence-embedding encoder embeds the type vocabulary, nearest neighbours become candidate parents, and a selection step picks -the parent for each child. Two selectors are provided: +the parent for each child. Three selectors are provided: * ``"embedding"`` (default) — fully offline/deterministic; the most *general* candidate (highest mean similarity to the vocabulary) is chosen as parent. No API key needed. * ``"openai"`` — the competition champion; an OpenAI chat model picks the parent from the retrieved candidates. Enabled only when an ``api_key`` is supplied (never hard-coded). +* ``"ollama"`` — free, local reproduction of the champion: the same selection prompt is + served by a local Ollama model through its OpenAI-compatible endpoint. No API key needed. The team's finding is that the *encoder* is the bottleneck for this task, so the default encoder is ``mixedbread-ai/mxbai-embed-large-v1`` (a clean, no-fine-tuning +0.03 over MiniLM). @@ -43,20 +45,34 @@ class SemanticSwingersTaxonomyLearner(AutoLearner): embedding_model: SentenceTransformer id used to embed type labels. Defaults to ``mixedbread-ai/mxbai-embed-large-v1`` (the team's champion encoder). top_k: Number of candidate parents retrieved per child before selection. - selector: ``"embedding"`` (offline heuristic) or ``"openai"`` (LLM selection). - openai_model: Chat model id used when ``selector="openai"``. - api_key: OpenAI API key. If ``None``, falls back to the ``OPENAI_API_KEY`` env - var; if still unset, the learner silently degrades to the embedding selector. + selector: ``"embedding"`` (offline heuristic), ``"openai"`` (champion LLM + selection), or ``"ollama"`` (local LLM selection, no API key). + llm_model: Chat model id used by the LLM selectors. Defaults to + ``gpt-4.1-mini`` for ``"openai"`` and ``llama3.1:8b`` for ``"ollama"``. + api_key: OpenAI API key for ``selector="openai"``. If ``None``, falls back to + the ``OPENAI_API_KEY`` env var; if still unset, the learner silently + degrades to the embedding selector. Ignored by ``"ollama"``. + base_url: OpenAI-compatible endpoint for the LLM selector. Defaults to the + local Ollama server (``http://localhost:11434/v1``) when + ``selector="ollama"``, and to the OpenAI API otherwise. + max_tokens: Completion budget for the LLM selector. Direct-answering models + need very little; thinking models (e.g. Qwen3.5) spend reasoning tokens + before answering and need a much larger budget (512+). device: Torch device for the encoder. """ + _OLLAMA_BASE_URL = "http://localhost:11434/v1" + _DEFAULT_LLM = {"openai": "gpt-4.1-mini", "ollama": "llama3.1:8b"} + def __init__( self, embedding_model: str = "mixedbread-ai/mxbai-embed-large-v1", top_k: int = 30, selector: str = "embedding", - openai_model: str = "gpt-4.1-mini", + llm_model: Optional[str] = None, api_key: Optional[str] = None, + base_url: Optional[str] = None, + max_tokens: int = 64, device: str = "cpu", ) -> None: """Initialise the learner and record configuration (no I/O yet).""" @@ -64,8 +80,12 @@ def __init__( self.embedding_model = embedding_model self.top_k = top_k self.selector = selector - self.openai_model = openai_model + self.llm_model = llm_model or self._DEFAULT_LLM.get(selector) self.api_key = api_key or os.environ.get("OPENAI_API_KEY") + if base_url is None and selector == "ollama": + base_url = self._OLLAMA_BASE_URL + self.base_url = base_url + self.max_tokens = max_tokens self.device = device self._encoder: Optional[SentenceTransformer] = None @@ -118,11 +138,16 @@ def _select_embedding(self, nodes: List[str], sim: np.ndarray) -> List[dict]: break return preds - def _select_openai(self, nodes: List[str], sim: np.ndarray) -> List[dict]: - """Champion selector: an OpenAI chat model picks the parent from candidates.""" + def _select_llm(self, nodes: List[str], sim: np.ndarray) -> List[dict]: + """LLM selector: a chat model picks the parent from the retrieved candidates. + + Serves both the ``"openai"`` champion and the local ``"ollama"`` fallback — + the latter is just an OpenAI-compatible endpoint with a placeholder key. + """ from openai import OpenAI - client = OpenAI(api_key=self.api_key) + api_key = self.api_key if self.selector == "openai" else (self.api_key or "ollama") + client = OpenAI(api_key=api_key, base_url=self.base_url) preds: List[dict] = [] for child_idx, child in enumerate(nodes): candidates = [ @@ -138,10 +163,10 @@ def _select_openai(self, nodes: List[str], sim: np.ndarray) -> List[dict]: + "\n".join(f"- {c}" for c in candidates) ) resp = client.chat.completions.create( - model=self.openai_model, + model=self.llm_model, messages=[{"role": "user", "content": prompt}], temperature=0, - max_tokens=32, + max_tokens=self.max_tokens, ) answer = (resp.choices[0].message.content or "").strip() for cand in candidates: @@ -161,5 +186,5 @@ def _taxonomy_discovery(self, data: Any, test: bool = False) -> Optional[Any]: self._encoder.encode(nodes, normalize_embeddings=True, show_progress_bar=False) ) sim = emb @ emb.T - use_openai = self.selector == "openai" and self.api_key - return self._select_openai(nodes, sim) if use_openai else self._select_embedding(nodes, sim) + use_llm = self.selector == "ollama" or (self.selector == "openai" and self.api_key) + return self._select_llm(nodes, sim) if use_llm else self._select_embedding(nodes, sim) From ef598cc8c3cd6695a98a08a129daf96d8347b991 Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Wed, 8 Jul 2026 22:05:03 +0200 Subject: [PATCH 03/23] Add Semantic-Swingers term-typing learner [Task B] - closed-vocabulary term typing: fit learns the type inventory from the train split; inference assigns types from that inventory only - three selectors mirroring the Task C learner: embedding (offline nearest-type), openai champion (gpt-4.1-mini), ollama local - registered in learner/term_typing; validated offline on Wine via LearnerPipeline (embedding selector f1=0.39) --- CHANGELOG.md | 3 + ontolearner/learner/term_typing/__init__.py | 1 + .../learner/term_typing/semanticswingers.py | 209 ++++++++++++++++++ 3 files changed, 213 insertions(+) create mode 100644 ontolearner/learner/term_typing/semanticswingers.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e3eb17a..8879a20f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ ## Changelog ### Unreleased +- Add Semantic-Swingers term-typing learner (LLMs4OL 2026, Task B): closed-vocabulary + term typing with a swappable type selector (offline embedding baseline / OpenAI + champion / local Ollama reproduction). - Add Semantic-Swingers taxonomy-discovery learner (LLMs4OL 2026, Task C): retrieval-first taxonomy induction with a swappable parent selector (offline embedding baseline / OpenAI champion / local Ollama reproduction), plus documentation page and example. diff --git a/ontolearner/learner/term_typing/__init__.py b/ontolearner/learner/term_typing/__init__.py index dec8b9f9..f70f3f1c 100644 --- a/ontolearner/learner/term_typing/__init__.py +++ b/ontolearner/learner/term_typing/__init__.py @@ -15,3 +15,4 @@ from .alexbek import AlexbekRAGLearner, AlexbekRFLearner from .rwthdbis import RWTHDBISSFTLearner from .sbunlp import SBUNLPZSLearner +from .semanticswingers import SemanticSwingersTermTypingLearner diff --git a/ontolearner/learner/term_typing/semanticswingers.py b/ontolearner/learner/term_typing/semanticswingers.py new file mode 100644 index 00000000..7d9a7469 --- /dev/null +++ b/ontolearner/learner/term_typing/semanticswingers.py @@ -0,0 +1,209 @@ +# Copyright (c) 2025 SciKnowOrg +# +# Licensed under the MIT License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/MIT +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Semantic-Swingers term-typing learner (LLMs4OL 2026, Task B). + +Closed-vocabulary term typing: ``fit`` learns the inventory of allowed type labels +from the train split, and at inference a selector assigns types to each term from +that inventory only. Three selectors are provided (mirroring the team's Task C +taxonomy-discovery learner): + +* ``"embedding"`` (default) — fully offline/deterministic; each term gets its + nearest type label by sentence-embedding cosine similarity. No API key needed. +* ``"openai"`` — the competition champion; an OpenAI chat model classifies terms + against the closed vocabulary with a precision-biased prompt. Enabled only when + an ``api_key`` is supplied (never hard-coded). +* ``"ollama"`` — free, local reproduction of the champion: the same classification + prompt is served by a local Ollama model through its OpenAI-compatible endpoint. +""" +from __future__ import annotations + +import json +import os +import re +from typing import Any, Dict, List, Optional + +import numpy as np +from sentence_transformers import SentenceTransformer + +from ...base import AutoLearner + +_SYSTEM = ( + "You are an expert ontology engineer specialising in term-typing. " + "You are given terms and a closed vocabulary of allowed type labels. " + "Rules: (1) assign a type only when confident — precision over recall; " + "(2) a term may have multiple types; " + "(3) if no allowed type fits, return an empty list for that term; " + "(4) copy type labels EXACTLY as given, case-sensitive. " + 'Answer as JSON: {"typings": [{"term": "...", "types": ["..."]}]}' +) + + +class SemanticSwingersTermTypingLearner(AutoLearner): + """Closed-vocabulary term typing with a swappable type selector. + + Args: + embedding_model: SentenceTransformer id used by the ``"embedding"`` selector. + Defaults to ``mixedbread-ai/mxbai-embed-large-v1`` (the team's champion encoder). + selector: ``"embedding"`` (offline nearest-type), ``"openai"`` (champion LLM + classification), or ``"ollama"`` (local LLM classification, no API key). + llm_model: Chat model id used by the LLM selectors. Defaults to + ``gpt-4.1-mini`` for ``"openai"`` and ``llama3.1:8b`` for ``"ollama"``. + api_key: OpenAI API key for ``selector="openai"``. If ``None``, falls back to + the ``OPENAI_API_KEY`` env var; if still unset, the learner silently + degrades to the embedding selector. Ignored by ``"ollama"``. + base_url: OpenAI-compatible endpoint for the LLM selector. Defaults to the + local Ollama server (``http://localhost:11434/v1``) when + ``selector="ollama"``, and to the OpenAI API otherwise. + max_tokens: Completion budget per LLM request. Thinking models (e.g. Qwen3.5) + spend reasoning tokens before answering and need a much larger budget. + batch_size: Number of terms classified per LLM request. + device: Torch device for the encoder. + """ + + _OLLAMA_BASE_URL = "http://localhost:11434/v1" + _DEFAULT_LLM = {"openai": "gpt-4.1-mini", "ollama": "llama3.1:8b"} + + def __init__( + self, + embedding_model: str = "mixedbread-ai/mxbai-embed-large-v1", + selector: str = "embedding", + llm_model: Optional[str] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + max_tokens: int = 1024, + batch_size: int = 20, + device: str = "cpu", + ) -> None: + """Initialise the learner and record configuration (no I/O yet).""" + super().__init__() + self.embedding_model = embedding_model + self.selector = selector + self.llm_model = llm_model or self._DEFAULT_LLM.get(selector) + self.api_key = api_key or os.environ.get("OPENAI_API_KEY") + if base_url is None and selector == "ollama": + base_url = self._OLLAMA_BASE_URL + self.base_url = base_url + self.max_tokens = max_tokens + self.batch_size = batch_size + self.device = device + self._encoder: Optional[SentenceTransformer] = None + self._allowed_types: List[str] = [] + + def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: + """Load the sentence-embedding encoder. + + Called by ``LearnerPipeline`` as ``load(model_id=llm_id)``. A model id that + looks like a SentenceTransformer path (contains ``/``) overrides the default; + a plain bookkeeping label is ignored. + """ + if model_id and "/" in model_id: + self.embedding_model = model_id + self._encoder = SentenceTransformer(self.embedding_model, device=self.device) + + def _select_embedding(self, terms: List[str]) -> List[Dict[str, Any]]: + """Offline selector: each term gets its nearest type label by cosine similarity.""" + if self._encoder is None: + self.load() + type_emb = np.asarray( + self._encoder.encode( + self._allowed_types, normalize_embeddings=True, show_progress_bar=False + ) + ) + term_emb = np.asarray( + self._encoder.encode(terms, normalize_embeddings=True, show_progress_bar=False) + ) + sim = term_emb @ type_emb.T + best = np.argmax(sim, axis=1) + return [ + {"term": term, "types": [self._allowed_types[int(best[i])]]} + for i, term in enumerate(terms) + ] + + def _parse_typings(self, content: str, allowed: set) -> Dict[str, List[str]]: + """Parse an LLM answer into ``term -> types``, enforcing the closed vocabulary.""" + text = content.strip() + try: + payload = json.loads(text) + except json.JSONDecodeError: + match = re.search(r"\{.*\}", text, flags=re.DOTALL) + if not match: + return {} + try: + payload = json.loads(match.group(0)) + except json.JSONDecodeError: + return {} + items = payload.get("typings", payload) if isinstance(payload, dict) else payload + out: Dict[str, List[str]] = {} + if not isinstance(items, list): + return out + for item in items: + if not isinstance(item, dict): + continue + term = item.get("term") + types = [t for t in item.get("types", []) if t in allowed] + if term: + out[str(term)] = types + return out + + def _select_llm(self, terms: List[str]) -> List[Dict[str, Any]]: + """LLM selector: a chat model classifies term batches against the vocabulary. + + Serves both the ``"openai"`` champion and the local ``"ollama"`` fallback — + the latter is just an OpenAI-compatible endpoint with a placeholder key. + """ + from openai import OpenAI + + api_key = self.api_key if self.selector == "openai" else (self.api_key or "ollama") + client = OpenAI(api_key=api_key, base_url=self.base_url) + allowed = set(self._allowed_types) + vocab_block = "\n".join(f"- {t}" for t in self._allowed_types) + preds: List[Dict[str, Any]] = [] + for start in range(0, len(terms), self.batch_size): + batch = terms[start : start + self.batch_size] + user = ( + "Terms to classify:\n" + + "\n".join(f"- {t}" for t in batch) + + "\n\nAllowed type vocabulary (use ONLY these exact labels):\n" + + vocab_block + + "\n\nAssign types to each term. Answer as JSON." + ) + params: Dict[str, Any] = dict( + model=self.llm_model, + messages=[ + {"role": "system", "content": _SYSTEM}, + {"role": "user", "content": user}, + ], + temperature=0, + max_tokens=self.max_tokens, + ) + try: + resp = client.chat.completions.create( + response_format={"type": "json_object"}, **params + ) + except Exception: + resp = client.chat.completions.create(**params) + typed = self._parse_typings(resp.choices[0].message.content or "", allowed) + preds.extend({"term": t, "types": typed.get(t, [])} for t in batch) + return preds + + def _term_typing(self, data: Any, test: bool = False) -> Optional[List[Dict[str, Any]]]: + """Train mode: learn the type inventory. Test mode: type each term.""" + if not test: + self._allowed_types = sorted(set(data)) + return None + terms = [str(t) for t in data] + if not terms or not self._allowed_types: + return [{"term": t, "types": []} for t in terms] + use_llm = self.selector == "ollama" or (self.selector == "openai" and self.api_key) + return self._select_llm(terms) if use_llm else self._select_embedding(terms) From 2d8e6e04448d28bd752a4e5181747617502ce02a Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Wed, 8 Jul 2026 22:21:05 +0200 Subject: [PATCH 04/23] Add Task B example + docs; restructure Semantic-Swingers docs page for both tasks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wine micro-probe via native LearnerPipeline (term-typing pair F1): embedding 0.39, ollama llama3.1:8b 0.69, openai gpt-4.1-mini 0.69 — the local selector matches the champion on this task. --- .../semanticswingers_learner.rst | 50 ++++++++++++++++--- ...lm_learner_semanticswingers_term_typing.py | 35 +++++++++++++ 2 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 examples/llm_learner_semanticswingers_term_typing.py diff --git a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst index 6581e132..1f9ba5c2 100644 --- a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst +++ b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst @@ -4,14 +4,50 @@ Semantic-Swingers Learner .. sidebar:: Semantic-Swingers Learner Examples + * Term Typing: `llm_learner_semanticswingers_term_typing.py `_ * Taxonomy Discovery: `llm_learner_semanticswingers_taxonomy_discovery.py `_ -The Semantic-Swingers team participated in the LLMs4OL 2026 Shared Task. This page documents the -taxonomy-discovery learner (Task C): a retrieval-first taxonomy inducer with a swappable -parent-selection step. +The Semantic-Swingers team participated in the LLMs4OL 2026 Shared Task. This page documents +the term-typing learner (Task B) and the taxonomy-discovery learner (Task C). Both share the +same design: a strong sentence-embedding encoder plus a swappable selection step with three +interchangeable backends — an offline embedding heuristic (default, no API key), the OpenAI +competition champion, and a free local Ollama reproduction of the champion pipeline. +Term Typing (Task B) +--------------------------------- + +Closed-vocabulary term typing: ``fit`` learns the inventory of allowed type labels from the +train split, and at inference the selector assigns types to each term from that inventory only. + +- ``"embedding"`` (default) — each term gets its nearest type label by sentence-embedding + cosine similarity. Fully offline and deterministic. +- ``"openai"`` — the champion. An OpenAI chat model (default ``gpt-4.1-mini``) classifies + term batches against the closed vocabulary with a precision-biased prompt (multi-type + allowed, abstains when nothing fits, labels copied exactly). +- ``"ollama"`` — the same classification prompt served by a local Ollama model (default + ``llama3.1:8b``). No API key required. + +.. code-block:: python + + from ontolearner import Wine, train_test_split, LearnerPipeline + from ontolearner.learner.term_typing import SemanticSwingersTermTypingLearner + + ontology = Wine() + ontology.load() + train_data, test_data = train_test_split(ontology.extract(), test_size=0.2, random_state=42) + + learner = SemanticSwingersTermTypingLearner(selector="embedding", device="cpu") + + pipeline = LearnerPipeline(llm=learner, llm_id="semanticswingers-term-typing") + outputs = pipeline( + train_data=train_data, + test_data=test_data, + task="term-typing", + evaluate=True, + ) + print(outputs["metrics"]) -Overview +Taxonomy Discovery (Task C) --------------------------------- The learner treats taxonomy discovery as *retrieve-then-select*: @@ -43,7 +79,7 @@ inference time, so it works on unseen ontologies without any target-vocabulary assumptions. Loading Ontological Data ----------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: python @@ -56,7 +92,7 @@ Loading Ontological Data train_data, test_data = train_test_split(data, test_size=0.2, random_state=42) Initialize Learner ------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: python @@ -81,7 +117,7 @@ Initialize Learner # ) Run the Pipeline ------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The learner runs on raw ontology objects, so pass ``ontologizer_data=False``. diff --git a/examples/llm_learner_semanticswingers_term_typing.py b/examples/llm_learner_semanticswingers_term_typing.py new file mode 100644 index 00000000..d7c22979 --- /dev/null +++ b/examples/llm_learner_semanticswingers_term_typing.py @@ -0,0 +1,35 @@ +from ontolearner import Wine, train_test_split, LearnerPipeline +from ontolearner.learner.term_typing import SemanticSwingersTermTypingLearner + +# 1) Load & split +ontology = Wine() +ontology.load() +data = ontology.extract() +train_data, test_data = train_test_split(data, test_size=0.2, random_state=42) + +# 2) Configure the semantic-swingers term-typing learner. +# Offline default (no API key): the "embedding" selector (nearest type label). +# Champion: selector="openai" + api_key=... for gpt-4.1-mini classification. +# Local champion-reproduction (no API key): selector="ollama" (local Ollama server). +learner = SemanticSwingersTermTypingLearner( + embedding_model="mixedbread-ai/mxbai-embed-large-v1", + selector="embedding", + device="cpu", +) + +# 3) Build pipeline (pass our learner as `llm`) +pipeline = LearnerPipeline( + llm=learner, + llm_id="semanticswingers-term-typing", +) + +# 4) Train (learn the type inventory) + predict + evaluate +outputs = pipeline( + train_data=train_data, + test_data=test_data, + task="term-typing", + evaluate=True, +) + +print("Metrics:", outputs.get("metrics")) +print("Elapsed time:", outputs["elapsed_time"]) From 706999fb90800f96753fd93587fbd150629e64d8 Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Wed, 8 Jul 2026 22:58:50 +0200 Subject: [PATCH 05/23] test: unit tests + reproducibility notes for Semantic-Swingers learners Hermetic unit tests for SemanticSwingersTermTypingLearner and SemanticSwingersTaxonomyLearner (Tasks B/C): fit/predict contract, embedding-selector determinism, closed-vocabulary parsing in _parse_typings, and openai-selector fallback to the offline embedding selector when no api_key is available (no network attempted, verified via mocked OpenAI client). Encoders are mocked with fixed vectors so tests need no downloaded models, network, or ontology data. Adds a Reproducibility subsection to the learner docs page documenting which selector reproduces which reported number and how api_key resolution/fallback behaves. --- CHANGELOG.md | 3 + .../semanticswingers_learner.rst | 23 +++ ...est_semanticswingers_taxonomy_discovery.py | 156 ++++++++++++++++++ tests/test_semanticswingers_term_typing.py | 153 +++++++++++++++++ 4 files changed, 335 insertions(+) create mode 100644 tests/test_semanticswingers_taxonomy_discovery.py create mode 100644 tests/test_semanticswingers_term_typing.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8879a20f..4e4dc549 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ - Add Semantic-Swingers taxonomy-discovery learner (LLMs4OL 2026, Task C): retrieval-first taxonomy induction with a swappable parent selector (offline embedding baseline / OpenAI champion / local Ollama reproduction), plus documentation page and example. +- Add hermetic unit tests for both Semantic-Swingers learners (fit/predict contract, + embedding-selector determinism, closed-vocabulary parsing, openai-selector fallback with + no api_key) and a Reproducibility subsection to the Semantic-Swingers docs page. ### v1.5.1 (March 30, 2026) - Fix challenge learner diff --git a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst index 1f9ba5c2..da563db6 100644 --- a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst +++ b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst @@ -140,3 +140,26 @@ The learner runs on raw ontology objects, so pass ``ontologizer_data=False``. ) print(outputs["metrics"]) + +Reproducibility +--------------------------------- + +Which selector reproduces which reported number, and what is required to run it: + +- **Term typing (Task B)** — the offline ``"embedding"`` selector alone gets close to the + competition champion on Wine (local ``≈0.687`` vs. the champion's ``0.690``). No API key + is needed to reproduce this figure. +- **Taxonomy discovery (Task C)** — the gap is much larger: the paid champion selector + (``"openai"``) reaches ``0.21``, while the offline ``"embedding"`` heuristic reaches only + ``0.07``. Reproducing the champion number for this task requires an OpenAI API key (or the + local ``"ollama"`` selector as a free, unverified approximation of the same prompting + strategy). +- **Determinism** — both offline ``"embedding"`` selectors are fully deterministic: same + encoder, same inputs, same outputs, every run (no sampling, ``temperature`` is irrelevant + since no LLM is called). The ``"openai"``/``"ollama"`` selectors call ``temperature=0`` + but LLM outputs are not guaranteed bit-for-bit reproducible across provider versions. +- **API key handling** — ``selector="openai"`` reads ``api_key`` if passed explicitly, + otherwise falls back to the ``OPENAI_API_KEY`` environment variable; if neither is set, + the learner silently degrades to the offline ``"embedding"`` selector rather than raising, + so pipelines never hard-fail for lack of a key. ``selector="ollama"`` never reads + ``OPENAI_API_KEY`` and needs no key at all — only a local Ollama server. diff --git a/tests/test_semanticswingers_taxonomy_discovery.py b/tests/test_semanticswingers_taxonomy_discovery.py new file mode 100644 index 00000000..a0026a6b --- /dev/null +++ b/tests/test_semanticswingers_taxonomy_discovery.py @@ -0,0 +1,156 @@ +from types import SimpleNamespace + +import numpy as np +import pytest +from unittest.mock import MagicMock, patch + +from ontolearner.learner.taxonomy_discovery.semanticswingers import ( + SemanticSwingersTaxonomyLearner, +) + +MODULE = "ontolearner.learner.taxonomy_discovery.semanticswingers" + + +def _relation(parent, child): + return SimpleNamespace(parent=parent, child=child) + + +def _ontology_data(types, taxonomies): + return SimpleNamespace( + type_taxonomies=SimpleNamespace( + types=types, + taxonomies=[_relation(p, c) for p, c in taxonomies], + ) + ) + + +def _make_encoder(vectors): + """A fake SentenceTransformer whose ``encode`` looks vectors up by text.""" + def encode(texts, normalize_embeddings=True, show_progress_bar=False): + return np.array([vectors[t] for t in texts]) + + encoder = MagicMock() + encoder.encode.side_effect = encode + return encoder + + +# Unit vectors (the real encoder is called with normalize_embeddings=True). +# "Animal" sits at 45 degrees, equidistant from "Dog" (10 degrees) and "Cat" +# (80 degrees), so it has the highest mean similarity to the vocabulary and +# wins the generality tie-break; "Dog" and "Cat" are far apart from each other. +VECTORS = { + "Animal": [0.70711, 0.70711], + "Dog": [0.98481, 0.17365], + "Cat": [0.17365, 0.98481], +} + + +@pytest.fixture +def mock_encoder(): + with patch(f"{MODULE}.SentenceTransformer") as MockST: + MockST.return_value = _make_encoder(VECTORS) + yield MockST + + +@pytest.fixture(autouse=True) +def no_openai_key(monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + + +def test_nodes_dedups_declared_types_and_edge_endpoints(): + data = _ontology_data( + types=["Animal", "Dog"], + taxonomies=[("Animal", "Dog"), ("Animal", "Cat")], + ) + nodes = SemanticSwingersTaxonomyLearner._nodes(data) + + assert nodes == ["Animal", "Dog", "Cat"] + + +def test_tasks_data_former_passes_data_through_unchanged(): + data = _ontology_data(types=["Animal"], taxonomies=[]) + learner = SemanticSwingersTaxonomyLearner() + + assert learner.tasks_data_former(data, task="taxonomy-discovery", test=True) is data + + +def test_fit_is_a_noop(): + learner = SemanticSwingersTaxonomyLearner() + data = _ontology_data(types=["Animal"], taxonomies=[]) + + assert learner._taxonomy_discovery(data, test=False) is None + + +def test_predict_returns_parent_child_dicts(mock_encoder): + learner = SemanticSwingersTaxonomyLearner(device="cpu") + data = _ontology_data( + types=["Animal", "Dog", "Cat"], + taxonomies=[("Animal", "Dog"), ("Animal", "Cat")], + ) + + preds = learner._taxonomy_discovery(data, test=True) + + assert len(preds) > 0 + assert all(set(p.keys()) == {"parent", "child"} for p in preds) + + +def test_embedding_selector_picks_most_general_candidate(mock_encoder): + learner = SemanticSwingersTaxonomyLearner(device="cpu", top_k=10) + data = _ontology_data( + types=["Animal", "Dog", "Cat"], + taxonomies=[("Animal", "Dog"), ("Animal", "Cat")], + ) + + preds = learner._taxonomy_discovery(data, test=True) + by_child = {p["child"]: p["parent"] for p in preds} + + assert by_child["Dog"] == "Animal" + assert by_child["Cat"] == "Animal" + # "Animal" is the most general node overall, so it never gets a parent. + assert "Animal" not in by_child + + +def test_embedding_selector_is_deterministic(mock_encoder): + learner = SemanticSwingersTaxonomyLearner(device="cpu") + data = _ontology_data( + types=["Animal", "Dog", "Cat"], + taxonomies=[("Animal", "Dog"), ("Animal", "Cat")], + ) + + first = learner._taxonomy_discovery(data, test=True) + second = learner._taxonomy_discovery(data, test=True) + + assert first == second + + +def test_openai_selector_without_api_key_falls_back_to_embedding(mock_encoder): + with patch("openai.OpenAI") as MockOpenAI: + learner = SemanticSwingersTaxonomyLearner(selector="openai", device="cpu") + data = _ontology_data( + types=["Animal", "Dog", "Cat"], + taxonomies=[("Animal", "Dog"), ("Animal", "Cat")], + ) + + assert learner.api_key is None + preds = learner._taxonomy_discovery(data, test=True) + + by_child = {p["child"]: p["parent"] for p in preds} + assert by_child["Dog"] == "Animal" + MockOpenAI.assert_not_called() + + +def test_ollama_selector_defaults(): + learner = SemanticSwingersTaxonomyLearner(selector="ollama") + assert learner.llm_model == "llama3.1:8b" + assert learner.base_url == "http://localhost:11434/v1" + + +def test_openai_selector_defaults(): + learner = SemanticSwingersTaxonomyLearner(selector="openai") + assert learner.llm_model == "gpt-4.1-mini" + assert learner.base_url is None + + +def test_explicit_llm_model_overrides_default(): + learner = SemanticSwingersTaxonomyLearner(selector="ollama", llm_model="qwen3.5") + assert learner.llm_model == "qwen3.5" diff --git a/tests/test_semanticswingers_term_typing.py b/tests/test_semanticswingers_term_typing.py new file mode 100644 index 00000000..69838acf --- /dev/null +++ b/tests/test_semanticswingers_term_typing.py @@ -0,0 +1,153 @@ +import numpy as np +import pytest +from unittest.mock import MagicMock, patch + +from ontolearner.learner.term_typing.semanticswingers import ( + SemanticSwingersTermTypingLearner, +) + +MODULE = "ontolearner.learner.term_typing.semanticswingers" + + +def _make_encoder(vectors): + """A fake SentenceTransformer whose ``encode`` looks vectors up by text.""" + def encode(texts, normalize_embeddings=True, show_progress_bar=False): + return np.array([vectors[t] for t in texts]) + + encoder = MagicMock() + encoder.encode.side_effect = encode + return encoder + + +VECTORS = { + "Animal": [1.0, 0.0], + "Plant": [0.0, 1.0], + "dog": [0.9, 0.1], + "rose": [0.1, 0.9], +} + + +@pytest.fixture +def mock_encoder(): + with patch(f"{MODULE}.SentenceTransformer") as MockST: + MockST.return_value = _make_encoder(VECTORS) + yield MockST + + +@pytest.fixture(autouse=True) +def no_openai_key(monkeypatch): + """Hermetic default: no OPENAI_API_KEY leaks in from the environment.""" + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + + +def test_fit_learns_type_inventory_from_list(): + learner = SemanticSwingersTermTypingLearner() + learner._term_typing(["Animal", "Plant", "Animal"], test=False) + + assert learner._allowed_types == ["Animal", "Plant"] + + +def test_predict_returns_entry_per_term(mock_encoder): + learner = SemanticSwingersTermTypingLearner(device="cpu") + learner._allowed_types = ["Animal", "Plant"] + + preds = learner._term_typing(["dog", "rose"], test=True) + + assert len(preds) == 2 + assert {p["term"] for p in preds} == {"dog", "rose"} + assert all(set(p.keys()) == {"term", "types"} for p in preds) + + +def test_embedding_selector_picks_nearest_type(mock_encoder): + learner = SemanticSwingersTermTypingLearner(device="cpu") + learner._allowed_types = ["Animal", "Plant"] + + preds = learner._term_typing(["dog", "rose"], test=True) + by_term = {p["term"]: p["types"] for p in preds} + + assert by_term["dog"] == ["Animal"] + assert by_term["rose"] == ["Plant"] + + +def test_predict_empty_terms_and_empty_vocab_short_circuit(): + learner = SemanticSwingersTermTypingLearner() + learner._allowed_types = ["Animal"] + assert learner._term_typing([], test=True) == [] + + learner2 = SemanticSwingersTermTypingLearner() + learner2._allowed_types = [] + assert learner2._term_typing(["dog"], test=True) == [{"term": "dog", "types": []}] + + +def test_embedding_selector_is_deterministic(mock_encoder): + learner = SemanticSwingersTermTypingLearner(device="cpu") + learner._allowed_types = ["Animal", "Plant"] + + first = learner._term_typing(["dog", "rose"], test=True) + second = learner._term_typing(["dog", "rose"], test=True) + + assert first == second + + +class TestParseTypings: + def setup_method(self): + self.learner = SemanticSwingersTermTypingLearner() + self.allowed = {"Animal", "Plant"} + + def test_parses_plain_json(self): + content = '{"typings": [{"term": "dog", "types": ["Animal"]}]}' + result = self.learner._parse_typings(content, self.allowed) + assert result == {"dog": ["Animal"]} + + def test_drops_types_outside_closed_vocabulary(self): + content = '{"typings": [{"term": "dog", "types": ["Animal", "Robot"]}]}' + result = self.learner._parse_typings(content, self.allowed) + assert result == {"dog": ["Animal"]} + + def test_malformed_json_returns_empty_dict(self): + result = self.learner._parse_typings("not json at all", self.allowed) + assert result == {} + + def test_recovers_fenced_json(self): + content = ( + "Here is the answer:\n```json\n" + '{"typings": [{"term": "rose", "types": ["Plant"]}]}\n```' + ) + result = self.learner._parse_typings(content, self.allowed) + assert result == {"rose": ["Plant"]} + + def test_top_level_list_payload(self): + content = '[{"term": "dog", "types": ["Animal"]}]' + result = self.learner._parse_typings(content, self.allowed) + assert result == {"dog": ["Animal"]} + + +def test_openai_selector_without_api_key_falls_back_to_embedding(mock_encoder): + with patch("openai.OpenAI") as MockOpenAI: + learner = SemanticSwingersTermTypingLearner(selector="openai", device="cpu") + learner._allowed_types = ["Animal", "Plant"] + + assert learner.api_key is None + preds = learner._term_typing(["dog", "rose"], test=True) + + by_term = {p["term"]: p["types"] for p in preds} + assert by_term["dog"] == ["Animal"] + assert by_term["rose"] == ["Plant"] + MockOpenAI.assert_not_called() + + +def test_ollama_selector_defaults(): + learner = SemanticSwingersTermTypingLearner(selector="ollama") + assert learner.llm_model == "llama3.1:8b" + assert learner.base_url == "http://localhost:11434/v1" + + +def test_openai_selector_defaults(): + learner = SemanticSwingersTermTypingLearner(selector="openai") + assert learner.llm_model == "gpt-4.1-mini" + assert learner.base_url is None + + +def test_explicit_llm_model_overrides_default(): + learner = SemanticSwingersTermTypingLearner(selector="ollama", llm_model="qwen3.5") + assert learner.llm_model == "qwen3.5" From 5c87d2a6b79ac38e7b6f8cc464e5837e370a694a Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Thu, 9 Jul 2026 07:40:30 +0200 Subject: [PATCH 06/23] feat: add Semantic-Swingers text2onto + taxonomy-discovery learner (Task A, flagship) One learner class, two hooks, dispatched on the task string: - _text2onto: RAG (top-k document exemplars) generation with a LoRA fine-tuned Qwen/Qwen3.5-9B (RA-FT/base-FT), the competition champion, projecting extracted triples onto OntoLearner's native {terms, types}. - _taxonomy_discovery: composed (not rewritten) from the existing SemanticSwingersTaxonomyLearner, since the native harness for that task hands over a bare type vocabulary with no source document text. qwen3_5/qwen3_next has no released transformers support yet, so the git-source requirement is a lazy import inside the learner raising an actionable ImportError with the exact pinned-commit pip install line (verified 2026-07-09 to both register the architecture and produce coherent triple extraction with the real PEFT adapter) rather than a core pyproject.toml dependency. Adds the docs section, example script, and 19 hermetic unit tests (mocked LLM/retriever, no network). --- CHANGELOG.md | 7 + .../semanticswingers_learner.rst | 77 +++- .../llm_learner_semanticswingers_text2onto.py | 85 ++++ ontolearner/learner/text2onto/__init__.py | 1 + .../learner/text2onto/semanticswingers.py | 363 ++++++++++++++++++ tests/test_semanticswingers_text2onto.py | 219 +++++++++++ 6 files changed, 748 insertions(+), 4 deletions(-) create mode 100644 examples/llm_learner_semanticswingers_text2onto.py create mode 100644 ontolearner/learner/text2onto/semanticswingers.py create mode 100644 tests/test_semanticswingers_text2onto.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e4dc549..1a4ba843 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ - Add hermetic unit tests for both Semantic-Swingers learners (fit/predict contract, embedding-selector determinism, closed-vocabulary parsing, openai-selector fallback with no api_key) and a Reproducibility subsection to the Semantic-Swingers docs page. +- Add Semantic-Swingers text2onto + taxonomy-discovery learner (LLMs4OL 2026, Task A, + flagship): one class, two hooks — retrieval-augmented generation with a LoRA fine-tuned + Qwen3.5-9B (RA-FT/base-FT) for text2onto, composed with the existing taxonomy-discovery + learner (not a rewrite) for the vocabulary-only native harness. Requires transformers + installed from git source for the qwen3_5 architecture (lazy-imported, actionable + ImportError if missing — not added to core pyproject.toml), plus documentation section, + example, and hermetic unit tests. ### v1.5.1 (March 30, 2026) - Fix challenge learner diff --git a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst index da563db6..b9851553 100644 --- a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst +++ b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst @@ -6,12 +6,16 @@ Semantic-Swingers Learner * Term Typing: `llm_learner_semanticswingers_term_typing.py `_ * Taxonomy Discovery: `llm_learner_semanticswingers_taxonomy_discovery.py `_ + * Text2Onto (Task A, flagship): `llm_learner_semanticswingers_text2onto.py `_ The Semantic-Swingers team participated in the LLMs4OL 2026 Shared Task. This page documents -the term-typing learner (Task B) and the taxonomy-discovery learner (Task C). Both share the -same design: a strong sentence-embedding encoder plus a swappable selection step with three -interchangeable backends — an offline embedding heuristic (default, no API key), the OpenAI -competition champion, and a free local Ollama reproduction of the champion pipeline. +the term-typing learner (Task B), the taxonomy-discovery learner (Task C), and the flagship +text2onto + taxonomy-discovery learner (Task A). Tasks B/C share the same design: a strong +sentence-embedding encoder plus a swappable selection step with three interchangeable +backends — an offline embedding heuristic (default, no API key), the OpenAI competition +champion, and a free local Ollama reproduction of the champion pipeline. Task A is different: +its champion is not a prompted API model but the team's own LoRA-fine-tuned open model, so its +learner is a generative retrieval-augmented-generation pipeline instead. Term Typing (Task B) --------------------------------- @@ -141,6 +145,71 @@ The learner runs on raw ontology objects, so pass ``ontologizer_data=False``. print(outputs["metrics"]) +Text2Onto + Taxonomy Discovery, joint (Task A, flagship) +--------------------------------------------------------- + +``SemanticSwingersText2OntoLearner`` is ONE class implementing TWO hooks, dispatched via the +``task`` string ``AutoLearner.fit``/``predict`` already receive: + +- ``_text2onto`` — the team's competition champion: retrieval-augmented generation (RAG, + top-``k`` document exemplars) with a LoRA fine-tuned ``Qwen/Qwen3.5-9B`` (RA-FT), extracting + ``[subject, relation, object]`` triples per document and projecting them onto the native + ``{"terms": [...], "types": [...]}`` shape. +- ``_taxonomy_discovery`` — delegates to :class:`SemanticSwingersTaxonomyLearner` (Task C, + documented above) **by composition, not a rewrite**. The native taxonomy-discovery harness + hands the learner a bare type vocabulary with no source document text, so the RAG+FT + generator — which needs text to extract triples from — cannot serve that path; the team's + proven embedding-retrieval taxonomy inducer is the right tool there instead. Expect the + native taxonomy F1 this hook reports to differ from the team's own joint + ``graph_similarity`` figure (RA-FT k10, val_20: ``0.6688``) — that score is a different, + combined metric (term + type + edge overlap together) computed on the team's own document + corpus, not OntoLearner's standalone taxonomy metric on a vocabulary-only benchmark ontology. + A gap here is an expected apples-to-oranges artifact, not a regression. + +Model portability (why this needs an unusual install) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``Qwen/Qwen3.5-9B`` uses the ``qwen3_5``/``qwen3_next`` hybrid (dense + linear-attention) +architecture. As of 2026-07-09 no *released* ``transformers`` version registers it — only +``transformers`` installed from git source does: + +.. code-block:: bash + + pip install "transformers @ git+https://github.com/huggingface/transformers.git@1f2fd05824a7ef71a767a122ebd7526ca4e55e40" \ + "peft>=0.19" "accelerate>=1.0" + +This exact commit was verified (2026-07-09) to both register the architecture *and* produce +coherent, on-topic triple extraction when loading the base model plus the team's PEFT adapter — +**never a manually merged/fused checkpoint**: that route was tried and abandoned after producing +a byte-identical-but-semantically-garbage state dict load (see +``docs/ontolearner-native-integration-poc.md`` in the team's main repo, ``llms4ol-2026``, for +the full investigation). This requirement is deliberately **not** added to OntoLearner's core +``pyproject.toml`` — it is heavy (a from-source build) and a moving target that only this one +learner needs. Calling ``learner.load()`` without it raises a clear ``ImportError`` naming the +exact command above. + +.. code-block:: python + + from ontolearner import LearnerPipeline + from ontolearner.learner.text2onto import SemanticSwingersText2OntoLearner + + train_data = { + "documents": [{"doc_id": "d1", "text": "A poodle is a dog. A dog is a mammal."}], + "triples": {"d1": [["poodle", "is-a", "dog"], ["dog", "is-a", "mammal"]]}, + } + test_data = {"documents": [{"doc_id": "d2", "text": "A tabby is a cat."}]} + + # RA-FT (champion): trained WITH exemplars baked in, wants top_k > 0. + # adapter="baseft", top_k=0 selects the retrieval-free standard fine-tune instead. + learner = SemanticSwingersText2OntoLearner(adapter="raft", top_k=1, device="cpu") + + pipeline = LearnerPipeline(llm=learner, llm_id="semanticswingers-text2onto", ontologizer_data=False) + outputs = pipeline( + train_data=train_data, test_data=test_data, + task="text2onto", evaluate=False, ontologizer_data=False, + ) + print(outputs["predictions"]) + Reproducibility --------------------------------- diff --git a/examples/llm_learner_semanticswingers_text2onto.py b/examples/llm_learner_semanticswingers_text2onto.py new file mode 100644 index 00000000..c76a9e98 --- /dev/null +++ b/examples/llm_learner_semanticswingers_text2onto.py @@ -0,0 +1,85 @@ +"""Semantic-Swingers Task A (flagship): text2onto with the RA-FT Qwen3.5-9B generator. + +Requires transformers installed from git source (no released version registers the qwen3_5 +architecture yet) plus peft. See the ImportError raised by `learner.load()` for the exact +pinned pip install command, or docs/ontolearner-native-integration-poc.md in the team's main +repo (llms4ol-2026) for the full model-portability investigation: + + pip install "transformers @ git+https://github.com/huggingface/transformers.git@\ +" \ +"peft>=0.19" "accelerate>=1.0" + +The `adapter` argument accepts a local filesystem path (fastest way to try this without HF +access — point it at the team's `data/ft/_runners/raft_adapter/final` or `baseft_adapter/final`) +or a HuggingFace repo id once the adapters are published (see yp4). +""" + +from ontolearner import LearnerPipeline +from ontolearner.learner.text2onto import SemanticSwingersText2OntoLearner + +# ---- Toy document set (self-contained; swap in the competition's real train/test docs) ---- +train_data = { + "documents": [ + { + "doc_id": "d1", + "text": "A poodle is a dog. A dog is a mammal. A mammal is an animal.", + }, + ], + "triples": { + "d1": [ + ["poodle", "is-a", "dog"], + ["dog", "is-a", "mammal"], + ["mammal", "is-a", "animal"], + ], + }, +} +test_data = { + "documents": [ + { + "doc_id": "d2", + "text": "A tabby is a cat. A cat is a mammal.", + }, + ], +} + +# ---- Configure the learner ---- +# RA-FT (competition champion): trained WITH exemplars baked into the prompt, wants top_k > 0. +# Swap adapter="baseft", top_k=0 for the retrieval-free standard fine-tune instead. +learner = SemanticSwingersText2OntoLearner( + adapter="raft", # or a local path, e.g. "../llms4ol-2026/data/ft/_runners/raft_adapter/final" + top_k=1, + device="cpu", # "cuda" strongly recommended — see the class docstring on CPU speed +) + +pipeline = LearnerPipeline( + llm=learner, + llm_id="semanticswingers-text2onto", + ontologizer_data=False, +) + +outputs = pipeline( + train_data=train_data, + test_data=test_data, + task="text2onto", + evaluate=False, # no gold terms2docs/terms2types provided for this toy example + ontologizer_data=False, +) + +print(outputs["predictions"]) +print("Elapsed time:", outputs["elapsed_time"]) + +# ---- The same learner also serves taxonomy-discovery (composed, not a rewrite) ---- +# from ontolearner import Wine, train_test_split +# +# wine = Wine() +# wine.load() +# wine_train, wine_test = train_test_split(wine.extract(), test_size=0.2, random_state=42) +# +# taxonomy_outputs = pipeline( +# train_data=wine_train, +# test_data=wine_test, +# task="taxonomy-discovery", +# evaluate=True, +# ontologizer_data=False, +# ) +# print(taxonomy_outputs["metrics"]) diff --git a/ontolearner/learner/text2onto/__init__.py b/ontolearner/learner/text2onto/__init__.py index af31523c..83651169 100644 --- a/ontolearner/learner/text2onto/__init__.py +++ b/ontolearner/learner/text2onto/__init__.py @@ -14,3 +14,4 @@ from .alexbek import AlexbekRAGFewShotLearner from .sbunlp import SBUNLPFewShotLearner +from .semanticswingers import SemanticSwingersText2OntoLearner diff --git a/ontolearner/learner/text2onto/semanticswingers.py b/ontolearner/learner/text2onto/semanticswingers.py new file mode 100644 index 00000000..45ac6f8d --- /dev/null +++ b/ontolearner/learner/text2onto/semanticswingers.py @@ -0,0 +1,363 @@ +# Copyright (c) 2025 SciKnowOrg +# +# Licensed under the MIT License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/MIT +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Semantic-Swingers Task A (flagship) learner (LLMs4OL 2026): joint text2onto + taxonomy-discovery. + +ONE learner class, TWO hooks, dispatched via ``AutoLearner.fit``/``predict``'s ``task`` string: + +* ``_text2onto`` — the team's competition champion: retrieval-augmented generation (RAG, top-k + document exemplars) with a LoRA fine-tuned ``Qwen/Qwen3.5-9B`` (RA-FT), extracting + ``[subject, relation, object]`` triples per document and projecting them onto OntoLearner's + native ``{"terms": [...], "types": [...]}`` shape. This is a document-driven generative method + (needs source text), unlike Task B/C's classification-style selectors. +* ``_taxonomy_discovery`` — reuses :class:`~ontolearner.learner.taxonomy_discovery.semanticswingers. + SemanticSwingersTaxonomyLearner` **by composition** (delegation, not a rewrite). The native + OntoLearner taxonomy-discovery harness hands the learner a bare type *vocabulary* with no source + text (see ``AutoLearner.tasks_data_former``'s default reshaping for that task), so the RAG+FT + generator — which needs document text to extract triples from — cannot serve that invocation + path; the team's proven embedding-retrieval taxonomy inducer is the right tool there instead. + Expect the native taxonomy-discovery F1 reported by this hook to differ from the team's joint + ``graph_similarity`` score (val_20 RA-FT k10 = 0.6688) — that figure scores a *different*, + combined metric (term/type/edge overlap together) on the team's own document corpus, not + OntoLearner's standalone taxonomy metric on a vocabulary-only benchmark ontology. A gap here is + an expected apples-to-oranges artifact, not a regression. + +Model portability background (``docs/ontolearner-native-integration-poc.md`` in the team's main +repo, section "Model portability"): ``Qwen/Qwen3.5-9B`` uses the ``qwen3_5``/``qwen3_next`` hybrid +(dense + linear-attention) architecture, which no *released* ``transformers`` version registers as +of 2026-07-09 — only ``transformers`` installed from git source does. Loading the base model plus +the team's PEFT LoRA adapter **unmerged** (never a manually fused/merged checkpoint — that route +was tried and abandoned after producing incoherent output despite a byte-identical state dict) is +the exact recipe already used to produce every graded Task A result the team has, and is the one +implemented here. +""" +from __future__ import annotations + +import ast +import json +import re +from typing import Any, Dict, List, Optional, Tuple + +from ...base import AutoLearner, AutoRetriever +from ..taxonomy_discovery.semanticswingers import SemanticSwingersTaxonomyLearner + +#: Exact transformers commit verified (2026-07-09) to (a) register `qwen3_5`/`qwen3_next` in +#: `CONFIG_MAPPING` and (b) load base `Qwen/Qwen3.5-9B` + the team's PEFT adapter producing +#: coherent, on-topic triple extraction on a real held-out document (not just a token count). +#: Never float `@main` — the qwen3_5 modeling code is young enough that semantics have already +#: drifted once between dev snapshots (see the POC doc's fused-checkpoint post-mortem). +_VERIFIED_TRANSFORMERS_COMMIT = "1f2fd05824a7ef71a767a122ebd7526ca4e55e40" +_PIP_INSTALL_LINE = ( + f'pip install "transformers @ git+https://github.com/huggingface/transformers.git' + f'@{_VERIFIED_TRANSFORMERS_COMMIT}" "peft>=0.19" "accelerate>=1.0"' +) + +_SYSTEM_PROMPT = ( + "You are an expert ontology engineer. Extract a primitive ontology from the document as " + "triples [subject, relation, object].\n" + "RULES:\n" + "1. Extract Terms (instances) and Types (classes).\n" + "2. Dominant relations: 'is-a' (subclassing), 'instance-of' or 'type' (term typing), " + "'equivalent class' (synonyms), 'disjoint with', 'part_of'/'has part'. Use these exact " + "labels; reuse consistent labels.\n" + "3. WARNING: Most texts only contain taxonomic definitions. Do NOT extract semantic " + "relationships unless explicitly demanded by strict logical constraints. If there are no " + "relations other than 'is-a', do not invent them.\n" + "4. Direction matters: emit Child -> is-a -> Parent.\n" + 'Output ONLY JSON {"triples": [[subject, relation, object], ...]}; entities grounded in ' + "the text." +) +_NOTHINK_SUFFIX = "<|im_start|>assistant\n\n\n\n\n" +_TYPING_RELATIONS = {"is-a", "instance-of", "type"} + +# HuggingFace repo ids for the two published PEFT adapters (see yp4 for the publish step; these +# are placeholders until the private HF repos exist — override via the `adapter` constructor arg +# with a local path in the meantime, e.g. the team's `data/ft/_runners/raft_adapter/final`). +_ADAPTER_REPOS = { + "raft": "semanticswingers/qwen3.5-9b-raft-taska", + "baseft": "semanticswingers/qwen3.5-9b-baseft-taska", +} + + +def _require_qwen35_transformers() -> None: + """Fail loudly and actionably if the installed transformers doesn't know qwen3_5. + + Lazy import guard (RWTH/g4f optional-dep precedent): the git-source transformers requirement + is NOT added to OntoLearner's core ``pyproject.toml`` — it is heavy (build-from-source), a + moving target, and only this one learner needs it. Callers see a clear, copy-pasteable fix. + """ + try: + from transformers.models.auto.configuration_auto import CONFIG_MAPPING + except ImportError as e: # pragma: no cover - transformers itself missing + raise ImportError( + "SemanticSwingersText2OntoLearner requires transformers with qwen3_5/qwen3_next " + f"support, but transformers is not importable at all. Install it with:\n" + f" {_PIP_INSTALL_LINE}" + ) from e + if "qwen3_5" not in CONFIG_MAPPING: + raise ImportError( + "SemanticSwingersText2OntoLearner requires a transformers build that registers the " + "'qwen3_5' architecture (Qwen/Qwen3.5-9B's model type). No released transformers " + "version ships this yet (verified 2026-07-09) — install from git source, pinned to " + "the commit this learner was validated against:\n" + f" {_PIP_INSTALL_LINE}\n" + "(see docs/ontolearner-native-integration-poc.md in the team's main repo for the " + "full portability investigation)." + ) + + +def _parse_triples_json(text: str) -> List[Any]: + """Robust JSON/markdown-fence extraction of a ``{"triples": [...]}`` payload. + + Ported (self-contained, no cross-repo import) from the team's ``src/experiments/ + triple_utils.py::_parse`` — kept intentionally small so the fork carries no dependency on the + private llms4ol-2026 competition repo. + """ + if not isinstance(text, str) or not text: + return [] + m = re.search(r"`{3}(?:json)?\s*([\s\S]*?)\s*`{3}", text, re.IGNORECASE) + body = m.group(1).strip() if m else text + if not m: + a, b = body.find("["), body.rfind("]") + if a != -1 and b != -1: + body = body[a : b + 1] + for loader in (json.loads, ast.literal_eval): + try: + d = loader(body) + if isinstance(d, dict): + d = d.get("triples", []) + if isinstance(d, list): + return d + except Exception: + continue + return [] + + +def _to_triples(items: List[Any]) -> List[Tuple[str, str, str]]: + """Coerce parsed triple items (list-of-3 or dict shapes) into clean ``(s, r, o)`` tuples.""" + out: List[Tuple[str, str, str]] = [] + for t in items or []: + if isinstance(t, dict): + s = t.get("subject", t.get("head", t.get("source", ""))) + r = t.get("relation", t.get("predicate", t.get("type", ""))) + o = t.get("object", t.get("tail", t.get("target", ""))) + elif isinstance(t, (list, tuple)) and len(t) == 3: + s, r, o = t + else: + continue + s, r, o = str(s).strip(), str(r).strip().lower(), str(o).strip() + if s and r and o: + out.append((s, r, o)) + return out + + +class SemanticSwingersText2OntoLearner(AutoLearner): + """RAG + LoRA-fine-tuned Qwen3.5-9B triple extractor, serving text2onto and taxonomy-discovery. + + Args: + adapter: ``"raft"`` (retrieval-aware fine-tune, the competition champion — trained with + exemplars baked into the prompt, wants ``top_k > 0`` at inference) or ``"baseft"`` + (standard fine-tune, trained without exemplars — best used retrieval-free, + ``top_k=0``). Either may also be a local filesystem path or a HuggingFace repo id to + a PEFT adapter directory, overriding the published default for that name. + base_model_id: HuggingFace id of the frozen base model the adapter was trained on. + top_k: Number of retrieved training-document exemplars baked into the generation prompt + (RA-FT's regime; ``0`` degrades to zero-shot, base-FT's regime). + retriever_model_id: SentenceTransformer id used to embed documents for exemplar retrieval. + max_new_tokens: Generation budget per document. + device: Torch device string (``"cpu"``, ``"cuda"``, ``"mps"``, ...). CPU works but is + slow without the ``fla``/``causal-conv1d`` fast kernels (GPU-only); see the class + docstring's model-portability note. + taxonomy_kwargs: Extra keyword arguments forwarded to the composed + :class:`SemanticSwingersTaxonomyLearner` used for the ``_taxonomy_discovery`` hook. + """ + + def __init__( + self, + adapter: str = "raft", + base_model_id: str = "Qwen/Qwen3.5-9B", + top_k: int = 10, + retriever_model_id: str = "sentence-transformers/all-MiniLM-L6-v2", + max_new_tokens: int = 1500, + device: str = "cpu", + taxonomy_kwargs: Optional[Dict[str, Any]] = None, + ) -> None: + """Initialise the learner and record configuration (no I/O, no model load yet).""" + super().__init__() + self.adapter = _ADAPTER_REPOS.get(adapter, adapter) + self.base_model_id = base_model_id + self.top_k = top_k + self.retriever_model_id = retriever_model_id + self.max_new_tokens = max_new_tokens + self.device = device + + self._model: Optional[Any] = None + self._tokenizer: Optional[Any] = None + self._retriever = AutoRetriever() + self._train_docs: List[Dict[str, Any]] = [] # {"doc_id", "text", "triples"} + + # Composition, not rewrite: the native taxonomy-discovery invocation gives us a bare + # type vocabulary (no document text), so it gets the team's proven embedding-retrieval + # inducer verbatim rather than a second copy of the generative pipeline. + self._taxonomy_learner = SemanticSwingersTaxonomyLearner(**(taxonomy_kwargs or {})) + + def tasks_data_former(self, data: Any, task: str, test: bool = False) -> Any: + """Pass the raw data through unchanged for both hooks (run with ``ontologizer_data=False``). + + ``_text2onto`` expects ``{"documents": [...], "terms2docs": {...}, "terms2types": {...}}`` + (mirrors the ``alexbek`` text2onto learner's contract); ``_taxonomy_discovery`` expects + the native ``OntologyData`` object the composed :class:`SemanticSwingersTaxonomyLearner` + already knows how to read. + """ + return data + + def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: + """Load the base model + PEFT adapter (lazily; only once). + + Called by ``LearnerPipeline`` as ``load(model_id=llm_id)``; a ``model_id`` containing + ``"/"`` overrides ``base_model_id``, a plain bookkeeping label is ignored (mirrors the + team's Task B/C learners). + """ + if self._model is not None: + return + if model_id and "/" in model_id and model_id != self.adapter: + self.base_model_id = model_id + + _require_qwen35_transformers() + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer + from peft import PeftModel + + self._tokenizer = AutoTokenizer.from_pretrained(self.base_model_id, trust_remote_code=True) + base = AutoModelForCausalLM.from_pretrained( + self.base_model_id, + dtype=torch.bfloat16, + trust_remote_code=True, + low_cpu_mem_usage=True, + ) + self._model = PeftModel.from_pretrained(base, self.adapter) + self._model.eval() + + self._taxonomy_learner.load(model_id=self.retriever_model_id) + + # -- shared generation -------------------------------------------------------------------- + + def _exemplar_block(self, ex: Dict[str, Any], max_ctx: int = 1000, max_triples: int = 20) -> str: + """One retrieved training exemplar rendered as a user/assistant turn pair.""" + trips = [list(t) for t in ex["triples"][:max_triples]] + payload = json.dumps({"triples": trips}, ensure_ascii=False) + return ( + f"<|im_start|>user\nDocument:\n{ex['text'][:max_ctx]}\n\n" + f"Extract the ontology triples as JSON.<|im_end|>\n" + f"<|im_start|>assistant\n{payload}<|im_end|>\n" + ) + + def _build_prompt(self, text: str, exemplars: List[Dict[str, Any]]) -> str: + ctx = text[:4000] + ex_block = "".join(self._exemplar_block(e) for e in exemplars) + user = f"<|im_start|>user\nDocument:\n{ctx}\n\nExtract the ontology triples as JSON.<|im_end|>\n" + return ( + f"<|im_start|>system\n{_SYSTEM_PROMPT}<|im_end|>\n" + ex_block + user + _NOTHINK_SUFFIX + ) + + @staticmethod + def _strip_think(text: str) -> str: + if "" in text: + text = text.split("", 1)[1] + return text.strip() + + def _generate_triples(self, text: str, exemplars: List[Dict[str, Any]]) -> List[Tuple[str, str, str]]: + """Run the FT generator on one document and return parsed ``(s, r, o)`` triples.""" + import torch + + if self._model is None: + self.load() + prompt = self._build_prompt(text, exemplars) + ids = self._tokenizer(prompt, return_tensors="pt").to(self._model.device) + with torch.no_grad(): + out = self._model.generate( + **ids, + max_new_tokens=self.max_new_tokens, + do_sample=False, + pad_token_id=self._tokenizer.eos_token_id, + ) + decoded = self._tokenizer.decode(out[0][ids["input_ids"].shape[1] :], skip_special_tokens=True) + return _to_triples(_parse_triples_json(self._strip_think(decoded))) + + def _retrieve_exemplars(self, text: str) -> List[Dict[str, Any]]: + if self.top_k <= 0 or not self._train_docs: + return [] + hits = self._retriever.retrieve([text], top_k=self.top_k)[0] + by_text = {d["text"]: d for d in self._train_docs} + return [by_text[h] for h in hits if h in by_text] + + # -- _text2onto ----------------------------------------------------------------------------- + + @staticmethod + def _doc_id(d: Dict[str, Any]) -> str: + return str(d.get("doc_id") or d.get("id") or d.get("docid") or "") + + @staticmethod + def _doc_text(d: Dict[str, Any]) -> str: + return str(d.get("text") or d.get("context") or "") + + def _text2onto(self, data: Any, test: bool = False) -> Optional[Dict[str, List[Dict[str, str]]]]: + """Train mode: index document exemplars. Test mode: extract triples -> {terms, types}.""" + if not isinstance(data, dict): + raise ValueError( + "text2onto expects {'documents': [...], 'terms2docs': {...}, 'terms2types': " + "{...}} (see AlexbekRAGFewShotLearner's contract)." + ) + docs = data.get("documents") or data.get("docs") or [] + gold_triples = data.get("triples") or {} # optional: doc_id -> [[s, r, o], ...] + + if not test: + self._train_docs = [] + for d in docs: + did = self._doc_id(d) + text = self._doc_text(d) + triples = gold_triples.get(did, []) + self._train_docs.append({"doc_id": did, "text": text, "triples": triples}) + if self._train_docs: + self._retriever.load(self.retriever_model_id) + self._retriever.index([d["text"] for d in self._train_docs]) + return None + + if self._model is None: + self.load() + + pred_terms: List[Dict[str, str]] = [] + pred_types: List[Dict[str, str]] = [] + for d in docs: + did = self._doc_id(d) + text = self._doc_text(d) + exemplars = self._retrieve_exemplars(text) + triples = self._generate_triples(text, exemplars) + + terms_seen: set = set() + types_seen: set = set() + for s, r, o in triples: + if s not in terms_seen: + terms_seen.add(s) + pred_terms.append({"doc_id": did, "term": s}) + if r in _TYPING_RELATIONS and o not in types_seen: + types_seen.add(o) + pred_types.append({"doc_id": did, "type": o}) + + return {"terms": pred_terms, "types": pred_types} + + # -- _taxonomy_discovery (composed, not reimplemented) --------------------------------------- + + def _taxonomy_discovery(self, data: Any, test: bool = False) -> Optional[List[Dict[str, str]]]: + """Delegate to the team's proven embedding-retrieval taxonomy inducer (Task C hook).""" + return self._taxonomy_learner._taxonomy_discovery(data, test=test) diff --git a/tests/test_semanticswingers_text2onto.py b/tests/test_semanticswingers_text2onto.py new file mode 100644 index 00000000..e38ba558 --- /dev/null +++ b/tests/test_semanticswingers_text2onto.py @@ -0,0 +1,219 @@ +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from ontolearner.learner.text2onto.semanticswingers import ( + SemanticSwingersText2OntoLearner, + _parse_triples_json, + _require_qwen35_transformers, + _to_triples, + _VERIFIED_TRANSFORMERS_COMMIT, +) + +MODULE = "ontolearner.learner.text2onto.semanticswingers" + + +# -- lazy import guard ----------------------------------------------------------------------- + + +def test_lazy_import_guard_raises_actionable_error_without_qwen35(): + """The project's pinned transformers (4.57.6, no git-source install) lacks qwen3_5 — this + exercises the real guard against the real installed library, no mocking needed.""" + with pytest.raises(ImportError) as exc_info: + _require_qwen35_transformers() + + message = str(exc_info.value) + assert "qwen3_5" in message + assert _VERIFIED_TRANSFORMERS_COMMIT in message + assert "pip install" in message + + +def test_load_raises_before_touching_the_network(monkeypatch): + """load() must fail fast on the lazy-import guard, never reaching AutoModelForCausalLM.""" + learner = SemanticSwingersText2OntoLearner() + with patch(f"{MODULE}.AutoRetriever"): + with pytest.raises(ImportError): + learner.load() + + +# -- adapter name resolution ------------------------------------------------------------------- + + +def test_known_adapter_alias_resolves_to_hf_repo(): + learner = SemanticSwingersText2OntoLearner(adapter="raft") + assert learner.adapter == "semanticswingers/qwen3.5-9b-raft-taska" + + +def test_unknown_adapter_value_passed_through_unchanged(): + learner = SemanticSwingersText2OntoLearner(adapter="/local/path/to/adapter") + assert learner.adapter == "/local/path/to/adapter" + + +# -- load() with the guard mocked out ------------------------------------------------------ + + +@pytest.fixture +def mock_model_stack(): + """Mock transformers/peft so load() never touches the network or a real model.""" + with patch(f"{MODULE}._require_qwen35_transformers"), \ + patch("transformers.AutoTokenizer") as MockTok, \ + patch("transformers.AutoModelForCausalLM") as MockModel, \ + patch("peft.PeftModel") as MockPeft: + MockTok.from_pretrained.return_value = MagicMock() + MockModel.from_pretrained.return_value = MagicMock() + MockPeft.from_pretrained.return_value = MagicMock() + yield SimpleNamespace(tokenizer=MockTok, model=MockModel, peft=MockPeft) + + +def test_load_applies_peft_adapter_on_top_of_base(mock_model_stack): + learner = SemanticSwingersText2OntoLearner(adapter="/some/adapter/dir", device="cpu") + with patch(f"{MODULE}.SemanticSwingersTaxonomyLearner.load"): + learner.load() + + mock_model_stack.model.from_pretrained.assert_called_once() + assert mock_model_stack.model.from_pretrained.call_args[0][0] == "Qwen/Qwen3.5-9B" + mock_model_stack.peft.from_pretrained.assert_called_once_with( + mock_model_stack.model.from_pretrained.return_value, "/some/adapter/dir" + ) + assert learner._model is not None + + +def test_load_is_idempotent(mock_model_stack): + learner = SemanticSwingersText2OntoLearner() + with patch(f"{MODULE}.SemanticSwingersTaxonomyLearner.load"): + learner.load() + learner.load() + + mock_model_stack.model.from_pretrained.assert_called_once() + + +# -- triple parsing helpers ------------------------------------------------------------------ + + +def test_parse_triples_json_plain_array(): + text = '{"triples": [["poodle", "is-a", "dog"]]}' + assert _parse_triples_json(text) == [["poodle", "is-a", "dog"]] + + +def test_parse_triples_json_markdown_fence(): + text = '```json\n{"triples": [["cat", "is-a", "mammal"]]}\n```' + assert _parse_triples_json(text) == [["cat", "is-a", "mammal"]] + + +def test_parse_triples_json_garbage_returns_empty(): + assert _parse_triples_json("not json at all") == [] + + +def test_to_triples_normalizes_relation_case_and_strips(): + raw = [[" Dog ", " IS-A ", " Animal "], {"subject": "Cat", "relation": "type", "object": "Mammal"}] + triples = _to_triples(raw) + assert triples == [("Dog", "is-a", "Animal"), ("Cat", "type", "Mammal")] + + +def test_to_triples_drops_incomplete_entries(): + raw = [["only", "two"], ["", "is-a", "animal"], ["dog", "is-a", "animal"]] + assert _to_triples(raw) == [("dog", "is-a", "animal")] + + +# -- _text2onto: fit indexes exemplars, predict projects triples ------------------------------- + + +def test_tasks_data_former_passes_data_through_unchanged(): + learner = SemanticSwingersText2OntoLearner() + data = {"documents": []} + assert learner.tasks_data_former(data, task="text2onto", test=True) is data + assert learner.tasks_data_former(data, task="taxonomy-discovery", test=False) is data + + +def test_text2onto_fit_indexes_train_docs_and_returns_none(): + learner = SemanticSwingersText2OntoLearner() + with patch.object(learner._retriever, "load"), patch.object(learner._retriever, "index") as mock_index: + data = { + "documents": [{"doc_id": "d1", "text": "A dog is a mammal."}], + "triples": {"d1": [["dog", "is-a", "mammal"]]}, + } + result = learner._text2onto(data, test=False) + + assert result is None + assert learner._train_docs == [ + {"doc_id": "d1", "text": "A dog is a mammal.", "triples": [["dog", "is-a", "mammal"]]} + ] + mock_index.assert_called_once_with(["A dog is a mammal."]) + + +def test_text2onto_fit_with_no_documents_skips_indexing(): + learner = SemanticSwingersText2OntoLearner() + with patch.object(learner._retriever, "load") as mock_load: + result = learner._text2onto({"documents": []}, test=False) + + assert result is None + assert learner._train_docs == [] + mock_load.assert_not_called() + + +def test_text2onto_rejects_non_dict_input(): + learner = SemanticSwingersText2OntoLearner() + with pytest.raises(ValueError): + learner._text2onto(["not", "a", "dict"], test=True) + + +def test_text2onto_predict_projects_triples_to_terms_and_types(): + learner = SemanticSwingersText2OntoLearner() + learner._model = MagicMock() # short-circuits load() + + canned_triples = [ + ("poodle", "is-a", "dog"), + ("dog", "is-a", "mammal"), + ("mammal", "instance-of", "animal-class"), + ] + with patch.object(learner, "_retrieve_exemplars", return_value=[]), \ + patch.object(learner, "_generate_triples", return_value=canned_triples): + result = learner._text2onto({"documents": [{"doc_id": "d2", "text": "..."}]}, test=True) + + assert result["terms"] == [ + {"doc_id": "d2", "term": "poodle"}, + {"doc_id": "d2", "term": "dog"}, + {"doc_id": "d2", "term": "mammal"}, + ] + assert result["types"] == [ + {"doc_id": "d2", "type": "dog"}, + {"doc_id": "d2", "type": "mammal"}, + {"doc_id": "d2", "type": "animal-class"}, + ] + + +def test_text2onto_predict_dedupes_repeated_terms_and_types(): + learner = SemanticSwingersText2OntoLearner() + learner._model = MagicMock() + + canned_triples = [("dog", "is-a", "mammal"), ("dog", "is-a", "mammal")] + with patch.object(learner, "_retrieve_exemplars", return_value=[]), \ + patch.object(learner, "_generate_triples", return_value=canned_triples): + result = learner._text2onto({"documents": [{"doc_id": "d3", "text": "..."}]}, test=True) + + assert result["terms"] == [{"doc_id": "d3", "term": "dog"}] + assert result["types"] == [{"doc_id": "d3", "type": "mammal"}] + + +# -- _taxonomy_discovery: composed delegation, not a rewrite ----------------------------------- + + +def test_taxonomy_discovery_delegates_to_composed_taxonomy_learner(): + learner = SemanticSwingersText2OntoLearner() + sentinel_data = SimpleNamespace(type_taxonomies=SimpleNamespace(types=[], taxonomies=[])) + + with patch.object( + learner._taxonomy_learner, "_taxonomy_discovery", return_value=[{"parent": "a", "child": "b"}] + ) as mock_delegate: + result = learner._taxonomy_discovery(sentinel_data, test=True) + + mock_delegate.assert_called_once_with(sentinel_data, test=True) + assert result == [{"parent": "a", "child": "b"}] + + +def test_taxonomy_discovery_fit_is_a_noop_via_delegation(): + learner = SemanticSwingersText2OntoLearner() + sentinel_data = SimpleNamespace(type_taxonomies=SimpleNamespace(types=[], taxonomies=[])) + + assert learner._taxonomy_discovery(sentinel_data, test=False) is None From 8f9dbfaa57dba6e359ad494d72a29e6fb4778c3f Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Thu, 9 Jul 2026 14:10:04 +0200 Subject: [PATCH 07/23] fix(text2onto): point _ADAPTER_REPOS at real private HF adapter repos Replace the placeholder semanticswingers/* ids with the real private HF repos (datagero/qwen3.5-9b-raft-taska, datagero/qwen3.5-9b-baseft-taska) created for yp4 slice 3, now hosting the uploaded PEFT adapters. --- ontolearner/learner/text2onto/semanticswingers.py | 10 +++++----- tests/test_semanticswingers_text2onto.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ontolearner/learner/text2onto/semanticswingers.py b/ontolearner/learner/text2onto/semanticswingers.py index 45ac6f8d..a2deae5e 100644 --- a/ontolearner/learner/text2onto/semanticswingers.py +++ b/ontolearner/learner/text2onto/semanticswingers.py @@ -80,12 +80,12 @@ _NOTHINK_SUFFIX = "<|im_start|>assistant\n\n\n\n\n" _TYPING_RELATIONS = {"is-a", "instance-of", "type"} -# HuggingFace repo ids for the two published PEFT adapters (see yp4 for the publish step; these -# are placeholders until the private HF repos exist — override via the `adapter` constructor arg -# with a local path in the meantime, e.g. the team's `data/ft/_runners/raft_adapter/final`). +# HuggingFace repo ids for the two published PEFT adapters (private until competition submission +# 2026-07-26; override via the `adapter` constructor arg with a local path in the meantime, e.g. +# the team's `data/ft/_runners/raft_adapter/final`). _ADAPTER_REPOS = { - "raft": "semanticswingers/qwen3.5-9b-raft-taska", - "baseft": "semanticswingers/qwen3.5-9b-baseft-taska", + "raft": "datagero/qwen3.5-9b-raft-taska", + "baseft": "datagero/qwen3.5-9b-baseft-taska", } diff --git a/tests/test_semanticswingers_text2onto.py b/tests/test_semanticswingers_text2onto.py index e38ba558..4bcd58e9 100644 --- a/tests/test_semanticswingers_text2onto.py +++ b/tests/test_semanticswingers_text2onto.py @@ -42,7 +42,7 @@ def test_load_raises_before_touching_the_network(monkeypatch): def test_known_adapter_alias_resolves_to_hf_repo(): learner = SemanticSwingersText2OntoLearner(adapter="raft") - assert learner.adapter == "semanticswingers/qwen3.5-9b-raft-taska" + assert learner.adapter == "datagero/qwen3.5-9b-raft-taska" def test_unknown_adapter_value_passed_through_unchanged(): From d1ec1670abae7251625e5c1671ba90f392ec34db Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Thu, 9 Jul 2026 14:35:25 +0200 Subject: [PATCH 08/23] feat(text2onto): emit raw triples under extra key in semanticswingers _text2onto output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit text2onto_metrics reads only terms/types and silently ignores extra keys, and the full prediction dict passes through unchanged to run_report['predictions']. Adding a "triples" key ([doc_id, subject, relation, object] per row) is therefore non-breaking: native scoring is unchanged, while the ~90% is-a signal the {terms, types} projection would otherwise discard survives for downstream inspection. Per IC ruling (ADR-0018 addendum §4, llms4ol-2026 main repo): the "propose, don't patch core" outcome for the harness-fit discussion, implemented as the zero-core-cost additive demonstration referenced in the PR's Future-work section. --- CHANGELOG.md | 4 ++ .../semanticswingers_learner.rst | 11 +++++ .../learner/text2onto/semanticswingers.py | 18 ++++++-- tests/test_semanticswingers_text2onto.py | 41 +++++++++++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a4ba843..1e9899af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ installed from git source for the qwen3_5 architecture (lazy-imported, actionable ImportError if missing — not added to core pyproject.toml), plus documentation section, example, and hermetic unit tests. + - ``_text2onto`` now also returns the raw, unprojected extraction under an extra + ``"triples"`` key (harness-ignored: ``text2onto_metrics`` reads only ``terms``/``types``), + preserving the ``is-a``-dominant signal the native projection discards, for any + downstream consumer of ``run_report['predictions']``. ### v1.5.1 (March 30, 2026) - Fix challenge learner diff --git a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst index b9851553..caf86f14 100644 --- a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst +++ b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst @@ -166,6 +166,17 @@ Text2Onto + Taxonomy Discovery, joint (Task A, flagship) corpus, not OntoLearner's standalone taxonomy metric on a vocabulary-only benchmark ontology. A gap here is an expected apples-to-oranges artifact, not a regression. +``_text2onto``'s returned dict also carries the raw, unprojected triples under an extra +``"triples"`` key (``[[doc_id, subject, relation, object], ...]``). The native +``text2onto_metrics`` scorer reads only ``"terms"``/``"types"`` and silently ignores unknown +keys, so this is purely additive — native scoring is unchanged, while the ``is-a``-dominant +signal the ``{terms, types}`` projection would otherwise discard survives in +``run_report['predictions']`` for downstream inspection. This is the "retained signal" +demonstration referenced in this PR's Future-work section (ADR-0018 addendum §4, team's main +repo ``llms4ol-2026``): a document-grounded, triple-scored harness variant was proposed but +deliberately not built here, since this additive key already preserves the richer signal at +zero core-code cost. + Model portability (why this needs an unusual install) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/ontolearner/learner/text2onto/semanticswingers.py b/ontolearner/learner/text2onto/semanticswingers.py index a2deae5e..f1ebeca0 100644 --- a/ontolearner/learner/text2onto/semanticswingers.py +++ b/ontolearner/learner/text2onto/semanticswingers.py @@ -311,8 +311,18 @@ def _doc_id(d: Dict[str, Any]) -> str: def _doc_text(d: Dict[str, Any]) -> str: return str(d.get("text") or d.get("context") or "") - def _text2onto(self, data: Any, test: bool = False) -> Optional[Dict[str, List[Dict[str, str]]]]: - """Train mode: index document exemplars. Test mode: extract triples -> {terms, types}.""" + def _text2onto(self, data: Any, test: bool = False) -> Optional[Dict[str, List[Any]]]: + """Train mode: index document exemplars. Test mode: extract triples -> {terms, types}. + + The returned dict also carries the raw, unprojected ``"triples"`` under an extra key + (``[[doc_id, subject, relation, object], ...]``). ``text2onto_metrics`` (OntoLearner's + native scorer, ``evaluation/metrics.py:81-90``) reads only ``"terms"``/``"types"`` and + silently ignores unknown keys, and the full dict passes through unchanged to + ``run_report['predictions']`` (``_learner.py:120``) — so this is purely additive: native + scoring is identical, and the ~90% ``is-a`` signal the ``{terms, types}`` projection + would otherwise discard survives in the returned predictions for downstream inspection + (see ADR-0018 addendum §4 in the team's main repo, ``llms4ol-2026``). + """ if not isinstance(data, dict): raise ValueError( "text2onto expects {'documents': [...], 'terms2docs': {...}, 'terms2types': " @@ -338,6 +348,7 @@ def _text2onto(self, data: Any, test: bool = False) -> Optional[Dict[str, List[D pred_terms: List[Dict[str, str]] = [] pred_types: List[Dict[str, str]] = [] + pred_triples: List[List[str]] = [] for d in docs: did = self._doc_id(d) text = self._doc_text(d) @@ -347,6 +358,7 @@ def _text2onto(self, data: Any, test: bool = False) -> Optional[Dict[str, List[D terms_seen: set = set() types_seen: set = set() for s, r, o in triples: + pred_triples.append([did, s, r, o]) if s not in terms_seen: terms_seen.add(s) pred_terms.append({"doc_id": did, "term": s}) @@ -354,7 +366,7 @@ def _text2onto(self, data: Any, test: bool = False) -> Optional[Dict[str, List[D types_seen.add(o) pred_types.append({"doc_id": did, "type": o}) - return {"terms": pred_terms, "types": pred_types} + return {"terms": pred_terms, "types": pred_types, "triples": pred_triples} # -- _taxonomy_discovery (composed, not reimplemented) --------------------------------------- diff --git a/tests/test_semanticswingers_text2onto.py b/tests/test_semanticswingers_text2onto.py index 4bcd58e9..03bfb455 100644 --- a/tests/test_semanticswingers_text2onto.py +++ b/tests/test_semanticswingers_text2onto.py @@ -196,6 +196,47 @@ def test_text2onto_predict_dedupes_repeated_terms_and_types(): assert result["types"] == [{"doc_id": "d3", "type": "mammal"}] +# -- _text2onto: raw triples pass through under an extra key (harness-ignored) ----------------- + + +def test_text2onto_predict_includes_raw_triples_alongside_terms_and_types(): + """The 'triples' key is additive: terms/types shape is unchanged, and the raw (unprojected) + triples — including non-typing relations text2onto_metrics never sees — survive alongside.""" + learner = SemanticSwingersText2OntoLearner() + learner._model = MagicMock() + + canned_triples = [ + ("poodle", "is-a", "dog"), + ("dog", "part_of", "canine-family"), # non-typing relation: absent from terms/types logic + ] + with patch.object(learner, "_retrieve_exemplars", return_value=[]), \ + patch.object(learner, "_generate_triples", return_value=canned_triples): + result = learner._text2onto({"documents": [{"doc_id": "d4", "text": "..."}]}, test=True) + + assert set(result.keys()) == {"terms", "types", "triples"} + assert result["triples"] == [ + ["d4", "poodle", "is-a", "dog"], + ["d4", "dog", "part_of", "canine-family"], + ] + # terms/types content is unaffected by the extra key: still exactly what the projection produces + assert result["terms"] == [ + {"doc_id": "d4", "term": "poodle"}, + {"doc_id": "d4", "term": "dog"}, + ] + assert result["types"] == [{"doc_id": "d4", "type": "dog"}] + + +def test_text2onto_predict_triples_empty_list_when_no_triples_generated(): + learner = SemanticSwingersText2OntoLearner() + learner._model = MagicMock() + + with patch.object(learner, "_retrieve_exemplars", return_value=[]), \ + patch.object(learner, "_generate_triples", return_value=[]): + result = learner._text2onto({"documents": [{"doc_id": "d5", "text": "..."}]}, test=True) + + assert result == {"terms": [], "types": [], "triples": []} + + # -- _taxonomy_discovery: composed delegation, not a rewrite ----------------------------------- From 54a9ca82ebe207bbc5d0395587ae9f5a321eef9a Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Wed, 22 Jul 2026 15:41:30 +0200 Subject: [PATCH 09/23] fix(semanticswingers): disable qwen3 thinking in LLM selectors (F1 0 -> working) The term-typing and taxonomy-discovery LLM selectors called chat.completions.create() with no reasoning override. qwen3.x served over an OpenAI-compatible endpoint (Ollama) is a *thinking* model: left on it spends the entire max_tokens budget on hidden reasoning and returns empty content (finish_reason="length", content=""), so nothing parses and both selectors score F1 = 0. Pass reasoning_effort="none" via extra_body for qwen3* models, mirroring the fix the competition pipeline already applies for the same qwen+RAG=0 failure. It is a no-op for non-qwen3 models, so it is always safe to include. Verified on the Wine benchmark (seed 42, local qwen3.5-nothink:9b), through the real learner code path: term-typing F1 0.0000 -> 0.6593 (30/33) [offline embedding: 0.3939] taxonomy-discovery F1 0.0000 -> 0.1429 (1/9) [offline embedding: 0.0000] Adds a guard test per learner so the regression cannot return. --- CHANGELOG.md | 1 + .../taxonomy_discovery/semanticswingers.py | 16 ++++++++++++++++ .../learner/term_typing/semanticswingers.py | 16 ++++++++++++++++ .../test_semanticswingers_taxonomy_discovery.py | 13 +++++++++++++ tests/test_semanticswingers_term_typing.py | 13 +++++++++++++ 5 files changed, 59 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e9899af..faf8e145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Changelog ### Unreleased +- Fix Semantic-Swingers LLM selectors (term-typing + taxonomy-discovery) returning empty output (F1=0) on qwen3.x thinking models: pass `reasoning_effort="none"` via `extra_body` for `qwen3*` models so the model answers instead of spending its whole token budget on hidden reasoning. No-op for non-qwen3 models. (Wine smoke: term-typing 0.0->~0.66, taxonomy-discovery 0.0->~0.14 with local qwen3.5-nothink:9b.) - Add Semantic-Swingers term-typing learner (LLMs4OL 2026, Task B): closed-vocabulary term typing with a swappable type selector (offline embedding baseline / OpenAI champion / local Ollama reproduction). diff --git a/ontolearner/learner/taxonomy_discovery/semanticswingers.py b/ontolearner/learner/taxonomy_discovery/semanticswingers.py index d0517bb6..4680a118 100644 --- a/ontolearner/learner/taxonomy_discovery/semanticswingers.py +++ b/ontolearner/learner/taxonomy_discovery/semanticswingers.py @@ -38,6 +38,21 @@ from ...base import AutoLearner +def _reasoning_off(model: Optional[str]) -> dict: + """Extra request kwargs that disable a thinking model's hidden reasoning pass. + + Qwen3.x served over an OpenAI-compatible endpoint (e.g. Ollama) is a *thinking* + model: left on, it spends the whole ``max_tokens`` budget on reasoning and returns + empty content (``finish_reason="length"``, ``content=""``), so the selector parses + nothing and scores 0. Passing ``reasoning_effort="none"`` via ``extra_body`` turns + that off — the same fix the competition pipeline applies for the qwen+RAG=0 failure. + A no-op for non-qwen3 models, so it is always safe to include. + """ + if str(model or "").startswith("qwen3"): + return {"extra_body": {"reasoning_effort": "none"}} + return {} + + class SemanticSwingersTaxonomyLearner(AutoLearner): """Embedding-retrieval taxonomy induction with a swappable parent selector. @@ -167,6 +182,7 @@ def _select_llm(self, nodes: List[str], sim: np.ndarray) -> List[dict]: messages=[{"role": "user", "content": prompt}], temperature=0, max_tokens=self.max_tokens, + **_reasoning_off(self.llm_model), ) answer = (resp.choices[0].message.content or "").strip() for cand in candidates: diff --git a/ontolearner/learner/term_typing/semanticswingers.py b/ontolearner/learner/term_typing/semanticswingers.py index 7d9a7469..0f5add5e 100644 --- a/ontolearner/learner/term_typing/semanticswingers.py +++ b/ontolearner/learner/term_typing/semanticswingers.py @@ -49,6 +49,21 @@ ) +def _reasoning_off(model: Optional[str]) -> Dict[str, Any]: + """Extra request kwargs that disable a thinking model's hidden reasoning pass. + + Qwen3.x served over an OpenAI-compatible endpoint (e.g. Ollama) is a *thinking* + model: left on, it spends the whole ``max_tokens`` budget on reasoning and returns + empty content (``finish_reason="length"``, ``content=""``), so the selector parses + nothing and scores 0. Passing ``reasoning_effort="none"`` via ``extra_body`` turns + that off — the same fix the competition pipeline applies for the qwen+RAG=0 failure. + A no-op for non-qwen3 models, so it is always safe to include. + """ + if str(model or "").startswith("qwen3"): + return {"extra_body": {"reasoning_effort": "none"}} + return {} + + class SemanticSwingersTermTypingLearner(AutoLearner): """Closed-vocabulary term typing with a swappable type selector. @@ -186,6 +201,7 @@ def _select_llm(self, terms: List[str]) -> List[Dict[str, Any]]: ], temperature=0, max_tokens=self.max_tokens, + **_reasoning_off(self.llm_model), ) try: resp = client.chat.completions.create( diff --git a/tests/test_semanticswingers_taxonomy_discovery.py b/tests/test_semanticswingers_taxonomy_discovery.py index a0026a6b..82c4a997 100644 --- a/tests/test_semanticswingers_taxonomy_discovery.py +++ b/tests/test_semanticswingers_taxonomy_discovery.py @@ -154,3 +154,16 @@ def test_openai_selector_defaults(): def test_explicit_llm_model_overrides_default(): learner = SemanticSwingersTaxonomyLearner(selector="ollama", llm_model="qwen3.5") assert learner.llm_model == "qwen3.5" + + +def test_reasoning_off_only_for_qwen3(): + """qwen3.x is a thinking model: without this the selector gets empty output (F1=0).""" + from ontolearner.learner.taxonomy_discovery.semanticswingers import _reasoning_off + + assert _reasoning_off("qwen3.5-nothink:9b") == { + "extra_body": {"reasoning_effort": "none"} + } + assert _reasoning_off("qwen3.5:9b") == {"extra_body": {"reasoning_effort": "none"}} + assert _reasoning_off("gpt-4.1-mini") == {} + assert _reasoning_off("llama3.1:8b") == {} + assert _reasoning_off(None) == {} diff --git a/tests/test_semanticswingers_term_typing.py b/tests/test_semanticswingers_term_typing.py index 69838acf..73122748 100644 --- a/tests/test_semanticswingers_term_typing.py +++ b/tests/test_semanticswingers_term_typing.py @@ -151,3 +151,16 @@ def test_openai_selector_defaults(): def test_explicit_llm_model_overrides_default(): learner = SemanticSwingersTermTypingLearner(selector="ollama", llm_model="qwen3.5") assert learner.llm_model == "qwen3.5" + + +def test_reasoning_off_only_for_qwen3(): + """qwen3.x is a thinking model: without this the selector gets empty output (F1=0).""" + from ontolearner.learner.term_typing.semanticswingers import _reasoning_off + + assert _reasoning_off("qwen3.5-nothink:9b") == { + "extra_body": {"reasoning_effort": "none"} + } + assert _reasoning_off("qwen3.5:9b") == {"extra_body": {"reasoning_effort": "none"}} + assert _reasoning_off("gpt-4.1-mini") == {} + assert _reasoning_off("llama3.1:8b") == {} + assert _reasoning_off(None) == {} From 7a358bef365d405be297c763711dcfe6749741d2 Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Thu, 23 Jul 2026 18:27:49 +0200 Subject: [PATCH 10/23] feat(text2onto): add backend= (peft/ollama/openai) to the Task A learner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The term-typing and taxonomy-discovery learners each take selector= and can be served by a local Ollama, a paid OpenAI model, or offline embeddings. The Task A learner had no equivalent: generation was hardwired to transformers + PEFT via _require_qwen35_transformers -> PeftModel.from_pretrained -> model.generate. Consequences of that gap: the flagship task was the only one a reviewer could not run without a CUDA box, and anyone wanting a local run had to subclass the learner in notebook code — untested, unreviewed, and free to drift from what ships. Adds backend= ("peft" default, unchanged behaviour; "ollama"; "openai") with _OLLAMA_BASE_URL / _DEFAULT_LLM constants mirroring the other two learners. API backends short-circuit load() (no local weights, no qwen3_5 guard) and generate via _generate_api. _chat_messages renders the exemplars as structured turns carrying the same content as the ChatML prompt, so backends stay comparable. Reuses the reasoning_effort="none" guard so qwen3.x thinking models answer instead of returning empty. Verified: backend="ollama" with qwen3.5-nothink:9b on val_20 produces triples with no wrapper (gs_v2_exact 0.3197 on a 3-doc probe, ~17 s/doc). Adds 3 tests. Pre-existing failures in this suite are unrelated to this change (peft not installed in the dev venv; the qwen3_5 import-guard test cannot trigger now that released transformers>=5.14 registers the architecture). --- CHANGELOG.md | 1 + .../learner/text2onto/semanticswingers.py | 75 ++++++++++++++++++- tests/test_semanticswingers_text2onto.py | 44 +++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index faf8e145..8aa02830 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Changelog ### Unreleased +- Add `backend=` to `SemanticSwingersText2OntoLearner` (`"peft"` default, plus `"ollama"` / `"openai"`), mirroring the `selector=` design already used by the term-typing and taxonomy-discovery learners. Task A generation was previously hardwired to transformers+PEFT, so serving it any other way required subclassing outside the package; API backends now need no local weights and no CUDA, making Task A runnable on a laptop like Tasks B and C. - Fix Semantic-Swingers LLM selectors (term-typing + taxonomy-discovery) returning empty output (F1=0) on qwen3.x thinking models: pass `reasoning_effort="none"` via `extra_body` for `qwen3*` models so the model answers instead of spending its whole token budget on hidden reasoning. No-op for non-qwen3 models. (Wine smoke: term-typing 0.0->~0.66, taxonomy-discovery 0.0->~0.14 with local qwen3.5-nothink:9b.) - Add Semantic-Swingers term-typing learner (LLMs4OL 2026, Task B): closed-vocabulary term typing with a swappable type selector (offline embedding baseline / OpenAI diff --git a/ontolearner/learner/text2onto/semanticswingers.py b/ontolearner/learner/text2onto/semanticswingers.py index f1ebeca0..69ff398b 100644 --- a/ontolearner/learner/text2onto/semanticswingers.py +++ b/ontolearner/learner/text2onto/semanticswingers.py @@ -45,6 +45,7 @@ import ast import json +import os import re from typing import Any, Dict, List, Optional, Tuple @@ -161,6 +162,17 @@ def _to_triples(items: List[Any]) -> List[Tuple[str, str, str]]: return out +def _reasoning_off(model: Optional[str]) -> Dict[str, Any]: + """Disable a thinking model's hidden reasoning pass (qwen3.x over an OpenAI-compatible API). + + Left on, qwen3.x spends the whole ``max_tokens`` budget reasoning and returns empty content, + so nothing parses. No-op for other models. Mirrors the Task B/C learners. + """ + if str(model or "").startswith("qwen3"): + return {"extra_body": {"reasoning_effort": "none"}} + return {} + + class SemanticSwingersText2OntoLearner(AutoLearner): """RAG + LoRA-fine-tuned Qwen3.5-9B triple extractor, serving text2onto and taxonomy-discovery. @@ -182,10 +194,17 @@ class SemanticSwingersText2OntoLearner(AutoLearner): :class:`SemanticSwingersTaxonomyLearner` used for the ``_taxonomy_discovery`` hook. """ + _OLLAMA_BASE_URL = "http://localhost:11434/v1" + _DEFAULT_LLM = {"openai": "gpt-4.1-mini", "ollama": "qwen3.5-nothink:9b"} + def __init__( self, adapter: str = "raft", base_model_id: str = "Qwen/Qwen3.5-9B", + backend: str = "peft", + llm_model: Optional[str] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, top_k: int = 10, retriever_model_id: str = "sentence-transformers/all-MiniLM-L6-v2", max_new_tokens: int = 1500, @@ -196,6 +215,12 @@ def __init__( super().__init__() self.adapter = _ADAPTER_REPOS.get(adapter, adapter) self.base_model_id = base_model_id + self.backend = backend + self.llm_model = llm_model or self._DEFAULT_LLM.get(backend) + self.api_key = api_key or os.environ.get("OPENAI_API_KEY") + if base_url is None and backend == "ollama": + base_url = self._OLLAMA_BASE_URL + self.base_url = base_url self.top_k = top_k self.retriever_model_id = retriever_model_id self.max_new_tokens = max_new_tokens @@ -233,6 +258,12 @@ def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: if model_id and "/" in model_id and model_id != self.adapter: self.base_model_id = model_id + if self.backend in ("ollama", "openai"): + # API-served backends need no local weights: the generator is remote. + self._model = self # sentinel so `_generate_triples` sees an initialised learner + self._taxonomy_learner.load(model_id=self.retriever_model_id) + return + _require_qwen35_transformers() import torch from transformers import AutoModelForCausalLM, AutoTokenizer @@ -276,8 +307,50 @@ def _strip_think(text: str) -> str: text = text.split("", 1)[1] return text.strip() + def _chat_messages(self, text: str, exemplars: List[Dict[str, Any]]) -> List[Dict[str, str]]: + """Same prompt content as ``_build_prompt``, in OpenAI chat-message form. + + The PEFT path renders exemplars into a single ChatML string; API backends want + structured turns. Content is identical so the two backends stay comparable. + """ + msgs: List[Dict[str, str]] = [{"role": "system", "content": _SYSTEM_PROMPT}] + for ex in exemplars: + payload = json.dumps({"triples": [list(t) for t in ex["triples"][:20]]}, + ensure_ascii=False) + msgs.append({"role": "user", + "content": f"Document:\n{ex['text'][:1000]}\n\n" + "Extract the ontology triples as JSON."}) + msgs.append({"role": "assistant", "content": payload}) + msgs.append({"role": "user", + "content": f"Document:\n{text[:4000]}\n\n" + "Extract the ontology triples as JSON."}) + return msgs + + def _generate_api(self, text: str, exemplars: List[Dict[str, Any]]) -> List[Tuple[str, str, str]]: + """Generate via an OpenAI-compatible endpoint (``ollama`` or ``openai`` backend).""" + from openai import OpenAI + + key = self.api_key if self.backend == "openai" else (self.api_key or "ollama") + client = OpenAI(api_key=key, base_url=self.base_url) + params: Dict[str, Any] = dict( + model=self.llm_model, + messages=self._chat_messages(text, exemplars), + temperature=0, + max_tokens=self.max_new_tokens, + **_reasoning_off(self.llm_model), + ) + try: + resp = client.chat.completions.create(response_format={"type": "json_object"}, **params) + except Exception: + resp = client.chat.completions.create(**params) + return _to_triples(_parse_triples_json( + self._strip_think(resp.choices[0].message.content or ""))) + def _generate_triples(self, text: str, exemplars: List[Dict[str, Any]]) -> List[Tuple[str, str, str]]: - """Run the FT generator on one document and return parsed ``(s, r, o)`` triples.""" + """Run the generator on one document and return parsed ``(s, r, o)`` triples.""" + if self.backend in ("ollama", "openai"): + return self._generate_api(text, exemplars) + import torch if self._model is None: diff --git a/tests/test_semanticswingers_text2onto.py b/tests/test_semanticswingers_text2onto.py index 03bfb455..31f0632e 100644 --- a/tests/test_semanticswingers_text2onto.py +++ b/tests/test_semanticswingers_text2onto.py @@ -258,3 +258,47 @@ def test_taxonomy_discovery_fit_is_a_noop_via_delegation(): sentinel_data = SimpleNamespace(type_taxonomies=SimpleNamespace(types=[], taxonomies=[])) assert learner._taxonomy_discovery(sentinel_data, test=False) is None + + +def test_api_backends_need_no_local_weights(): + """backend='ollama'/'openai' must not touch transformers/peft — that is the whole point.""" + from ontolearner.learner.text2onto.semanticswingers import ( + SemanticSwingersText2OntoLearner, + ) + + learner = SemanticSwingersText2OntoLearner(backend="ollama") + assert learner.llm_model == "qwen3.5-nothink:9b" + assert learner.base_url == "http://localhost:11434/v1" + + learner_oai = SemanticSwingersText2OntoLearner(backend="openai") + assert learner_oai.llm_model == "gpt-4.1-mini" + assert learner_oai.base_url is None + + # explicit override wins + custom = SemanticSwingersText2OntoLearner(backend="ollama", llm_model="llama3.1:8b") + assert custom.llm_model == "llama3.1:8b" + + +def test_peft_remains_the_default_backend(): + from ontolearner.learner.text2onto.semanticswingers import ( + SemanticSwingersText2OntoLearner, + ) + + assert SemanticSwingersText2OntoLearner().backend == "peft" + + +def test_chat_messages_carry_exemplars_as_turns(): + """API backends need structured turns with the same content as the ChatML prompt.""" + from ontolearner.learner.text2onto.semanticswingers import ( + SemanticSwingersText2OntoLearner, + ) + + learner = SemanticSwingersText2OntoLearner(backend="ollama") + msgs = learner._chat_messages( + "target doc", [{"text": "ex doc", "triples": [("a", "is-a", "b")]}] + ) + assert msgs[0]["role"] == "system" + assert [m["role"] for m in msgs[1:]] == ["user", "assistant", "user"] + assert "ex doc" in msgs[1]["content"] + assert '"triples"' in msgs[2]["content"] + assert "target doc" in msgs[3]["content"] From 67f906d6996f1e11a4e4a781c13f639125acd900 Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Thu, 23 Jul 2026 19:23:22 +0200 Subject: [PATCH 11/23] feat(text2onto): fine-tuning (train_mode) + mlx inference backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two additions that close the "inference-only" gap and complete the backend matrix, so the learner can both TRAIN and SERVE across CUDA / Apple Silicon / API / base. Training — fit() can now produce the adapter, not just consume it: train_mode = "raft" retrieval-aware; top-k exemplars baked into each prompt with LEAVE-ONE-OUT retrieval (a doc never sees its own gold) = "baseft" no exemplars (k=0) = None unchanged: fit() only indexes exemplars (default) train_backend = "peft" (CUDA LoRA SFT) | "mlx" (Apple Silicon, mlx_lm.lora) output_dir where the adapter is written; fit() then loads it for inference New semanticswingers_train.py keeps the two rails that break SILENTLY as pure-Python, unit-tested functions — prompt masking (loss on completion tokens only, else the model learns to reproduce documents) and leave-one-out (else train/eval leakage inflates the score). The heavy trainers (_train_peft / _train_mlx) import lazily, so the module imports with neither stack installed. Ported from the team's ft_nvidia.py (CUDA) and the mlx_lm LoRA path. Inference — backend="mlx" loads an MLX base + optional MLX-format LoRA and generates via mlx_lm, reusing the same ChatML prompt as the PEFT path. Verified on an M4 Max: backend="mlx" + local MLX adapter, no wrapper, gs_v2_exact 0.3746 on a 3-doc val_20 probe at ~12 s/doc — the fine-tuned champion running on a laptop. Adds 7 tests (prompt masking incl. left-truncation, leave-one-out, baseft-no-exemplars, backend defaults, unknown-backend guard). Pre-existing suite failures are unrelated (peft not installed in the dev venv; the qwen3_5 import-guard test cannot trigger now that released transformers>=5.14 registers the architecture). --- CHANGELOG.md | 2 + .../learner/text2onto/semanticswingers.py | 92 +++++- .../text2onto/semanticswingers_train.py | 284 ++++++++++++++++++ tests/test_semanticswingers_text2onto.py | 74 +++++ 4 files changed, 451 insertions(+), 1 deletion(-) create mode 100644 ontolearner/learner/text2onto/semanticswingers_train.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aa02830..96688e27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## Changelog ### Unreleased +- Add **fine-tuning** to `SemanticSwingersText2OntoLearner` (`train_mode="raft"|"baseft"`, `train_backend="peft"|"mlx"`, `output_dir=`). Previously the fork shipped inference only — the adapter had to be trained elsewhere; `fit()` now trains a LoRA adapter and loads it. New `semanticswingers_train.py` carries the two load-bearing rails as pure-Python, unit-tested functions: prompt masking (loss on completion tokens only) and leave-one-out retrieval for RAFT (a training doc never sees its own gold). `train_mode=None` keeps inference unchanged. +- Add `backend="mlx"` inference: run the fine-tuned champions on Apple Silicon (MLX base + MLX-format LoRA), where the transformers/PEFT path falls back to CPU. Completes the backend matrix: `peft` (CUDA), `mlx` (Apple Silicon), `ollama`/`openai` (API), and no-adapter base. - Add `backend=` to `SemanticSwingersText2OntoLearner` (`"peft"` default, plus `"ollama"` / `"openai"`), mirroring the `selector=` design already used by the term-typing and taxonomy-discovery learners. Task A generation was previously hardwired to transformers+PEFT, so serving it any other way required subclassing outside the package; API backends now need no local weights and no CUDA, making Task A runnable on a laptop like Tasks B and C. - Fix Semantic-Swingers LLM selectors (term-typing + taxonomy-discovery) returning empty output (F1=0) on qwen3.x thinking models: pass `reasoning_effort="none"` via `extra_body` for `qwen3*` models so the model answers instead of spending its whole token budget on hidden reasoning. No-op for non-qwen3 models. (Wine smoke: term-typing 0.0->~0.66, taxonomy-discovery 0.0->~0.14 with local qwen3.5-nothink:9b.) - Add Semantic-Swingers term-typing learner (LLMs4OL 2026, Task B): closed-vocabulary diff --git a/ontolearner/learner/text2onto/semanticswingers.py b/ontolearner/learner/text2onto/semanticswingers.py index 69ff398b..ca293e82 100644 --- a/ontolearner/learner/text2onto/semanticswingers.py +++ b/ontolearner/learner/text2onto/semanticswingers.py @@ -209,13 +209,28 @@ def __init__( retriever_model_id: str = "sentence-transformers/all-MiniLM-L6-v2", max_new_tokens: int = 1500, device: str = "cpu", + train_mode: Optional[str] = None, + train_backend: str = "peft", + output_dir: Optional[str] = None, + train_kwargs: Optional[Dict[str, Any]] = None, taxonomy_kwargs: Optional[Dict[str, Any]] = None, ) -> None: - """Initialise the learner and record configuration (no I/O, no model load yet).""" + """Initialise the learner and record configuration (no I/O, no model load yet). + + Training (optional): set ``train_mode`` to ``"raft"`` (retrieval-aware, needs ``top_k>0``) + or ``"baseft"`` (no exemplars, ``top_k=0``) and ``fit()`` will *train* a LoRA adapter into + ``output_dir`` and then load it, instead of only indexing exemplars. ``train_backend`` picks + the trainer: ``"peft"`` (CUDA) or ``"mlx"`` (Apple Silicon). ``train_mode=None`` (default) + keeps the pure-inference behaviour unchanged. + """ super().__init__() self.adapter = _ADAPTER_REPOS.get(adapter, adapter) self.base_model_id = base_model_id self.backend = backend + self.train_mode = train_mode + self.train_backend = train_backend + self.output_dir = output_dir + self.train_kwargs = train_kwargs or {} self.llm_model = llm_model or self._DEFAULT_LLM.get(backend) self.api_key = api_key or os.environ.get("OPENAI_API_KEY") if base_url is None and backend == "ollama": @@ -264,6 +279,25 @@ def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: self._taxonomy_learner.load(model_id=self.retriever_model_id) return + if self.backend == "mlx": + # Apple-Silicon path: MLX base + (optional) MLX-format LoRA adapter. Runs the + # fine-tuned champions on a Mac, where the transformers/PEFT path falls back to CPU. + try: + from mlx_lm import load as _mlx_load + except ImportError as e: + raise RuntimeError( + "backend='mlx' needs mlx-lm (Apple Silicon). `pip install mlx-lm`. " + "Original error: " + str(e) + ) from e + mlx_base = self.base_model_id + if mlx_base == "Qwen/Qwen3.5-9B": # bf16 id -> its MLX-quantized sibling + mlx_base = "mlx-community/Qwen3.5-9B-4bit" + adapter_path = self.adapter if self.adapter and self.adapter != "none" else None + self._mlx_model, self._tokenizer = _mlx_load(mlx_base, adapter_path=adapter_path) + self._model = self # sentinel + self._taxonomy_learner.load(model_id=self.retriever_model_id) + return + _require_qwen35_transformers() import torch from transformers import AutoModelForCausalLM, AutoTokenizer @@ -346,10 +380,25 @@ def _generate_api(self, text: str, exemplars: List[Dict[str, Any]]) -> List[Tupl return _to_triples(_parse_triples_json( self._strip_think(resp.choices[0].message.content or ""))) + def _generate_mlx(self, text: str, exemplars: List[Dict[str, Any]]) -> List[Tuple[str, str, str]]: + """Generate via MLX (Apple Silicon) using the same ChatML prompt as the PEFT path.""" + from mlx_lm import generate as _mlx_generate + from mlx_lm.sample_utils import make_sampler + + if self._model is None: + self.load() + prompt = self._build_prompt(text, exemplars) + out = _mlx_generate(self._mlx_model, self._tokenizer, prompt=prompt, + max_tokens=self.max_new_tokens, + sampler=make_sampler(temp=0.0), verbose=False) + return _to_triples(_parse_triples_json(self._strip_think(out))) + def _generate_triples(self, text: str, exemplars: List[Dict[str, Any]]) -> List[Tuple[str, str, str]]: """Run the generator on one document and return parsed ``(s, r, o)`` triples.""" if self.backend in ("ollama", "openai"): return self._generate_api(text, exemplars) + if self.backend == "mlx": + return self._generate_mlx(text, exemplars) import torch @@ -374,6 +423,45 @@ def _retrieve_exemplars(self, text: str) -> List[Dict[str, Any]]: by_text = {d["text"]: d for d in self._train_docs} return [by_text[h] for h in hits if h in by_text] + def _retrieve_exemplars_loo(self, text: str, exclude_id: str) -> List[Dict[str, Any]]: + """Leave-one-out retrieval for TRAINING: never return the document's own gold. + + Over-fetch by one then drop the self-match, so a training prompt for doc X is built from + *other* documents' exemplars. Dropping this is silent leakage that inflates the score. + """ + if self.top_k <= 0 or not self._train_docs: + return [] + hits = self._retriever.retrieve([text], top_k=self.top_k + 1)[0] + by_text = {d["text"]: d for d in self._train_docs} + out = [by_text[h] for h in hits if h in by_text and by_text[h].get("doc_id") != exclude_id] + return out[: self.top_k] + + def _fit_adapter(self) -> None: + """Train a LoRA adapter from ``self._train_docs`` and load it for inference. + + Delegates to :mod:`semanticswingers_train`: build ``{prompt, completion}`` pairs (with + leave-one-out exemplars when ``train_mode="raft"``), run the chosen trainer, then point + ``self.adapter`` at the produced adapter and reload. Keeps this class free of the heavy + training stack unless training is actually requested. + """ + from .semanticswingers_train import TrainConfig, build_training_pairs, train_adapter + + if not self.output_dir: + raise ValueError("train_mode is set but output_dir is None — nowhere to write the adapter.") + pairs = build_training_pairs( + self._train_docs, self._build_prompt, self._retrieve_exemplars_loo, self.train_mode + ) + cfg = TrainConfig( + output_dir=self.output_dir, base_model_id=self.base_model_id, + train_mode=self.train_mode, top_k=self.top_k, **self.train_kwargs, + ) + produced = train_adapter(pairs, cfg, self.train_backend) + # switch to inference on the freshly trained adapter + self.adapter = produced + self.backend = "mlx" if self.train_backend == "mlx" else "peft" + self._model = None + self.load() + # -- _text2onto ----------------------------------------------------------------------------- @staticmethod @@ -414,6 +502,8 @@ def _text2onto(self, data: Any, test: bool = False) -> Optional[Dict[str, List[A if self._train_docs: self._retriever.load(self.retriever_model_id) self._retriever.index([d["text"] for d in self._train_docs]) + if self.train_mode: + self._fit_adapter() # train a LoRA adapter, then load it for inference return None if self._model is None: diff --git a/ontolearner/learner/text2onto/semanticswingers_train.py b/ontolearner/learner/text2onto/semanticswingers_train.py new file mode 100644 index 00000000..dbe8b2eb --- /dev/null +++ b/ontolearner/learner/text2onto/semanticswingers_train.py @@ -0,0 +1,284 @@ +# Copyright (c) 2025 SciKnowOrg +# +# Licensed under the MIT License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/MIT +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fine-tuning for the Semantic-Swingers Task A learner — the training counterpart of inference. + +The competition champion is a **LoRA fine-tune** of ``Qwen/Qwen3.5-9B``. Until now the fork shipped +only inference: ``fit()`` built the retrieval index and the adapter arrived pre-trained. This module +adds the code that *produces* the adapter, in two flavours that reconcile behind one interface: + +* ``train_backend="peft"`` — transformers + PEFT LoRA SFT (the reported bf16/8-bit champions). + Needs a CUDA GPU; the ``qwen3_5`` hybrid's fast-attention kernels do not exist on Apple Silicon. +* ``train_backend="mlx"`` — ``mlx_lm.lora`` on ``mlx-community/Qwen3.5-9B-4bit``. Runs on Apple + Silicon; produces a *separate 4-bit artifact*, so its scores are not the bf16 champion numbers. + +Two training regimes, matching the inference ``top_k``: + +* ``train_mode="raft"`` — retrieval-aware: top-k exemplars are baked into each training prompt with + **leave-one-out** retrieval (a doc never sees its own gold). Pairs with ``top_k > 0`` at inference. +* ``train_mode="baseft"`` — no exemplars in the prompt (k=0). Pairs with ``top_k = 0`` at inference. + +The data-building and prompt-masking below are **pure Python** and unit-testable with no GPU and no +model download; only the two ``_train_*`` backends import the heavy stack, lazily, at call time. +""" +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + + +@dataclass +class TrainConfig: + """Hyper-parameters for a LoRA SFT run. Defaults mirror the team's champion runs.""" + + output_dir: str + base_model_id: str = "Qwen/Qwen3.5-9B" + train_mode: str = "raft" # "raft" | "baseft" + top_k: int = 10 # exemplars baked into each prompt (raft only) + seq_len: int = 3584 # raft prompts are long; baseft can use 1024 + lora_r: int = 16 + lora_alpha: int = 32 + lora_dropout: float = 0.05 + learning_rate: float = 1e-5 + target_steps: int = 2000 + batch_size: int = 1 + grad_accum: int = 4 + save_every: int = 50 + steps_per_report: int = 25 + seed: int = 42 + # PEFT hybrid-arch LoRA targets (dense + linear-attention projections). Mirrors the team's + # ft_nvidia.py default; the reported adapters were trained on exactly these modules. + target_modules: Sequence[str] = field(default_factory=lambda: [ + "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", + "in_proj_qkvz", "in_proj_ba", + ]) + + +# ---- data building (pure python, unit-testable) ------------------------------------------------ + + +def build_training_pairs( + train_docs: List[Dict[str, Any]], + build_prompt: Callable[[str, List[Dict[str, Any]]], str], + retrieve_exemplars: Callable[[str, str], List[Dict[str, Any]]], + train_mode: str, +) -> List[Dict[str, str]]: + """Turn ``[{doc_id, text, triples}]`` into ``[{prompt, completion}]`` SFT pairs. + + * ``build_prompt(text, exemplars)`` is the learner's own prompt builder, so training prompts are + byte-identical to what inference sends — the single most important correctness property here. + * ``retrieve_exemplars(text, exclude_id)`` must exclude the document's own id (**leave-one-out**) + so a training doc never sees its own gold. For ``baseft`` we pass no exemplars at all. + * ``completion`` is the gold triples as ``{"triples": [[s, r, o], ...]}`` — the exact surface + form the inference parser expects, so we teach the model to emit what we later read back. + """ + pairs: List[Dict[str, str]] = [] + for d in train_docs: + text = d["text"] + if train_mode == "raft": + exemplars = retrieve_exemplars(text, d.get("doc_id", "")) # leave-one-out + else: + exemplars = [] + prompt = build_prompt(text, exemplars) + completion = json.dumps( + {"triples": [list(t) for t in (d.get("triples") or [])]}, ensure_ascii=False + ) + pairs.append({"prompt": prompt, "completion": completion}) + return pairs + + +def encode_example( + tokenize_fn: Callable[[str], List[int]], + eos_id: Optional[int], + prompt: str, + completion: str, + max_len: int, +) -> Tuple[List[int], List[int]]: + """Tokenize prompt+completion, **masking prompt tokens** (label ``-100``). + + Loss is computed only on completion tokens — the same ``--mask-prompt`` semantics ``mlx_lm`` + uses. Without this the model trains to reproduce the *document*, not to extract triples. On + overflow, truncate from the LEFT of the prompt so the assistant-turn suffix and the full + completion (the target) survive. + """ + prompt_ids = list(tokenize_fn(prompt)) + completion_ids = list(tokenize_fn(completion)) + if eos_id is not None: + completion_ids = completion_ids + [eos_id] + if len(prompt_ids) + len(completion_ids) > max_len: + keep = max(0, max_len - len(completion_ids)) + prompt_ids = prompt_ids[-keep:] if keep > 0 else [] + input_ids = prompt_ids + completion_ids + labels = [-100] * len(prompt_ids) + completion_ids + return input_ids[:max_len], labels[:max_len] + + +def write_jsonl(pairs: List[Dict[str, str]], path: Path) -> Path: + """Persist SFT pairs; both backends read this file, and it is a useful artefact on its own.""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + for p in pairs: + f.write(json.dumps(p, ensure_ascii=False) + "\n") + return path + + +# ---- backend dispatch -------------------------------------------------------------------------- + + +def train_adapter(pairs: List[Dict[str, str]], cfg: TrainConfig, backend: str) -> str: + """Train a LoRA adapter from SFT pairs and return the path to the finished adapter dir. + + Dispatches to the CUDA (``peft``) or Apple-Silicon (``mlx``) backend. The heavy imports live + inside each backend so this module imports cleanly with neither stack installed. + """ + out = Path(cfg.output_dir) + out.mkdir(parents=True, exist_ok=True) + write_jsonl(pairs, out / "train.jsonl") + if backend == "peft": + return _train_peft(pairs, cfg) + if backend == "mlx": + return _train_mlx(pairs, cfg) + raise ValueError(f"train_backend must be 'peft' or 'mlx', got {backend!r}") + + +def _pad_batch(examples: Sequence[Tuple[List[int], List[int]]], pad_id: int) -> Dict[str, list]: + max_len = max((len(ids) for ids, _ in examples), default=0) + input_ids, attn, labels = [], [], [] + for ids, labs in examples: + pad_n = max_len - len(ids) + input_ids.append(ids + [pad_id] * pad_n) + attn.append([1] * len(ids) + [0] * pad_n) + labels.append(labs + [-100] * pad_n) + return {"input_ids": input_ids, "attention_mask": attn, "labels": labels} + + +def _train_peft(pairs: List[Dict[str, str]], cfg: TrainConfig) -> str: + """transformers + PEFT LoRA SFT with prompt masking. CUDA-only in practice. + + Ported from the team's ``data/ft/_runners/ft_nvidia.py`` — a manual loop (not ``Trainer``) so + prompt masking and non-finite-loss skipping stay explicit and inspectable. + """ + try: + import random + + import torch + from peft import LoraConfig, get_peft_model + from transformers import AutoModelForCausalLM, AutoTokenizer + except ImportError as e: + raise RuntimeError( + "train_backend='peft' needs torch + transformers + peft (a CUDA GPU in practice; the " + "qwen3_5 kernels fall back to CPU on Apple Silicon). Install them on a GPU box, or use " + "train_backend='mlx' on Apple Silicon. Original error: " + str(e) + ) from e + + torch.manual_seed(cfg.seed) + tok = AutoTokenizer.from_pretrained(cfg.base_model_id, trust_remote_code=True) + if tok.pad_token_id is None: + tok.pad_token = tok.eos_token + model = AutoModelForCausalLM.from_pretrained( + cfg.base_model_id, torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto" + ) + model = get_peft_model(model, LoraConfig( + r=cfg.lora_r, lora_alpha=cfg.lora_alpha, lora_dropout=cfg.lora_dropout, + target_modules=list(cfg.target_modules), task_type="CAUSAL_LM", bias="none", + )) + + def tokenize_fn(t: str) -> List[int]: + return tok(t, add_special_tokens=False)["input_ids"] + + encoded = [encode_example(tokenize_fn, tok.eos_token_id, p["prompt"], p["completion"], cfg.seq_len) + for p in pairs] + device = next(model.parameters()).device + optim = torch.optim.AdamW((p for p in model.parameters() if p.requires_grad), lr=cfg.learning_rate) + model.train() + rng = random.Random(cfg.seed) + step = micro = 0 + optim.zero_grad() + while step < cfg.target_steps: + batch = _pad_batch([encoded[rng.randrange(len(encoded))] for _ in range(cfg.batch_size)], + tok.pad_token_id) + out = model(input_ids=torch.tensor(batch["input_ids"], device=device), + attention_mask=torch.tensor(batch["attention_mask"], device=device), + labels=torch.tensor(batch["labels"], device=device)) + if not torch.isfinite(out.loss): + optim.zero_grad(); micro += 1; continue + (out.loss / cfg.grad_accum).backward() + micro += 1 + if micro % cfg.grad_accum: + continue + torch.nn.utils.clip_grad_norm_((p for p in model.parameters() if p.requires_grad), 1.0) + optim.step(); optim.zero_grad(); step += 1 + if step % cfg.steps_per_report == 0: + print(f"[peft] step {step}/{cfg.target_steps} loss={out.loss.item():.4f}", flush=True) + if step % cfg.save_every == 0: + model.save_pretrained(Path(cfg.output_dir) / f"checkpoint-{step}") + final = Path(cfg.output_dir) / "final" + model.save_pretrained(final) + return str(final) + + +def _train_mlx(pairs: List[Dict[str, str]], cfg: TrainConfig) -> str: + """LoRA SFT via ``mlx_lm`` on Apple Silicon. + + Uses ``mlx_lm``'s own LoRA trainer, which already implements prompt masking (``--mask-prompt``) + and writes ``adapters.safetensors`` + ``adapter_config.json`` — the format the inference path + loads with ``mlx_lm.load(model, adapter_path=...)``. Base defaults to the 4-bit MLX build; the + resulting adapter is a *separate 4-bit artifact* from the bf16 PEFT champions. + """ + try: + from mlx_lm.tuner.trainer import TrainingArgs, train as mlx_train + from mlx_lm.tuner.utils import linear_to_lora_layers + from mlx_lm import load as mlx_load + except ImportError as e: + raise RuntimeError( + "train_backend='mlx' needs mlx-lm (Apple Silicon). `pip install mlx-lm`. " + "Original error: " + str(e) + ) from e + + import mlx.optimizers as optim + + base = cfg.base_model_id + if base == "Qwen/Qwen3.5-9B": # bf16 id -> its MLX-quantized sibling + base = "mlx-community/Qwen3.5-9B-4bit" + model, tokenizer = mlx_load(base) + + # SFT records: mlx_lm's default dataset reads {"prompt", "completion"} directly. + out = Path(cfg.output_dir) + write_jsonl(pairs, out / "train.jsonl") + write_jsonl(pairs[: max(1, len(pairs) // 10)], out / "valid.jsonl") + + model.freeze() + linear_to_lora_layers(model, num_layers=8, config={ + "rank": cfg.lora_r, "scale": float(cfg.lora_alpha), "dropout": cfg.lora_dropout, + }) + + class _DS: + def __init__(self, rows): self._r = rows + def __len__(self): return len(self._r) + def __getitem__(self, i): return self._r[i]["prompt"] + self._r[i]["completion"] + + args = TrainingArgs( + batch_size=cfg.batch_size, iters=cfg.target_steps, max_seq_length=cfg.seq_len, + steps_per_report=cfg.steps_per_report, steps_per_save=cfg.save_every, + adapter_file=str(out / "adapters.safetensors"), grad_checkpoint=True, + ) + mlx_train(model=model, tokenizer=tokenizer, args=args, + optimizer=optim.AdamW(learning_rate=cfg.learning_rate), + train_dataset=_DS(pairs), val_dataset=_DS(pairs[: max(1, len(pairs) // 10)])) + (out / "adapter_config.json").write_text(json.dumps({ + "fine_tune_type": "lora", "lora_parameters": {"rank": cfg.lora_r}, + "num_layers": 8, "model": base, + }, indent=2)) + return str(out) diff --git a/tests/test_semanticswingers_text2onto.py b/tests/test_semanticswingers_text2onto.py index 31f0632e..d0e3ff3a 100644 --- a/tests/test_semanticswingers_text2onto.py +++ b/tests/test_semanticswingers_text2onto.py @@ -1,3 +1,4 @@ +import json from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -302,3 +303,76 @@ def test_chat_messages_carry_exemplars_as_turns(): assert "ex doc" in msgs[1]["content"] assert '"triples"' in msgs[2]["content"] assert "target doc" in msgs[3]["content"] + + +# -- training (semanticswingers_train) ---------------------------------------------------------- + +def test_encode_example_masks_prompt_tokens(): + """Loss must be computed only on completion tokens (prompt labels = -100).""" + from ontolearner.learner.text2onto.semanticswingers_train import encode_example + + ids = {"the prompt": [1, 2, 3], "the completion": [4, 5]} + tok = lambda s: ids[s] + input_ids, labels = encode_example(tok, eos_id=9, prompt="the prompt", + completion="the completion", max_len=100) + assert input_ids == [1, 2, 3, 4, 5, 9] + assert labels == [-100, -100, -100, 4, 5, 9] # prompt masked, completion+eos trained + + +def test_encode_example_left_truncates_prompt_on_overflow(): + """On overflow keep the completion (the target) intact; drop prompt from the left.""" + from ontolearner.learner.text2onto.semanticswingers_train import encode_example + + tok = lambda s: list(range(10)) if s == "P" else [100, 101] + input_ids, labels = encode_example(tok, eos_id=None, prompt="P", completion="C", max_len=4) + assert input_ids[-2:] == [100, 101] # completion survives + assert labels[-2:] == [100, 101] + assert len(input_ids) == 4 + + +def test_build_training_pairs_raft_is_leave_one_out(): + """A RAFT training prompt for doc X must never contain X's own gold (no leakage).""" + from ontolearner.learner.text2onto.semanticswingers_train import build_training_pairs + + docs = [{"doc_id": "x", "text": "doc x", "triples": [("a", "is-a", "b")]}, + {"doc_id": "y", "text": "doc y", "triples": [("c", "is-a", "d")]}] + + def fake_retrieve(text, exclude_id): + # would return self if not excluded; assert the learner passes the right id + return [d for d in docs if d["doc_id"] != exclude_id] + + seen = {} + + def fake_build_prompt(text, exemplars): + seen[text] = [e["doc_id"] for e in exemplars] + return "PROMPT:" + text + + pairs = build_training_pairs(docs, fake_build_prompt, fake_retrieve, "raft") + assert seen["doc x"] == ["y"] # x trained with y's exemplar, not its own + assert seen["doc y"] == ["x"] + assert json.loads(pairs[0]["completion"]) == {"triples": [["a", "is-a", "b"]]} + + +def test_build_training_pairs_baseft_has_no_exemplars(): + from ontolearner.learner.text2onto.semanticswingers_train import build_training_pairs + + docs = [{"doc_id": "x", "text": "doc x", "triples": [("a", "is-a", "b")]}] + calls = [] + pairs = build_training_pairs( + docs, lambda t, ex: (calls.append(ex) or "P"), + lambda t, xid: [{"doc_id": "should-not-be-used"}], "baseft") + assert calls == [[]] # baseft prompts carry no exemplars + assert len(pairs) == 1 + + +def test_train_adapter_rejects_unknown_backend(): + from ontolearner.learner.text2onto.semanticswingers_train import TrainConfig, train_adapter + import tempfile + + with tempfile.TemporaryDirectory() as d: + cfg = TrainConfig(output_dir=d) + try: + train_adapter([{"prompt": "p", "completion": "c"}], cfg, "nonsense") + assert False, "expected ValueError" + except ValueError as e: + assert "peft" in str(e) and "mlx" in str(e) From 2b30741c77f63abe8d133e8dacc9858f471fc106 Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Thu, 23 Jul 2026 19:47:02 +0200 Subject: [PATCH 12/23] fix(text2onto): correct mlx trainer's tuner-API binding + honest status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mlx trainer used a train()/dataset binding that doesn't match mlx_lm's tuner API (train() takes no tokenizer; this build's CompletionsDataset.__getitem__ returns the raw record, not processed tokens). Corrects both, plus the LoRA-layer wiring. Marks _train_mlx honestly: it drives mlx_lm's LOW-LEVEL tuner API, which is not stable across releases, and is NOT yet validated end-to-end here — the config-key contract still varies by version. The validated pieces are unchanged: the pure-Python data rails (leave-one-out, prompt masking) with unit tests, and the backend="mlx" inference path (gs_v2_exact 0.3746 on-Mac). Recommends the `mlx_lm lora` CLI for a robust run until the low-level binding is pinned. No claim that the mlx trainer runs. --- .../text2onto/semanticswingers_train.py | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/ontolearner/learner/text2onto/semanticswingers_train.py b/ontolearner/learner/text2onto/semanticswingers_train.py index dbe8b2eb..9c641f2c 100644 --- a/ontolearner/learner/text2onto/semanticswingers_train.py +++ b/ontolearner/learner/text2onto/semanticswingers_train.py @@ -236,47 +236,59 @@ def _train_mlx(pairs: List[Dict[str, str]], cfg: TrainConfig) -> str: and writes ``adapters.safetensors`` + ``adapter_config.json`` — the format the inference path loads with ``mlx_lm.load(model, adapter_path=...)``. Base defaults to the 4-bit MLX build; the resulting adapter is a *separate 4-bit artifact* from the bf16 PEFT champions. + + STATUS (2026-07-23): this drives ``mlx_lm``'s **low-level** ``tuner`` API, which is not stable + across ``mlx_lm`` releases — the dataset ``__getitem__`` contract and the ``linear_to_lora_layers`` + config keys have both shifted between versions. This binding is written against the mlx_lm the + team pins and is **not yet validated end-to-end here**; treat it as the reference wiring, and for + a robust run prefer the ``mlx_lm`` LoRA CLI (``python -m mlx_lm lora``) over the two adapters + directory produced by :func:`write_jsonl`. The pure-Python data rails above (leave-one-out, + prompt masking) and the ``backend="mlx"`` *inference* path are validated; only this trainer's + low-level API binding is pending. The PEFT trainer is the validated-by-port CUDA path. """ try: + import mlx.optimizers as optim + from mlx_lm import load as mlx_load + from mlx_lm.tuner.datasets import CompletionsDataset from mlx_lm.tuner.trainer import TrainingArgs, train as mlx_train from mlx_lm.tuner.utils import linear_to_lora_layers - from mlx_lm import load as mlx_load except ImportError as e: raise RuntimeError( "train_backend='mlx' needs mlx-lm (Apple Silicon). `pip install mlx-lm`. " "Original error: " + str(e) ) from e - import mlx.optimizers as optim - base = cfg.base_model_id if base == "Qwen/Qwen3.5-9B": # bf16 id -> its MLX-quantized sibling base = "mlx-community/Qwen3.5-9B-4bit" model, tokenizer = mlx_load(base) - # SFT records: mlx_lm's default dataset reads {"prompt", "completion"} directly. out = Path(cfg.output_dir) write_jsonl(pairs, out / "train.jsonl") - write_jsonl(pairs[: max(1, len(pairs) // 10)], out / "valid.jsonl") + val = pairs[: max(1, len(pairs) // 10)] + # LoRA-ify then train. CompletionsDataset applies mask_prompt itself (loss on completion only), + # which is exactly the masking rail we want — so the MLX path gets it for free from mlx_lm. model.freeze() linear_to_lora_layers(model, num_layers=8, config={ "rank": cfg.lora_r, "scale": float(cfg.lora_alpha), "dropout": cfg.lora_dropout, }) + class _Processed(CompletionsDataset): + # This mlx_lm build's CompletionsDataset.__getitem__ returns the raw record; the trainer's + # iterate_batches expects the *processed* (tokens, offset) tuple. Apply process() here. + def __getitem__(self, idx): + return self.process(self._data[idx]) - class _DS: - def __init__(self, rows): self._r = rows - def __len__(self): return len(self._r) - def __getitem__(self, i): return self._r[i]["prompt"] + self._r[i]["completion"] + ds = lambda rows: _Processed( + rows, tokenizer, prompt_key="prompt", completion_key="completion", mask_prompt=True) args = TrainingArgs( batch_size=cfg.batch_size, iters=cfg.target_steps, max_seq_length=cfg.seq_len, steps_per_report=cfg.steps_per_report, steps_per_save=cfg.save_every, adapter_file=str(out / "adapters.safetensors"), grad_checkpoint=True, ) - mlx_train(model=model, tokenizer=tokenizer, args=args, - optimizer=optim.AdamW(learning_rate=cfg.learning_rate), - train_dataset=_DS(pairs), val_dataset=_DS(pairs[: max(1, len(pairs) // 10)])) + mlx_train(model=model, optimizer=optim.AdamW(learning_rate=cfg.learning_rate), + train_dataset=ds(pairs), val_dataset=ds(val), args=args) (out / "adapter_config.json").write_text(json.dumps({ "fine_tune_type": "lora", "lora_parameters": {"rank": cfg.lora_r}, "num_layers": 8, "model": base, From 542c68914c5ed44d5580daf1d7e9d12968b0b69f Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Thu, 23 Jul 2026 20:16:32 +0200 Subject: [PATCH 13/23] refactor(text2onto): mlx trainer uses the stable `mlx_lm lora` CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prior mlx trainer bound mlx_lm's low-level tuner API (train()/CompletionsDataset/ linear_to_lora_layers), which shifts between mlx_lm releases — three separate API mismatches surfaced while validating it. Replace that with a subprocess call to `python -m mlx_lm lora --train`, the stable CLI and the exact command the team's ft35_overnight.py used to produce the real MLX adapters. Robust across versions, no low-level loop to keep in sync. Reads the same {prompt, completion} jsonl (mask_prompt handled by the CLI), writes adapters.safetensors + adapter_config.json for backend="mlx" inference. Data rails (leave-one-out, prompt masking) and their tests are unchanged. --- CHANGELOG.md | 1 + .../text2onto/semanticswingers_train.py | 96 ++++++++----------- 2 files changed, 40 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96688e27..3bafc4a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Changelog ### Unreleased +- Change the MLX training backend to invoke the stable `mlx_lm lora` **CLI** (`python -m mlx_lm lora --train`) instead of binding mlx_lm's low-level `tuner` API, which churns across releases. This is the exact interface the team's own MLX trainer used, so it is version-robust and needs no re-implementation of the training loop. - Add **fine-tuning** to `SemanticSwingersText2OntoLearner` (`train_mode="raft"|"baseft"`, `train_backend="peft"|"mlx"`, `output_dir=`). Previously the fork shipped inference only — the adapter had to be trained elsewhere; `fit()` now trains a LoRA adapter and loads it. New `semanticswingers_train.py` carries the two load-bearing rails as pure-Python, unit-tested functions: prompt masking (loss on completion tokens only) and leave-one-out retrieval for RAFT (a training doc never sees its own gold). `train_mode=None` keeps inference unchanged. - Add `backend="mlx"` inference: run the fine-tuned champions on Apple Silicon (MLX base + MLX-format LoRA), where the transformers/PEFT path falls back to CPU. Completes the backend matrix: `peft` (CUDA), `mlx` (Apple Silicon), `ollama`/`openai` (API), and no-adapter base. - Add `backend=` to `SemanticSwingersText2OntoLearner` (`"peft"` default, plus `"ollama"` / `"openai"`), mirroring the `selector=` design already used by the term-typing and taxonomy-discovery learners. Task A generation was previously hardwired to transformers+PEFT, so serving it any other way required subclassing outside the package; API backends now need no local weights and no CUDA, making Task A runnable on a laptop like Tasks B and C. diff --git a/ontolearner/learner/text2onto/semanticswingers_train.py b/ontolearner/learner/text2onto/semanticswingers_train.py index 9c641f2c..cf5dca4d 100644 --- a/ontolearner/learner/text2onto/semanticswingers_train.py +++ b/ontolearner/learner/text2onto/semanticswingers_train.py @@ -230,67 +230,49 @@ def tokenize_fn(t: str) -> List[int]: def _train_mlx(pairs: List[Dict[str, str]], cfg: TrainConfig) -> str: - """LoRA SFT via ``mlx_lm`` on Apple Silicon. - - Uses ``mlx_lm``'s own LoRA trainer, which already implements prompt masking (``--mask-prompt``) - and writes ``adapters.safetensors`` + ``adapter_config.json`` — the format the inference path - loads with ``mlx_lm.load(model, adapter_path=...)``. Base defaults to the 4-bit MLX build; the - resulting adapter is a *separate 4-bit artifact* from the bf16 PEFT champions. - - STATUS (2026-07-23): this drives ``mlx_lm``'s **low-level** ``tuner`` API, which is not stable - across ``mlx_lm`` releases — the dataset ``__getitem__`` contract and the ``linear_to_lora_layers`` - config keys have both shifted between versions. This binding is written against the mlx_lm the - team pins and is **not yet validated end-to-end here**; treat it as the reference wiring, and for - a robust run prefer the ``mlx_lm`` LoRA CLI (``python -m mlx_lm lora``) over the two adapters - directory produced by :func:`write_jsonl`. The pure-Python data rails above (leave-one-out, - prompt masking) and the ``backend="mlx"`` *inference* path are validated; only this trainer's - low-level API binding is pending. The PEFT trainer is the validated-by-port CUDA path. + """LoRA SFT via the ``mlx_lm lora`` **CLI** on Apple Silicon. + + Invokes ``python -m mlx_lm lora --train`` — the stable, version-robust entry point, and the exact + command the team's own ``ft35_overnight.py`` used to produce the real MLX adapters. We do **not** + bind ``mlx_lm``'s low-level ``tuner`` API: it churns across releases (the dataset ``__getitem__`` + contract and the LoRA-config keys have both shifted), so re-implementing the loop is fragile for + no benefit. The CLI reads ``{prompt, completion}`` jsonl, applies prompt masking itself, and writes + ``adapters.safetensors`` + ``adapter_config.json`` — the format ``backend="mlx"`` inference loads. + + The base defaults to the 4-bit MLX build, so the adapter is a *separate 4-bit artifact* from the + bf16 PEFT champions (a Mac-native FT variant, not a reproduction of the reported scores). """ - try: - import mlx.optimizers as optim - from mlx_lm import load as mlx_load - from mlx_lm.tuner.datasets import CompletionsDataset - from mlx_lm.tuner.trainer import TrainingArgs, train as mlx_train - from mlx_lm.tuner.utils import linear_to_lora_layers - except ImportError as e: - raise RuntimeError( - "train_backend='mlx' needs mlx-lm (Apple Silicon). `pip install mlx-lm`. " - "Original error: " + str(e) - ) from e + import subprocess + import sys + + out = Path(cfg.output_dir) + write_jsonl(pairs, out / "train.jsonl") + write_jsonl(pairs[: max(1, len(pairs) // 10)], out / "valid.jsonl") base = cfg.base_model_id if base == "Qwen/Qwen3.5-9B": # bf16 id -> its MLX-quantized sibling base = "mlx-community/Qwen3.5-9B-4bit" - model, tokenizer = mlx_load(base) - out = Path(cfg.output_dir) - write_jsonl(pairs, out / "train.jsonl") - val = pairs[: max(1, len(pairs) // 10)] - - # LoRA-ify then train. CompletionsDataset applies mask_prompt itself (loss on completion only), - # which is exactly the masking rail we want — so the MLX path gets it for free from mlx_lm. - model.freeze() - linear_to_lora_layers(model, num_layers=8, config={ - "rank": cfg.lora_r, "scale": float(cfg.lora_alpha), "dropout": cfg.lora_dropout, - }) - class _Processed(CompletionsDataset): - # This mlx_lm build's CompletionsDataset.__getitem__ returns the raw record; the trainer's - # iterate_batches expects the *processed* (tokens, offset) tuple. Apply process() here. - def __getitem__(self, idx): - return self.process(self._data[idx]) - - ds = lambda rows: _Processed( - rows, tokenizer, prompt_key="prompt", completion_key="completion", mask_prompt=True) - - args = TrainingArgs( - batch_size=cfg.batch_size, iters=cfg.target_steps, max_seq_length=cfg.seq_len, - steps_per_report=cfg.steps_per_report, steps_per_save=cfg.save_every, - adapter_file=str(out / "adapters.safetensors"), grad_checkpoint=True, - ) - mlx_train(model=model, optimizer=optim.AdamW(learning_rate=cfg.learning_rate), - train_dataset=ds(pairs), val_dataset=ds(val), args=args) - (out / "adapter_config.json").write_text(json.dumps({ - "fine_tune_type": "lora", "lora_parameters": {"rank": cfg.lora_r}, - "num_layers": 8, "model": base, - }, indent=2)) + cmd = [ + sys.executable, "-m", "mlx_lm", "lora", + "--model", base, + "--train", + "--data", str(out), + "--fine-tune-type", "lora", + "--num-layers", "8", + "--batch-size", str(cfg.batch_size), + "--iters", str(cfg.target_steps), + "--max-seq-length", str(cfg.seq_len), + "--learning-rate", str(cfg.learning_rate), + "--steps-per-report", str(cfg.steps_per_report), + "--save-every", str(cfg.save_every), + "--adapter-path", str(out), + "--grad-checkpoint", + ] + result = subprocess.run(cmd) + if result.returncode != 0: + raise RuntimeError( + f"`mlx_lm lora` training failed (exit {result.returncode}). Command:\n " + + " ".join(cmd) + ) return str(out) From 368cfec9f91d042add3459dd130968b61711df05 Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Thu, 23 Jul 2026 21:02:04 +0200 Subject: [PATCH 14/23] feat(text2onto): point adapters at the published HF registry + MLX repo-id loading Updates _ADAPTER_REPOS to the published, generically-named registry (datagero/qwen3.5-9b-ontology-extraction-{raft,baseft,baseft-mlx}) and adds the MLX alias. So adapter="raft"/"baseft"/"baseft-mlx" now resolve to real, uploaded repos. backend="mlx" can now load an adapter given a HF repo id: mlx_lm.load(adapter_path=) only accepts a LOCAL directory (unlike PEFT's from_pretrained, which fetches repo ids), so we snapshot_download the adapter first when it isn't an on-disk path. Verified end-to-end against the private MLX repo on Apple Silicon: adapter="baseft-mlx" pulls the adapter, loads base+adapter, and generates triples (6 on a val_20 doc, ~10s load). Updates the alias test for the new names incl. baseft-mlx. --- CHANGELOG.md | 1 + .../learner/text2onto/semanticswingers.py | 16 ++++++++++++++-- tests/test_semanticswingers_text2onto.py | 4 +++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bafc4a2..78be31f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Changelog ### Unreleased +- Point `_ADAPTER_REPOS` at the published registry (`datagero/qwen3.5-9b-ontology-extraction-{raft,baseft,baseft-mlx}`) — generic names, and add the MLX alias. `backend="mlx"` now resolves a HF repo id by `snapshot_download` first (mlx_lm.load only accepts a local adapter dir, unlike PEFT which fetches repo ids directly), so `adapter="baseft-mlx"` loads the adapter straight from the Hub. Verified end-to-end from the private repo. - Change the MLX training backend to invoke the stable `mlx_lm lora` **CLI** (`python -m mlx_lm lora --train`) instead of binding mlx_lm's low-level `tuner` API, which churns across releases. This is the exact interface the team's own MLX trainer used, so it is version-robust and needs no re-implementation of the training loop. - Add **fine-tuning** to `SemanticSwingersText2OntoLearner` (`train_mode="raft"|"baseft"`, `train_backend="peft"|"mlx"`, `output_dir=`). Previously the fork shipped inference only — the adapter had to be trained elsewhere; `fit()` now trains a LoRA adapter and loads it. New `semanticswingers_train.py` carries the two load-bearing rails as pure-Python, unit-tested functions: prompt masking (loss on completion tokens only) and leave-one-out retrieval for RAFT (a training doc never sees its own gold). `train_mode=None` keeps inference unchanged. - Add `backend="mlx"` inference: run the fine-tuned champions on Apple Silicon (MLX base + MLX-format LoRA), where the transformers/PEFT path falls back to CPU. Completes the backend matrix: `peft` (CUDA), `mlx` (Apple Silicon), `ollama`/`openai` (API), and no-adapter base. diff --git a/ontolearner/learner/text2onto/semanticswingers.py b/ontolearner/learner/text2onto/semanticswingers.py index ca293e82..b8113790 100644 --- a/ontolearner/learner/text2onto/semanticswingers.py +++ b/ontolearner/learner/text2onto/semanticswingers.py @@ -47,6 +47,7 @@ import json import os import re +from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from ...base import AutoLearner, AutoRetriever @@ -85,8 +86,10 @@ # 2026-07-26; override via the `adapter` constructor arg with a local path in the meantime, e.g. # the team's `data/ft/_runners/raft_adapter/final`). _ADAPTER_REPOS = { - "raft": "datagero/qwen3.5-9b-raft-taska", - "baseft": "datagero/qwen3.5-9b-baseft-taska", + "raft": "datagero/qwen3.5-9b-ontology-extraction-raft", + "baseft": "datagero/qwen3.5-9b-ontology-extraction-baseft", + # Apple-Silicon 4-bit MLX variant (base-FT regime); pair with backend="mlx". + "baseft-mlx": "datagero/qwen3.5-9b-ontology-extraction-baseft-mlx", } @@ -293,6 +296,15 @@ def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: if mlx_base == "Qwen/Qwen3.5-9B": # bf16 id -> its MLX-quantized sibling mlx_base = "mlx-community/Qwen3.5-9B-4bit" adapter_path = self.adapter if self.adapter and self.adapter != "none" else None + # mlx_lm.load(adapter_path=...) only accepts a LOCAL dir — unlike PEFT it does not + # fetch a HF repo id. If the adapter is a repo id (not an on-disk path), pull it first. + if adapter_path and not Path(adapter_path).exists(): + from huggingface_hub import snapshot_download + adapter_path = snapshot_download( + repo_id=adapter_path, + allow_patterns=["adapters.safetensors", "adapter_config.json", + "adapter_model.safetensors"], + ) self._mlx_model, self._tokenizer = _mlx_load(mlx_base, adapter_path=adapter_path) self._model = self # sentinel self._taxonomy_learner.load(model_id=self.retriever_model_id) diff --git a/tests/test_semanticswingers_text2onto.py b/tests/test_semanticswingers_text2onto.py index d0e3ff3a..a48343c6 100644 --- a/tests/test_semanticswingers_text2onto.py +++ b/tests/test_semanticswingers_text2onto.py @@ -43,7 +43,9 @@ def test_load_raises_before_touching_the_network(monkeypatch): def test_known_adapter_alias_resolves_to_hf_repo(): learner = SemanticSwingersText2OntoLearner(adapter="raft") - assert learner.adapter == "datagero/qwen3.5-9b-raft-taska" + assert learner.adapter == "datagero/qwen3.5-9b-ontology-extraction-raft" + assert (SemanticSwingersText2OntoLearner(adapter="baseft-mlx").adapter + == "datagero/qwen3.5-9b-ontology-extraction-baseft-mlx") def test_unknown_adapter_value_passed_through_unchanged(): From 639b11ff323555617abc2bcd79d60e4a6b9f12c6 Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Thu, 23 Jul 2026 21:34:19 +0200 Subject: [PATCH 15/23] feat(learners): make prompt + relation schema injectable for reuse by others MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The learners baked our specific choices in as module constants, forcing anyone adapting them to a different domain to edit the package or subclass. Expose the two that matter as constructor arguments, defaulting to the team's values so existing behaviour is byte-identical: Task A: system_prompt=, typing_relations= Task B: system_prompt= Everything else was already configuration-driven (model, backend, adapter, retriever, training regime). Adds examples/llm_learner_semanticswingers_extend.py demonstrating the three levels of customization — arguments only (model/backend/retriever), domain (prompt + relations), and behaviour (subclass one method) — plus a training snippet, and documents the extension points on the class docstring. Backward-compatible; 5 new tests. Pre-existing suite failures unchanged (peft not in the dev venv; the qwen3_5 import-guard test can't trigger on released transformers). --- CHANGELOG.md | 1 + .../llm_learner_semanticswingers_extend.py | 75 +++++++++++++++++++ .../learner/term_typing/semanticswingers.py | 5 +- .../learner/text2onto/semanticswingers.py | 28 ++++++- tests/test_semanticswingers_term_typing.py | 8 ++ tests/test_semanticswingers_text2onto.py | 25 +++++++ 6 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 examples/llm_learner_semanticswingers_extend.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 78be31f7..78c1cba3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Changelog ### Unreleased +- Make the learners extensible without subclassing: `system_prompt` (Task A + Task B) and `typing_relations` (Task A) are now constructor arguments that default to the team's values, so others can adapt the prompt / relation schema to their own domain by passing arguments. Adds `examples/llm_learner_semanticswingers_extend.py` showing the three levels of customization (arguments -> prompt/relations -> subclass one method) and documents the extension points on the class. - Point `_ADAPTER_REPOS` at the published registry (`datagero/qwen3.5-9b-ontology-extraction-{raft,baseft,baseft-mlx}`) — generic names, and add the MLX alias. `backend="mlx"` now resolves a HF repo id by `snapshot_download` first (mlx_lm.load only accepts a local adapter dir, unlike PEFT which fetches repo ids directly), so `adapter="baseft-mlx"` loads the adapter straight from the Hub. Verified end-to-end from the private repo. - Change the MLX training backend to invoke the stable `mlx_lm lora` **CLI** (`python -m mlx_lm lora --train`) instead of binding mlx_lm's low-level `tuner` API, which churns across releases. This is the exact interface the team's own MLX trainer used, so it is version-robust and needs no re-implementation of the training loop. - Add **fine-tuning** to `SemanticSwingersText2OntoLearner` (`train_mode="raft"|"baseft"`, `train_backend="peft"|"mlx"`, `output_dir=`). Previously the fork shipped inference only — the adapter had to be trained elsewhere; `fit()` now trains a LoRA adapter and loads it. New `semanticswingers_train.py` carries the two load-bearing rails as pure-Python, unit-tested functions: prompt masking (loss on completion tokens only) and leave-one-out retrieval for RAFT (a training doc never sees its own gold). `train_mode=None` keeps inference unchanged. diff --git a/examples/llm_learner_semanticswingers_extend.py b/examples/llm_learner_semanticswingers_extend.py new file mode 100644 index 00000000..98070415 --- /dev/null +++ b/examples/llm_learner_semanticswingers_extend.py @@ -0,0 +1,75 @@ +"""Extending the Semantic-Swingers Task A learner for *your own* experiments. + +The learner is deliberately configuration-driven: base model, adapter, generation backend, +retriever, training regime, the extraction **prompt**, and the relation set that projects to +`types` are all constructor arguments. So most adaptations need **no subclassing at all** — you +pass different arguments. This file shows the three levels of customization, cheapest first. + +Run with a local Ollama (`ollama serve` + a small model) so it needs no API key or GPU. +""" + +from ontolearner.learner.text2onto import SemanticSwingersText2OntoLearner + + +# --------------------------------------------------------------------------------------------- +# Level 1 — same method, YOUR model / backend / retriever (no code, just arguments) +# --------------------------------------------------------------------------------------------- +# Swap the generator (any OpenAI-compatible endpoint via backend="ollama"/"openai", any local +# checkpoint via backend="peft"/"mlx"), the retriever encoder, and how many exemplars to use. +learner = SemanticSwingersText2OntoLearner( + backend="ollama", + llm_model="llama3.1:8b", # <- your generator + retriever_model_id="sentence-transformers/all-mpnet-base-v2", # <- your retriever + top_k=5, +) + + +# --------------------------------------------------------------------------------------------- +# Level 2 — YOUR domain: a different extraction prompt and a different relation vocabulary +# --------------------------------------------------------------------------------------------- +# `system_prompt` replaces the extraction instructions; `typing_relations` controls which +# relations count as term→type edges when projecting to OntoLearner's {terms, types} shape. +# Neither requires touching the package. +MY_PROMPT = ( + "You are a biomedical ontology engineer. Extract triples [subject, relation, object] using " + "ONLY these relations: rdfs:subClassOf, rdf:type, part_of. " + 'Output ONLY JSON {"triples": [[s, r, o], ...]}.' +) +domain_learner = SemanticSwingersText2OntoLearner( + backend="ollama", + llm_model="llama3.1:8b", + system_prompt=MY_PROMPT, # <- your instructions + typing_relations={"rdf:type", "rdfs:subClassOf"}, # <- your typing relations +) + + +# --------------------------------------------------------------------------------------------- +# Level 3 — YOUR pipeline: subclass to change one step, reuse the rest +# --------------------------------------------------------------------------------------------- +# When a *behaviour* needs to change (not just a value), override a single method. Everything +# else — retrieval, backend dispatch, training, the {terms, types} projection — is inherited. +class MyText2OntoLearner(SemanticSwingersText2OntoLearner): + """Example: post-filter generated triples to a relation allow-list of your choosing.""" + + ALLOW = {"rdf:type", "rdfs:subClassOf", "part_of"} + + def _generate_triples(self, text, exemplars): + triples = super()._generate_triples(text, exemplars) # reuse the whole generation path + return [(s, r, o) for (s, r, o) in triples if r in self.ALLOW] + + +# --------------------------------------------------------------------------------------------- +# Training your own adapter is one more argument, not a separate script (train_mode + fit()). +# --------------------------------------------------------------------------------------------- +# trainer = SemanticSwingersText2OntoLearner( +# train_mode="raft", # or "baseft" +# train_backend="mlx", # or "peft" (CUDA) +# output_dir="my_adapter", +# system_prompt=MY_PROMPT, # trains against YOUR prompt +# ) +# trainer.fit(train_docs, task="text2onto") # builds pairs (leave-one-out for raft) -> trains -> loads + +if __name__ == "__main__": + print("Level 1 learner:", learner.llm_model, "top_k", learner.top_k) + print("Level 2 typing relations:", sorted(domain_learner.typing_relations)) + print("Level 3 subclass:", MyText2OntoLearner().__class__.__name__) diff --git a/ontolearner/learner/term_typing/semanticswingers.py b/ontolearner/learner/term_typing/semanticswingers.py index 0f5add5e..628dfa23 100644 --- a/ontolearner/learner/term_typing/semanticswingers.py +++ b/ontolearner/learner/term_typing/semanticswingers.py @@ -99,6 +99,7 @@ def __init__( max_tokens: int = 1024, batch_size: int = 20, device: str = "cpu", + system_prompt: Optional[str] = None, ) -> None: """Initialise the learner and record configuration (no I/O yet).""" super().__init__() @@ -112,6 +113,8 @@ def __init__( self.max_tokens = max_tokens self.batch_size = batch_size self.device = device + # Extension point: override the classification instructions without subclassing. + self.system_prompt = system_prompt or _SYSTEM self._encoder: Optional[SentenceTransformer] = None self._allowed_types: List[str] = [] @@ -196,7 +199,7 @@ def _select_llm(self, terms: List[str]) -> List[Dict[str, Any]]: params: Dict[str, Any] = dict( model=self.llm_model, messages=[ - {"role": "system", "content": _SYSTEM}, + {"role": "system", "content": self.system_prompt}, {"role": "user", "content": user}, ], temperature=0, diff --git a/ontolearner/learner/text2onto/semanticswingers.py b/ontolearner/learner/text2onto/semanticswingers.py index b8113790..ddfb778c 100644 --- a/ontolearner/learner/text2onto/semanticswingers.py +++ b/ontolearner/learner/text2onto/semanticswingers.py @@ -193,8 +193,24 @@ class SemanticSwingersText2OntoLearner(AutoLearner): device: Torch device string (``"cpu"``, ``"cuda"``, ``"mps"``, ...). CPU works but is slow without the ``fla``/``causal-conv1d`` fast kernels (GPU-only); see the class docstring's model-portability note. + system_prompt: Extraction instructions handed to the generator. Defaults to the team's + (:data:`_SYSTEM_PROMPT`). **Override to adapt to your domain / relation vocabulary** + without subclassing. + typing_relations: The relations that project a triple ``(s, r, o)`` to a ``type`` when + reshaping to OntoLearner's ``{terms, types}`` output. Defaults to + ``{"is-a", "instance-of", "type"}``; override for a different schema (e.g. + ``{"rdf:type", "rdfs:subClassOf"}``). taxonomy_kwargs: Extra keyword arguments forwarded to the composed :class:`SemanticSwingersTaxonomyLearner` used for the ``_taxonomy_discovery`` hook. + + Extending this learner (see ``examples/llm_learner_semanticswingers_extend.py``): + * **Different model / backend / retriever** — pass ``backend``, ``llm_model``, + ``base_model_id``, ``adapter``, ``retriever_model_id``. No code. + * **Different domain** — pass ``system_prompt`` and ``typing_relations``. No code. + * **Different behaviour** — subclass and override one method (e.g. ``_generate_triples``); + retrieval, backend dispatch, training and the projection are all inherited. + * **Your own adapter** — ``train_mode`` + ``fit()`` train against whatever ``system_prompt`` + you set. """ _OLLAMA_BASE_URL = "http://localhost:11434/v1" @@ -212,6 +228,8 @@ def __init__( retriever_model_id: str = "sentence-transformers/all-MiniLM-L6-v2", max_new_tokens: int = 1500, device: str = "cpu", + system_prompt: Optional[str] = None, + typing_relations: Optional[set] = None, train_mode: Optional[str] = None, train_backend: str = "peft", output_dir: Optional[str] = None, @@ -230,6 +248,10 @@ def __init__( self.adapter = _ADAPTER_REPOS.get(adapter, adapter) self.base_model_id = base_model_id self.backend = backend + # Extension points: bring your own extraction instructions / typing relations without + # subclassing. Default to the team's, so existing behaviour is unchanged. + self.system_prompt = system_prompt or _SYSTEM_PROMPT + self.typing_relations = set(typing_relations) if typing_relations else _TYPING_RELATIONS self.train_mode = train_mode self.train_backend = train_backend self.output_dir = output_dir @@ -344,7 +366,7 @@ def _build_prompt(self, text: str, exemplars: List[Dict[str, Any]]) -> str: ex_block = "".join(self._exemplar_block(e) for e in exemplars) user = f"<|im_start|>user\nDocument:\n{ctx}\n\nExtract the ontology triples as JSON.<|im_end|>\n" return ( - f"<|im_start|>system\n{_SYSTEM_PROMPT}<|im_end|>\n" + ex_block + user + _NOTHINK_SUFFIX + f"<|im_start|>system\n{self.system_prompt}<|im_end|>\n" + ex_block + user + _NOTHINK_SUFFIX ) @staticmethod @@ -359,7 +381,7 @@ def _chat_messages(self, text: str, exemplars: List[Dict[str, Any]]) -> List[Dic The PEFT path renders exemplars into a single ChatML string; API backends want structured turns. Content is identical so the two backends stay comparable. """ - msgs: List[Dict[str, str]] = [{"role": "system", "content": _SYSTEM_PROMPT}] + msgs: List[Dict[str, str]] = [{"role": "system", "content": self.system_prompt}] for ex in exemplars: payload = json.dumps({"triples": [list(t) for t in ex["triples"][:20]]}, ensure_ascii=False) @@ -537,7 +559,7 @@ def _text2onto(self, data: Any, test: bool = False) -> Optional[Dict[str, List[A if s not in terms_seen: terms_seen.add(s) pred_terms.append({"doc_id": did, "term": s}) - if r in _TYPING_RELATIONS and o not in types_seen: + if r in self.typing_relations and o not in types_seen: types_seen.add(o) pred_types.append({"doc_id": did, "type": o}) diff --git a/tests/test_semanticswingers_term_typing.py b/tests/test_semanticswingers_term_typing.py index 73122748..69b4064c 100644 --- a/tests/test_semanticswingers_term_typing.py +++ b/tests/test_semanticswingers_term_typing.py @@ -164,3 +164,11 @@ def test_reasoning_off_only_for_qwen3(): assert _reasoning_off("gpt-4.1-mini") == {} assert _reasoning_off("llama3.1:8b") == {} assert _reasoning_off(None) == {} + + +def test_system_prompt_is_injectable_and_defaults(): + from ontolearner.learner.term_typing.semanticswingers import ( + SemanticSwingersTermTypingLearner, _SYSTEM, + ) + assert SemanticSwingersTermTypingLearner().system_prompt == _SYSTEM + assert SemanticSwingersTermTypingLearner(system_prompt="CUSTOM").system_prompt == "CUSTOM" diff --git a/tests/test_semanticswingers_text2onto.py b/tests/test_semanticswingers_text2onto.py index a48343c6..5b857540 100644 --- a/tests/test_semanticswingers_text2onto.py +++ b/tests/test_semanticswingers_text2onto.py @@ -378,3 +378,28 @@ def test_train_adapter_rejects_unknown_backend(): assert False, "expected ValueError" except ValueError as e: assert "peft" in str(e) and "mlx" in str(e) + + +# -- extension points ------------------------------------------------------------------------- + +def test_system_prompt_is_injectable_and_defaults(): + """Others can bring their own extraction instructions without subclassing.""" + from ontolearner.learner.text2onto.semanticswingers import ( + SemanticSwingersText2OntoLearner, _SYSTEM_PROMPT, + ) + assert SemanticSwingersText2OntoLearner().system_prompt == _SYSTEM_PROMPT + custom = SemanticSwingersText2OntoLearner(system_prompt="MY DOMAIN RULES") + assert custom.system_prompt == "MY DOMAIN RULES" + # the custom prompt actually reaches the built prompt + chat messages + assert "MY DOMAIN RULES" in custom._build_prompt("doc", []) + assert custom._chat_messages("doc", [])[0]["content"] == "MY DOMAIN RULES" + + +def test_typing_relations_is_injectable_and_defaults(): + """The relation set that projects to `types` is domain-specific and overridable.""" + from ontolearner.learner.text2onto.semanticswingers import ( + SemanticSwingersText2OntoLearner, _TYPING_RELATIONS, + ) + assert SemanticSwingersText2OntoLearner().typing_relations == _TYPING_RELATIONS + custom = SemanticSwingersText2OntoLearner(typing_relations={"rdf:type", "subclass_of"}) + assert custom.typing_relations == {"rdf:type", "subclass_of"} From a242b278f5b199bd63dbd6f6f50e373799eba18f Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Thu, 23 Jul 2026 21:49:48 +0200 Subject: [PATCH 16/23] feat(learners): tokenizer-driven prompt template + Task C selection prompt injectable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #2 (generalize the local prompt to any model family): _build_prompt now renders the same message list through the loaded tokenizer's own chat template (apply_chat_template) instead of hardcoded Qwen ChatML, so a non-Qwen local base model works on the peft/mlx path (the ollama/openai backends were already model-agnostic via role messages). VERIFIED byte-identical to the old hand-built ChatML for Qwen/Qwen3.5-9B — enable_thinking=False reproduces the no-think suffix exactly — so the team's trained adapters see the same prompt. Falls back to the manual ChatML when no tokenizer/template is present. Confirmed end-to-end: mlx adapter still generates triples through the new path. #3 (Task C prompt extensibility): the parent-selection instruction is now a selection_prompt= constructor arg (a {child}/{candidates} template, default unchanged), matching the system_prompt/typing_relations extension points on Task A/B. Backward-compatible; +1 test. Pre-existing suite failures unchanged (peft absent in the dev venv; qwen3_5 import-guard can't trigger on released transformers). --- CHANGELOG.md | 2 ++ .../taxonomy_discovery/semanticswingers.py | 22 ++++++++++++++----- .../learner/text2onto/semanticswingers.py | 22 +++++++++++++++++-- ...est_semanticswingers_taxonomy_discovery.py | 10 +++++++++ 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78c1cba3..f5556e55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## Changelog ### Unreleased +- Make the local (peft/mlx) prompt use the tokenizer's own chat template instead of hardcoded Qwen ChatML, so the Task A learner works with any model family — verified byte-identical to the old hand-built prompt for Qwen (`enable_thinking=False` reproduces the no-think suffix), so the team's trained adapters are unaffected; falls back to the manual ChatML if no tokenizer. +- Make the Task C taxonomy learner's parent-selection prompt injectable (`selection_prompt=`, a `{child}`/`{candidates}` template) — same extensibility as Task A/B prompts. - Make the learners extensible without subclassing: `system_prompt` (Task A + Task B) and `typing_relations` (Task A) are now constructor arguments that default to the team's values, so others can adapt the prompt / relation schema to their own domain by passing arguments. Adds `examples/llm_learner_semanticswingers_extend.py` showing the three levels of customization (arguments -> prompt/relations -> subclass one method) and documents the extension points on the class. - Point `_ADAPTER_REPOS` at the published registry (`datagero/qwen3.5-9b-ontology-extraction-{raft,baseft,baseft-mlx}`) — generic names, and add the MLX alias. `backend="mlx"` now resolves a HF repo id by `snapshot_download` first (mlx_lm.load only accepts a local adapter dir, unlike PEFT which fetches repo ids directly), so `adapter="baseft-mlx"` loads the adapter straight from the Hub. Verified end-to-end from the private repo. - Change the MLX training backend to invoke the stable `mlx_lm lora` **CLI** (`python -m mlx_lm lora --train`) instead of binding mlx_lm's low-level `tuner` API, which churns across releases. This is the exact interface the team's own MLX trainer used, so it is version-robust and needs no re-implementation of the training loop. diff --git a/ontolearner/learner/taxonomy_discovery/semanticswingers.py b/ontolearner/learner/taxonomy_discovery/semanticswingers.py index 4680a118..c388dade 100644 --- a/ontolearner/learner/taxonomy_discovery/semanticswingers.py +++ b/ontolearner/learner/taxonomy_discovery/semanticswingers.py @@ -37,6 +37,13 @@ from ...base import AutoLearner +#: Default parent-selection instruction for the LLM selectors. A template with ``{child}`` and +#: ``{candidates}`` placeholders; override per-instance via the ``selection_prompt`` argument. +_SELECTION_PROMPT = ( + "Which of the following is the most likely direct parent (superclass) of '{child}'? " + "Answer with exactly one option or 'NONE'.\n{candidates}" +) + def _reasoning_off(model: Optional[str]) -> dict: """Extra request kwargs that disable a thinking model's hidden reasoning pass. @@ -89,8 +96,14 @@ def __init__( base_url: Optional[str] = None, max_tokens: int = 64, device: str = "cpu", + selection_prompt: Optional[str] = None, ) -> None: - """Initialise the learner and record configuration (no I/O yet).""" + """Initialise the learner and record configuration (no I/O yet). + + ``selection_prompt`` is the parent-selection instruction for the LLM selectors — a template + with ``{child}`` and ``{candidates}`` placeholders. Defaults to :data:`_SELECTION_PROMPT`; + override to change the phrasing / task framing without subclassing. + """ super().__init__() self.embedding_model = embedding_model self.top_k = top_k @@ -102,6 +115,7 @@ def __init__( self.base_url = base_url self.max_tokens = max_tokens self.device = device + self.selection_prompt = selection_prompt or _SELECTION_PROMPT self._encoder: Optional[SentenceTransformer] = None def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: @@ -172,10 +186,8 @@ def _select_llm(self, nodes: List[str], sim: np.ndarray) -> List[dict]: ] if not candidates: continue - prompt = ( - f"Which of the following is the most likely direct parent (superclass) " - f"of '{child}'? Answer with exactly one option or 'NONE'.\n" - + "\n".join(f"- {c}" for c in candidates) + prompt = self.selection_prompt.format( + child=child, candidates="\n".join(f"- {c}" for c in candidates) ) resp = client.chat.completions.create( model=self.llm_model, diff --git a/ontolearner/learner/text2onto/semanticswingers.py b/ontolearner/learner/text2onto/semanticswingers.py index ddfb778c..08fc16a1 100644 --- a/ontolearner/learner/text2onto/semanticswingers.py +++ b/ontolearner/learner/text2onto/semanticswingers.py @@ -362,9 +362,27 @@ def _exemplar_block(self, ex: Dict[str, Any], max_ctx: int = 1000, max_triples: ) def _build_prompt(self, text: str, exemplars: List[Dict[str, Any]]) -> str: - ctx = text[:4000] + """Render the generation prompt for the local (peft/mlx) backends. + + Uses the loaded tokenizer's **own chat template** so the learner works with any model + family, not just Qwen's ChatML. Verified byte-identical to the hand-built ChatML for + ``Qwen/Qwen3.5-9B`` (``enable_thinking=False`` reproduces the ```` no-think + suffix exactly), so this does not perturb the team's trained adapters. Falls back to the + Qwen ChatML string only if no tokenizer / chat template is available. + """ + msgs = self._chat_messages(text, exemplars) + tok = getattr(self, "_tokenizer", None) + if tok is not None and getattr(tok, "chat_template", None): + try: + return tok.apply_chat_template( + msgs, add_generation_prompt=True, tokenize=False, enable_thinking=False + ) + except TypeError: # tokenizer's template doesn't take enable_thinking + return tok.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False) + # Fallback: the original hand-built Qwen ChatML (no tokenizer available). ex_block = "".join(self._exemplar_block(e) for e in exemplars) - user = f"<|im_start|>user\nDocument:\n{ctx}\n\nExtract the ontology triples as JSON.<|im_end|>\n" + user = (f"<|im_start|>user\nDocument:\n{text[:4000]}\n\n" + "Extract the ontology triples as JSON.<|im_end|>\n") return ( f"<|im_start|>system\n{self.system_prompt}<|im_end|>\n" + ex_block + user + _NOTHINK_SUFFIX ) diff --git a/tests/test_semanticswingers_taxonomy_discovery.py b/tests/test_semanticswingers_taxonomy_discovery.py index 82c4a997..3529a507 100644 --- a/tests/test_semanticswingers_taxonomy_discovery.py +++ b/tests/test_semanticswingers_taxonomy_discovery.py @@ -167,3 +167,13 @@ def test_reasoning_off_only_for_qwen3(): assert _reasoning_off("gpt-4.1-mini") == {} assert _reasoning_off("llama3.1:8b") == {} assert _reasoning_off(None) == {} + + +def test_selection_prompt_is_injectable_and_defaults(): + from ontolearner.learner.taxonomy_discovery.semanticswingers import ( + SemanticSwingersTaxonomyLearner, _SELECTION_PROMPT, + ) + assert SemanticSwingersTaxonomyLearner().selection_prompt == _SELECTION_PROMPT + custom = SemanticSwingersTaxonomyLearner(selection_prompt="parent of {child}: {candidates}") + out = custom.selection_prompt.format(child="Chianti", candidates="- Wine") + assert out == "parent of Chianti: - Wine" From 9985026aa75417ea7d15fd040afb650af13be6f3 Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Fri, 24 Jul 2026 11:47:42 +0200 Subject: [PATCH 17/23] =?UTF-8?q?fix(text2onto):=20honor=20device=3D=20in?= =?UTF-8?q?=20peft=20load=20(device=5Fmap)=20=E2=80=94=20was=20loading=209?= =?UTF-8?q?B=20on=20CPU?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The peft backend called from_pretrained without device_map/.to, so device='cuda' was ignored and the base+adapter loaded on CPU — correct output but minutes/doc (GPU 0% util, ~390MiB) on a CUDA box, unusable at scale. Load weights directly onto the requested accelerator via device_map; CPU stays default when device=cpu. Surfaced by the notebook §5 RunPod CUDA validation (2026-07-24). --- ontolearner/learner/text2onto/semanticswingers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ontolearner/learner/text2onto/semanticswingers.py b/ontolearner/learner/text2onto/semanticswingers.py index 08fc16a1..da27feff 100644 --- a/ontolearner/learner/text2onto/semanticswingers.py +++ b/ontolearner/learner/text2onto/semanticswingers.py @@ -338,11 +338,17 @@ def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: from peft import PeftModel self._tokenizer = AutoTokenizer.from_pretrained(self.base_model_id, trust_remote_code=True) + # Place the model on the requested device. Without device_map the base loads on CPU + # regardless of `device=`, so a "cuda" run would silently execute the 9B on CPU + # (minutes/doc). device_map loads weights straight onto the accelerator (also avoids a + # CPU-RAM spike vs. load-then-.to). CPU stays the default when device is cpu/unset. + _device_map = self.device if self.device and self.device != "cpu" else None base = AutoModelForCausalLM.from_pretrained( self.base_model_id, dtype=torch.bfloat16, trust_remote_code=True, low_cpu_mem_usage=True, + device_map=_device_map, ) self._model = PeftModel.from_pretrained(base, self.adapter) self._model.eval() From 35f110f6f8b1d28ce368313459cd28a7350c1ace Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Sun, 26 Jul 2026 18:42:06 +0200 Subject: [PATCH 18/23] style: satisfy ruff E702/E731 in text2onto trainer + tests --- ontolearner/learner/text2onto/semanticswingers_train.py | 8 ++++++-- tests/test_semanticswingers_text2onto.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ontolearner/learner/text2onto/semanticswingers_train.py b/ontolearner/learner/text2onto/semanticswingers_train.py index cf5dca4d..f4e2db62 100644 --- a/ontolearner/learner/text2onto/semanticswingers_train.py +++ b/ontolearner/learner/text2onto/semanticswingers_train.py @@ -213,13 +213,17 @@ def tokenize_fn(t: str) -> List[int]: attention_mask=torch.tensor(batch["attention_mask"], device=device), labels=torch.tensor(batch["labels"], device=device)) if not torch.isfinite(out.loss): - optim.zero_grad(); micro += 1; continue + optim.zero_grad() + micro += 1 + continue (out.loss / cfg.grad_accum).backward() micro += 1 if micro % cfg.grad_accum: continue torch.nn.utils.clip_grad_norm_((p for p in model.parameters() if p.requires_grad), 1.0) - optim.step(); optim.zero_grad(); step += 1 + optim.step() + optim.zero_grad() + step += 1 if step % cfg.steps_per_report == 0: print(f"[peft] step {step}/{cfg.target_steps} loss={out.loss.item():.4f}", flush=True) if step % cfg.save_every == 0: diff --git a/tests/test_semanticswingers_text2onto.py b/tests/test_semanticswingers_text2onto.py index 5b857540..ae983daf 100644 --- a/tests/test_semanticswingers_text2onto.py +++ b/tests/test_semanticswingers_text2onto.py @@ -314,7 +314,7 @@ def test_encode_example_masks_prompt_tokens(): from ontolearner.learner.text2onto.semanticswingers_train import encode_example ids = {"the prompt": [1, 2, 3], "the completion": [4, 5]} - tok = lambda s: ids[s] + def tok(s): return ids[s] input_ids, labels = encode_example(tok, eos_id=9, prompt="the prompt", completion="the completion", max_len=100) assert input_ids == [1, 2, 3, 4, 5, 9] @@ -325,7 +325,7 @@ def test_encode_example_left_truncates_prompt_on_overflow(): """On overflow keep the completion (the target) intact; drop prompt from the left.""" from ontolearner.learner.text2onto.semanticswingers_train import encode_example - tok = lambda s: list(range(10)) if s == "P" else [100, 101] + def tok(s): return list(range(10)) if s == "P" else [100, 101] input_ids, labels = encode_example(tok, eos_id=None, prompt="P", completion="C", max_len=4) assert input_ids[-2:] == [100, 101] # completion survives assert labels[-2:] == [100, 101] From 1d2fb828c61484e04815588878963bf66eb993eb Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Sun, 26 Jul 2026 19:13:44 +0200 Subject: [PATCH 19/23] Drop CHANGELOG edits from PR (maintainer-owned, release-structured) --- CHANGELOG.md | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5556e55..e1125451 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,36 +1,5 @@ ## Changelog -### Unreleased -- Make the local (peft/mlx) prompt use the tokenizer's own chat template instead of hardcoded Qwen ChatML, so the Task A learner works with any model family — verified byte-identical to the old hand-built prompt for Qwen (`enable_thinking=False` reproduces the no-think suffix), so the team's trained adapters are unaffected; falls back to the manual ChatML if no tokenizer. -- Make the Task C taxonomy learner's parent-selection prompt injectable (`selection_prompt=`, a `{child}`/`{candidates}` template) — same extensibility as Task A/B prompts. -- Make the learners extensible without subclassing: `system_prompt` (Task A + Task B) and `typing_relations` (Task A) are now constructor arguments that default to the team's values, so others can adapt the prompt / relation schema to their own domain by passing arguments. Adds `examples/llm_learner_semanticswingers_extend.py` showing the three levels of customization (arguments -> prompt/relations -> subclass one method) and documents the extension points on the class. -- Point `_ADAPTER_REPOS` at the published registry (`datagero/qwen3.5-9b-ontology-extraction-{raft,baseft,baseft-mlx}`) — generic names, and add the MLX alias. `backend="mlx"` now resolves a HF repo id by `snapshot_download` first (mlx_lm.load only accepts a local adapter dir, unlike PEFT which fetches repo ids directly), so `adapter="baseft-mlx"` loads the adapter straight from the Hub. Verified end-to-end from the private repo. -- Change the MLX training backend to invoke the stable `mlx_lm lora` **CLI** (`python -m mlx_lm lora --train`) instead of binding mlx_lm's low-level `tuner` API, which churns across releases. This is the exact interface the team's own MLX trainer used, so it is version-robust and needs no re-implementation of the training loop. -- Add **fine-tuning** to `SemanticSwingersText2OntoLearner` (`train_mode="raft"|"baseft"`, `train_backend="peft"|"mlx"`, `output_dir=`). Previously the fork shipped inference only — the adapter had to be trained elsewhere; `fit()` now trains a LoRA adapter and loads it. New `semanticswingers_train.py` carries the two load-bearing rails as pure-Python, unit-tested functions: prompt masking (loss on completion tokens only) and leave-one-out retrieval for RAFT (a training doc never sees its own gold). `train_mode=None` keeps inference unchanged. -- Add `backend="mlx"` inference: run the fine-tuned champions on Apple Silicon (MLX base + MLX-format LoRA), where the transformers/PEFT path falls back to CPU. Completes the backend matrix: `peft` (CUDA), `mlx` (Apple Silicon), `ollama`/`openai` (API), and no-adapter base. -- Add `backend=` to `SemanticSwingersText2OntoLearner` (`"peft"` default, plus `"ollama"` / `"openai"`), mirroring the `selector=` design already used by the term-typing and taxonomy-discovery learners. Task A generation was previously hardwired to transformers+PEFT, so serving it any other way required subclassing outside the package; API backends now need no local weights and no CUDA, making Task A runnable on a laptop like Tasks B and C. -- Fix Semantic-Swingers LLM selectors (term-typing + taxonomy-discovery) returning empty output (F1=0) on qwen3.x thinking models: pass `reasoning_effort="none"` via `extra_body` for `qwen3*` models so the model answers instead of spending its whole token budget on hidden reasoning. No-op for non-qwen3 models. (Wine smoke: term-typing 0.0->~0.66, taxonomy-discovery 0.0->~0.14 with local qwen3.5-nothink:9b.) -- Add Semantic-Swingers term-typing learner (LLMs4OL 2026, Task B): closed-vocabulary - term typing with a swappable type selector (offline embedding baseline / OpenAI - champion / local Ollama reproduction). -- Add Semantic-Swingers taxonomy-discovery learner (LLMs4OL 2026, Task C): retrieval-first - taxonomy induction with a swappable parent selector (offline embedding baseline / OpenAI - champion / local Ollama reproduction), plus documentation page and example. -- Add hermetic unit tests for both Semantic-Swingers learners (fit/predict contract, - embedding-selector determinism, closed-vocabulary parsing, openai-selector fallback with - no api_key) and a Reproducibility subsection to the Semantic-Swingers docs page. -- Add Semantic-Swingers text2onto + taxonomy-discovery learner (LLMs4OL 2026, Task A, - flagship): one class, two hooks — retrieval-augmented generation with a LoRA fine-tuned - Qwen3.5-9B (RA-FT/base-FT) for text2onto, composed with the existing taxonomy-discovery - learner (not a rewrite) for the vocabulary-only native harness. Requires transformers - installed from git source for the qwen3_5 architecture (lazy-imported, actionable - ImportError if missing — not added to core pyproject.toml), plus documentation section, - example, and hermetic unit tests. - - ``_text2onto`` now also returns the raw, unprojected extraction under an extra - ``"triples"`` key (harness-ignored: ``text2onto_metrics`` reads only ``terms``/``types``), - preserving the ``is-a``-dominant signal the native projection discards, for any - downstream consumer of ``run_report['predictions']``. - ### v1.5.1 (March 30, 2026) - Fix challenge learner - Update requirements. From 77f13a2ae91fb1ec7d72ec21167cc8c51d27c302 Mon Sep 17 00:00:00 2001 From: andy-symonds Date: Sun, 26 Jul 2026 23:42:52 +0100 Subject: [PATCH 20/23] AS | Integrated 1024-D Matrix taxonomy learner with DAG cleanup and HF auto-download --- ...ner_semanticswingers_taxonomy_discovery.py | 35 +++- .../learner/taxonomy_discovery/__init__.py | 5 +- .../taxonomy_discovery/semanticswingers.py | 150 +++++++++++++++++- 3 files changed, 187 insertions(+), 3 deletions(-) diff --git a/examples/llm_learner_semanticswingers_taxonomy_discovery.py b/examples/llm_learner_semanticswingers_taxonomy_discovery.py index 2ea7bb92..4895dc1e 100644 --- a/examples/llm_learner_semanticswingers_taxonomy_discovery.py +++ b/examples/llm_learner_semanticswingers_taxonomy_discovery.py @@ -1,5 +1,8 @@ from ontolearner import Wine, train_test_split, LearnerPipeline -from ontolearner.learner.taxonomy_discovery import SemanticSwingersTaxonomyLearner +from ontolearner.learner.taxonomy_discovery import ( + SemanticSwingersTaxonomyLearner, + SemanticSwingersMatrixTaxonomyLearner +) # 1) Load & split ontology = Wine() @@ -7,6 +10,9 @@ data = ontology.extract() train_data, test_data = train_test_split(data, test_size=0.2, random_state=42) +# ===================================================================== +# APPROACH 1 +# ===================================================================== # 2) Configure the semantic-swingers taxonomy learner. # Offline default (no API key): the "embedding" selector. # Champion: selector="openai" + api_key=... for gpt-4.1-mini parent selection. @@ -36,3 +42,30 @@ print("Metrics:", outputs.get("metrics")) print("Elapsed time:", outputs["elapsed_time"]) + +# ===================================================================== +# APPROACH 2: 1024-D Structural Matrix + DAG Reduction +# ===================================================================== + +learner2 = SemanticSwingersMatrixTaxonomyLearner( + embedding_model="mixedbread-ai/mxbai-embed-large-v1", + matrix_weights_path="structural_matrix_w_1024_mxbai.pt", + top_k=10, + llm_threshold=1000, # Automatically skips LLM for massive datasets + selector="embedding", # change to 'openai' for full competition pipeline + device="cpu", +) + +pipeline2 = LearnerPipeline( + llm=learner2, + llm_id="semanticswingers-taxonomy-champion", + ontologizer_data=False, +) + +outputs2 = pipeline2( + train_data=train_data, + test_data=test_data, + task="taxonomy-discovery", + evaluate=True, + ontologizer_data=False, +) \ No newline at end of file diff --git a/ontolearner/learner/taxonomy_discovery/__init__.py b/ontolearner/learner/taxonomy_discovery/__init__.py index 22d173e5..361c7bef 100644 --- a/ontolearner/learner/taxonomy_discovery/__init__.py +++ b/ontolearner/learner/taxonomy_discovery/__init__.py @@ -15,5 +15,8 @@ from .alexbek import AlexbekCrossAttnLearner from .rwthdbis import RWTHDBISSFTLearner from .sbunlp import SBUNLPFewShotLearner -from .semanticswingers import SemanticSwingersTaxonomyLearner +from .semanticswingers import ( + SemanticSwingersTaxonomyLearner, + SemanticSwingersMatrixTaxonomyLearner, +) from .skhnlp import SKHNLPSequentialFTLearner, SKHNLPZSLearner diff --git a/ontolearner/learner/taxonomy_discovery/semanticswingers.py b/ontolearner/learner/taxonomy_discovery/semanticswingers.py index c388dade..0eb6ffd8 100644 --- a/ontolearner/learner/taxonomy_discovery/semanticswingers.py +++ b/ontolearner/learner/taxonomy_discovery/semanticswingers.py @@ -33,7 +33,11 @@ from typing import Any, List, Optional import numpy as np -from sentence_transformers import SentenceTransformer +from sentence_transformers import SentenceTransformer, utils + +import torch +import torch.nn as nn +import networkx as nx from ...base import AutoLearner @@ -216,3 +220,147 @@ def _taxonomy_discovery(self, data: Any, test: bool = False) -> Optional[Any]: sim = emb @ emb.T use_llm = self.selector == "ollama" or (self.selector == "openai" and self.api_key) return self._select_llm(nodes, sim) if use_llm else self._select_embedding(nodes, sim) + + +class BilinearAdjacencyLayer(nn.Module): + """1024-D Structural Matrix Layer mapping Child -> Parent asymmetric space.""" + def __init__(self, embedding_dim: int = 1024): + super().__init__() + self.W = nn.Linear(embedding_dim, embedding_dim, bias=False) + + def forward(self, child: torch.Tensor, parent: torch.Tensor) -> torch.Tensor: + transformed_child = self.W(child) + return torch.sum(transformed_child * parent, dim=-1) + + +class SemanticSwingersMatrixTaxonomyLearner(SemanticSwingersTaxonomyLearner): + """Hybrid Structural Matrix + LLM Taxonomy Discovery Learner (Champion Approach).""" + + def __init__( + self, + embedding_model: str = "mixedbread-ai/mxbai-embed-large-v1", + matrix_weights_path: str = "structural_matrix_w_1024_mxbai.pt", + top_k: int = 10, + llm_threshold: int = 1000, + **kwargs + ) -> None: + # Pass backend/LLM args up to his constructor + super().__init__(embedding_model=embedding_model, top_k=top_k, **kwargs) + self.matrix_weights_path = matrix_weights_path + self.llm_threshold = llm_threshold + # Force PyTorch device detection + self.device = "mps" if torch.backends.mps.is_available() else ("cuda" if torch.cuda.is_available() else "cpu") + self._structural_layer = None + + def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: + """Override load to instantiate PyTorch tensors and the Matrix layer.""" + super().load(model_id, **kwargs) # Loads the standard SentenceTransformer + + embedding_dim = self._encoder.get_sentence_embedding_dimension() + self._structural_layer = BilinearAdjacencyLayer(embedding_dim=embedding_dim).to(self.device) + + if os.path.exists(self.matrix_weights_path): + self._structural_layer.load_state_dict(torch.load(self.matrix_weights_path, map_location=self.device)) + self._structural_layer.eval() + + def _cleanup_graph(self, raw_predictions: List[dict]) -> List[dict]: + """Enforces DAG rules via NetworkX cycle breaking and Transitive Reduction.""" + if not raw_predictions: return [] + G = nx.DiGraph() + for p in raw_predictions: G.add_edge(p["parent"], p["child"]) + + while not nx.is_directed_acyclic_graph(G): + try: + cycle = nx.find_cycle(G) + G.remove_edge(cycle[0][0], cycle[0][1]) + except nx.NetworkXNoCycle: + break + + G_reduced = nx.transitive_reduction(G) + return [{"parent": str(u), "child": str(v)} for u, v in G_reduced.edges()] + + def _taxonomy_discovery(self, data: Any, test: bool = False) -> Optional[Any]: + """Override the core logic to use the 1024-D Matrix and hybrid fallback.""" + if not test: + return None + if self._encoder is None or self._structural_layer is None: + self.load() + + nodes = self._nodes(data) # Reusing his method! + if not nodes: + return [] + + # 1. Generate Embeddings for the entire vocabulary + import logging + logger = logging.getLogger(__name__) + logger.info(f"Encoding {len(nodes)} concept nodes...") + + embeddings = self._encoder.encode(nodes, convert_to_tensor=True, show_progress_bar=False) + + raw_preds = [] + # Fallback to Pure Matrix if dataset > llm_threshold to prevent timeout + use_llm = (len(nodes) <= self.llm_threshold) and (self.selector in ["openai", "ollama"]) + + for child_idx, child in enumerate(nodes): + child_vec = embeddings[child_idx] + + # Step 1: Semantic Candidate Retrieval (Top 41 to safely exclude self) + cos_scores = util.cos_sim(child_vec, embeddings)[0] + top_semantic = torch.topk(cos_scores, k=min(41, len(nodes))) + + candidate_indices = [idx for idx in top_semantic[1].tolist() if idx != child_idx] + if not candidate_indices: + continue + + candidate_vecs = torch.stack([embeddings[idx] for idx in candidate_indices]) + candidates = [nodes[idx] for idx in candidate_indices] + + child_vec_expanded = child_vec.unsqueeze(0).repeat(candidate_vecs.size(0), 1) + + # Step 2: 1024-D Structural Matrix Scoring + with torch.no_grad(): + structural_scores = self._structural_layer(child_vec_expanded, candidate_vecs) + + # Step 3: Routing (LLM Verification vs. High-Speed Matrix Bypass) + if use_llm: + # Send top K structural candidates to LLM selector + ranked_indices = torch.argsort(structural_scores, descending=True)[:min(self.top_k, len(candidates))] + top_candidates = [candidates[idx] for idx in ranked_indices] + parent = self._select_llm_parent(child, top_candidates) + if parent: + raw_preds.append({"parent": parent, "child": child}) + else: + # Dataset is massive: take the #1 matrix candidate instantly + best_idx = torch.argmax(structural_scores).item() + raw_preds.append({"parent": candidates[best_idx], "child": child}) + + # Step 4: Clean the graph (Cycle-Breaking + Transitive Reduction) + return self._cleanup_graph(raw_preds) + + def _select_llm_parent(self, child: str, candidates: List[str]) -> Optional[str]: + """Helper method to ask the LLM to pick the best parent from the matrix's short-list.""" + from openai import OpenAI + from ontolearner.learner.taxonomy_discovery.semanticswingers import _reasoning_off + + api_key = self.api_key if self.selector == "openai" else (self.api_key or "ollama") + client = OpenAI(api_key=api_key, base_url=self.base_url) + + prompt = self.selection_prompt.format( + child=child, candidates="\n".join(f"- {c}" for c in candidates) + ) + try: + resp = client.chat.completions.create( + model=self.llm_model, + messages=[{"role": "user", "content": prompt}], + temperature=0, + max_tokens=self.max_tokens, + **_reasoning_off(self.llm_model) # Reuses his Qwen fix! + ) + answer = (resp.choices[0].message.content or "").strip() + for cand in candidates: + if cand.lower() in answer.lower(): + return cand + except Exception as e: + import logging + logging.getLogger(__name__).error(f"LLM selection failed for {child}: {e}") + return None From 088192dfc3e3a1d0fd8b778ba5ea9b7a046920ad Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Mon, 27 Jul 2026 13:08:28 +0200 Subject: [PATCH 21/23] feat(taxonomy): HF auto-download for structural matrix + fix util import & tests - SemanticSwingersMatrixTaxonomyLearner.load() now fetches the trained structural matrix from the public HF registry (datagero/taxonomy-structural-matrix-1024-mxbai) when no local copy exists, so the champion runs from a fresh clone (was: silent no-op if file missing). - Fix broken import: the module imported 'utils' (absent in sentence_transformers 5.x -> ImportError on import) while the Matrix path actually used 'util.cos_sim' (NameError at runtime) -> import 'util'; the champion path was never runnable. - Add 4 unit tests for the Matrix learner (bilinear scoring, DAG cleanup, HF-download fallback, default registry id); ruff clean, 63 tests pass. - Document the structural-matrix champion variant in the learner docs page. Co-authored-by: andy-symonds --- .../semanticswingers_learner.rst | 23 ++++++ .../taxonomy_discovery/semanticswingers.py | 38 ++++++--- ...est_semanticswingers_taxonomy_discovery.py | 80 +++++++++++++++++++ 3 files changed, 132 insertions(+), 9 deletions(-) diff --git a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst index caf86f14..8b5d544f 100644 --- a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst +++ b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst @@ -145,6 +145,29 @@ The learner runs on raw ontology objects, so pass ``ontologizer_data=False``. print(outputs["metrics"]) +Structural-matrix champion variant +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``SemanticSwingersMatrixTaxonomyLearner`` is the hybrid champion configuration. On +top of the retrieve-then-select base it adds a trained bilinear structural matrix +``W`` (1024-D, over ``mxbai-embed-large-v1`` embeddings) that scores candidate is-a +edges by direction, plus a DAG cleanup pass (cycle breaking + transitive reduction) +and a matrix-only high-speed bypass for very large type vocabularies +(``llm_threshold``). The matrix weights download automatically from the public +Hugging Face registry ``datagero/taxonomy-structural-matrix-1024-mxbai`` on first +``load()`` when no local copy is present, so it runs from a fresh clone. + +.. code-block:: python + + from ontolearner.learner.taxonomy_discovery import SemanticSwingersMatrixTaxonomyLearner + + learner = SemanticSwingersMatrixTaxonomyLearner( + embedding_model="mixedbread-ai/mxbai-embed-large-v1", + top_k=10, + llm_threshold=1000, # matrix-only bypass above this many types + selector="openai", # champion; "embedding"/"ollama" run offline + ) + Text2Onto + Taxonomy Discovery, joint (Task A, flagship) --------------------------------------------------------- diff --git a/ontolearner/learner/taxonomy_discovery/semanticswingers.py b/ontolearner/learner/taxonomy_discovery/semanticswingers.py index 0eb6ffd8..f5c963d8 100644 --- a/ontolearner/learner/taxonomy_discovery/semanticswingers.py +++ b/ontolearner/learner/taxonomy_discovery/semanticswingers.py @@ -33,7 +33,7 @@ from typing import Any, List, Optional import numpy as np -from sentence_transformers import SentenceTransformer, utils +from sentence_transformers import SentenceTransformer, util import torch import torch.nn as nn @@ -236,39 +236,59 @@ def forward(self, child: torch.Tensor, parent: torch.Tensor) -> torch.Tensor: class SemanticSwingersMatrixTaxonomyLearner(SemanticSwingersTaxonomyLearner): """Hybrid Structural Matrix + LLM Taxonomy Discovery Learner (Champion Approach).""" + #: Public HuggingFace repo holding the trained structural matrix, so the + #: weights auto-download when no local copy is present (reviewers can run the + #: champion straight from a fresh clone). + _HF_REPO_ID = "datagero/taxonomy-structural-matrix-1024-mxbai" + def __init__( self, embedding_model: str = "mixedbread-ai/mxbai-embed-large-v1", matrix_weights_path: str = "structural_matrix_w_1024_mxbai.pt", top_k: int = 10, llm_threshold: int = 1000, + hf_repo_id: Optional[str] = None, **kwargs ) -> None: # Pass backend/LLM args up to his constructor super().__init__(embedding_model=embedding_model, top_k=top_k, **kwargs) self.matrix_weights_path = matrix_weights_path + self.hf_repo_id = hf_repo_id or self._HF_REPO_ID self.llm_threshold = llm_threshold # Force PyTorch device detection self.device = "mps" if torch.backends.mps.is_available() else ("cuda" if torch.cuda.is_available() else "cpu") self._structural_layer = None + def _resolve_matrix_weights(self) -> str: + """Return a local path to the matrix weights, downloading from the HF + registry on first use when no local file exists.""" + if os.path.exists(self.matrix_weights_path): + return self.matrix_weights_path + from huggingface_hub import hf_hub_download + return hf_hub_download( + repo_id=self.hf_repo_id, + filename=os.path.basename(self.matrix_weights_path), + ) + def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: """Override load to instantiate PyTorch tensors and the Matrix layer.""" super().load(model_id, **kwargs) # Loads the standard SentenceTransformer - + embedding_dim = self._encoder.get_sentence_embedding_dimension() self._structural_layer = BilinearAdjacencyLayer(embedding_dim=embedding_dim).to(self.device) - - if os.path.exists(self.matrix_weights_path): - self._structural_layer.load_state_dict(torch.load(self.matrix_weights_path, map_location=self.device)) - self._structural_layer.eval() + + weights_path = self._resolve_matrix_weights() + self._structural_layer.load_state_dict(torch.load(weights_path, map_location=self.device)) + self._structural_layer.eval() def _cleanup_graph(self, raw_predictions: List[dict]) -> List[dict]: """Enforces DAG rules via NetworkX cycle breaking and Transitive Reduction.""" - if not raw_predictions: return [] + if not raw_predictions: + return [] G = nx.DiGraph() - for p in raw_predictions: G.add_edge(p["parent"], p["child"]) - + for p in raw_predictions: + G.add_edge(p["parent"], p["child"]) + while not nx.is_directed_acyclic_graph(G): try: cycle = nx.find_cycle(G) diff --git a/tests/test_semanticswingers_taxonomy_discovery.py b/tests/test_semanticswingers_taxonomy_discovery.py index 3529a507..bbf8291e 100644 --- a/tests/test_semanticswingers_taxonomy_discovery.py +++ b/tests/test_semanticswingers_taxonomy_discovery.py @@ -177,3 +177,83 @@ def test_selection_prompt_is_injectable_and_defaults(): custom = SemanticSwingersTaxonomyLearner(selection_prompt="parent of {child}: {candidates}") out = custom.selection_prompt.format(child="Chianti", candidates="- Wine") assert out == "parent of Chianti: - Wine" + + +# --- Structural-matrix champion (SemanticSwingersMatrixTaxonomyLearner) -------- + +def test_bilinear_layer_scores_child_parent_pairs(): + import torch + from ontolearner.learner.taxonomy_discovery.semanticswingers import ( + BilinearAdjacencyLayer, + ) + + layer = BilinearAdjacencyLayer(embedding_dim=4) + child = torch.randn(3, 4) + parent = torch.randn(3, 4) + scores = layer(child, parent) + + # one score per (child, parent) row, and it equals + assert scores.shape == (3,) + expected = torch.sum(layer.W(child) * parent, dim=-1) + assert torch.allclose(scores, expected) + + +def test_cleanup_graph_breaks_cycles_and_transitively_reduces(): + from ontolearner.learner.taxonomy_discovery.semanticswingers import ( + SemanticSwingersMatrixTaxonomyLearner, + ) + + learner = SemanticSwingersMatrixTaxonomyLearner() + + # A->B, B->C, A->C : the redundant A->C must be dropped by transitive reduction. + reduced = learner._cleanup_graph([ + {"parent": "A", "child": "B"}, + {"parent": "B", "child": "C"}, + {"parent": "A", "child": "C"}, + ]) + edges = {(e["parent"], e["child"]) for e in reduced} + assert ("A", "C") not in edges + assert ("A", "B") in edges and ("B", "C") in edges + + # A->B, B->A : a cycle must be broken so the result is a DAG. + import networkx as nx + deacycled = learner._cleanup_graph([ + {"parent": "A", "child": "B"}, + {"parent": "B", "child": "A"}, + ]) + G = nx.DiGraph((e["parent"], e["child"]) for e in deacycled) + assert nx.is_directed_acyclic_graph(G) + + assert learner._cleanup_graph([]) == [] + + +def test_matrix_learner_defaults_to_public_hf_registry(): + from ontolearner.learner.taxonomy_discovery.semanticswingers import ( + SemanticSwingersMatrixTaxonomyLearner, + ) + + learner = SemanticSwingersMatrixTaxonomyLearner() + assert learner.hf_repo_id == "datagero/taxonomy-structural-matrix-1024-mxbai" + + +def test_resolve_matrix_weights_prefers_local_then_falls_back_to_hf(): + from ontolearner.learner.taxonomy_discovery.semanticswingers import ( + SemanticSwingersMatrixTaxonomyLearner, + ) + + learner = SemanticSwingersMatrixTaxonomyLearner( + matrix_weights_path="structural_matrix_w_1024_mxbai.pt", + ) + + # Local file present -> used as-is, no download. + with patch(f"{MODULE}.os.path.exists", return_value=True): + assert learner._resolve_matrix_weights() == "structural_matrix_w_1024_mxbai.pt" + + # Local file absent -> fetched from the HF registry by basename. + with patch(f"{MODULE}.os.path.exists", return_value=False), \ + patch("huggingface_hub.hf_hub_download", return_value="/cache/x.pt") as dl: + assert learner._resolve_matrix_weights() == "/cache/x.pt" + dl.assert_called_once_with( + repo_id="datagero/taxonomy-structural-matrix-1024-mxbai", + filename="structural_matrix_w_1024_mxbai.pt", + ) From d7c0f02798f828c1013790e6877f52e8d2dd5883 Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Mon, 27 Jul 2026 13:50:35 +0200 Subject: [PATCH 22/23] docs(taxonomy): frame structural-matrix learner as scalable alternative, not champion Aligns the docs with the paper + team framing: the primary Task C pipeline is the LLM selector (SemanticSwingersTaxonomyLearner); the structural-matrix learner is a scale-aware alternative for very large ontologies, not the champion. --- .../llms4ol_challenge/semanticswingers_learner.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst index 8b5d544f..421919d6 100644 --- a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst +++ b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst @@ -145,10 +145,11 @@ The learner runs on raw ontology objects, so pass ``ontologizer_data=False``. print(outputs["metrics"]) -Structural-matrix champion variant -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Scale-aware structural-matrix variant +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``SemanticSwingersMatrixTaxonomyLearner`` is the hybrid champion configuration. On +``SemanticSwingersMatrixTaxonomyLearner`` is a scalable alternative to the primary +LLM pipeline above, aimed at very large ontologies. On top of the retrieve-then-select base it adds a trained bilinear structural matrix ``W`` (1024-D, over ``mxbai-embed-large-v1`` embeddings) that scores candidate is-a edges by direction, plus a DAG cleanup pass (cycle breaking + transitive reduction) From b6ee297faa4ed78b55c2b6ddf1be97ff667c950b Mon Sep 17 00:00:00 2001 From: Matias Vizcaino Date: Mon, 27 Jul 2026 13:57:04 +0200 Subject: [PATCH 23/23] docs(taxonomy): reframe MatrixTaxonomyLearner docstring as scalable alternative Align the class docstring (and the HF-repo comment) with the paper + PR framing: the matrix learner is a scale-aware alternative to the primary LLM pipeline, not the champion. --- .../learner/taxonomy_discovery/semanticswingers.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ontolearner/learner/taxonomy_discovery/semanticswingers.py b/ontolearner/learner/taxonomy_discovery/semanticswingers.py index f5c963d8..2adf34c1 100644 --- a/ontolearner/learner/taxonomy_discovery/semanticswingers.py +++ b/ontolearner/learner/taxonomy_discovery/semanticswingers.py @@ -234,11 +234,17 @@ def forward(self, child: torch.Tensor, parent: torch.Tensor) -> torch.Tensor: class SemanticSwingersMatrixTaxonomyLearner(SemanticSwingersTaxonomyLearner): - """Hybrid Structural Matrix + LLM Taxonomy Discovery Learner (Champion Approach).""" + """Hybrid Structural Matrix + LLM Taxonomy Discovery Learner. + + A scalable alternative to the primary LLM pipeline + (:class:`SemanticSwingersTaxonomyLearner`): a trained bilinear structural matrix + scores candidate is-a edges so per-node LLM inference can be bypassed on very + large ontologies (see ``llm_threshold``), with DAG cleanup on the output. + """ #: Public HuggingFace repo holding the trained structural matrix, so the - #: weights auto-download when no local copy is present (reviewers can run the - #: champion straight from a fresh clone). + #: weights auto-download when no local copy is present (reviewers can run this + #: learner straight from a fresh clone). _HF_REPO_ID = "datagero/taxonomy-structural-matrix-1024-mxbai" def __init__(