From 0e830519dabb6c10e7cfbe02e7af018fa90d41dc Mon Sep 17 00:00:00 2001 From: Bindu Gajria Date: Mon, 20 Jul 2026 16:25:57 -0400 Subject: [PATCH 1/2] Skip contact and model-reference validation, and the dataset-injector execution that validation triggers for any DatasetPresenter not found in apidb.Datasource table. Earlier getContacts and getModelReferences were running unconditionally for every presenter in the presenters XML directory, regardless of whether that dataset was actually loaded into the specific database instance being processed, leading to error when Presenter xml has extra entries --- .../datasetPresenter/DatasetPresenterSetLoader.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSetLoader.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSetLoader.java index d8b4ee71..e90e884c 100644 --- a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSetLoader.java +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSetLoader.java @@ -92,11 +92,13 @@ Set syncPresenterSetWithDatasourceTable() { for (DatasetPresenter datasetPresenter : dps.getDatasetPresenters().values()) { getPresenterValuesFromDatasourceTable(datasetPresenter, datasourceTableStmt, datasetNamesFoundInDb); - if (!datasetPresenter.getFoundInDb()) - presenterNamesNotInDb.add(datasetPresenter.getDatasetName()); - datasetPresenter.getContacts(allContacts); // validate contacts - datasetPresenter.getModelReferences(); // validate model references + if (datasetPresenter.getFoundInDb()) { + datasetPresenter.getContacts(allContacts); // validate contacts + datasetPresenter.getModelReferences(); // validate model references (runs dataset injectors) + } else { + presenterNamesNotInDb.add(datasetPresenter.getDatasetName()); + } } if (presenterNamesNotInDb.size() != 0) { From 4dccc1849280d3da2adf84a5c02b319709262b61 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Wed, 29 Jul 2026 23:42:14 -0400 Subject: [PATCH 2/2] Add an optional dataset gate to presenter template injection datasetPresenters/*.xml is deliberately a superset of what any one instance has loaded, but injection ignored that: it built a DatasetInjector for every presenter it parsed, so a presenter whose dataset is absent failed the build. Seen building PlasmoDB against the unidb_shu_a subset: Error: A datasetInjector for class ...ClonedInsertEnds in DatasetPresenter null is missing the required property sourceIdField Five of the ten ClonedInsertEnds presenters have no ApiCommonDatasets declaration on this branch, so they get no prop file and no sourceIdField. Commit 12897924 added the same kind of gate to DatasetPresenterSetLoader, but that keys on getFoundInDb() -- a fact from a different lifecycle phase (presenterWriteToDb) that TemplatesInjector cannot have, since its only inputs are directories. That gate stays; it covers the DB-aware phase. This covers the build phase. The predicate becomes a LoadedDatasetSource, consulted once in DatasetPresenterSet.createFromPresentersDir so every consumer -- dataset injectors, contact validation, model references, the loader, anything added later -- inherits it without knowing it exists. That is what the previous fix lacked, and why it landed in only one of two paths. Off by default (AllDatasetsLoaded): no database call, injection identical to before. A developer opts in with a system property: -Dpresenter.dataset.gate=on # appDb from model-config.xml -Dpresenter.dataset.gate=jdbc:postgresql://host:5432/db # or name one explicitly -Dpresenter.dataset.gate.always=a_RSRC,b_RSRC # inject these regardless A property rather than an environment variable because the two build paths do not share an environment and only a property reaches both: rebuilder discards the environment on startup but parses --gusjvmopts afterwards, while wb inherits GUSJVMOPTS from the site's etc/setenv. Both exec the same installed presenterInjectTemplates, which now forwards GUSJVMOPTS to the JVM -- previously that variable was set by rebuilder and consumed by nothing. Matching mirrors the loader's "ds.NAME like ?" exactly, including '_' as a single-character wildcard, so the two gates can never disagree about membership. A presenter's datasetNamePattern is resolved the same way the loader resolves it, and one match is enough since a pattern presenter fans out to many datasets. Enabled but unable to read apidb.Datasource is a hard failure: degrading to "inject everything" would reproduce the failure this exists to prevent. Skipped presenters are always reported -- count to stderr, names to gus_home/lib/wdk/presentersNotLoaded.txt, because a gate that drops presenters silently turns a loud build failure into a website that builds green with searches missing. Verified on cedar against unidb_shu_a: off => byte-identical pre-change behaviour; on => 220 datasets read, 3171 presenters skipped, rebuilder reaches SUCCESS and deploys. 21 unit tests. Co-Authored-By: Claude Opus 5 --- DatasetPresenter/bin/presenterInjectTemplates | 9 +- .../datasetPresenter/AllDatasetsLoaded.java | 19 ++ .../AlwaysLoadedDatasets.java | 53 +++++ .../datasetPresenter/DatasetPresenterSet.java | 106 +++++++++- .../DatasourceTableDatasets.java | 187 ++++++++++++++++++ .../datasetPresenter/LikePattern.java | 58 ++++++ .../datasetPresenter/LoadedDatasetSource.java | 31 +++ .../LoadedDatasetSources.java | 96 +++++++++ .../datasetPresenter/TestDatasetGate.java | 145 ++++++++++++++ .../datasetPresenter/TestLikePattern.java | 153 ++++++++++++++ 10 files changed, 855 insertions(+), 2 deletions(-) create mode 100644 DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AllDatasetsLoaded.java create mode 100644 DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AlwaysLoadedDatasets.java create mode 100644 DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasourceTableDatasets.java create mode 100644 DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LikePattern.java create mode 100644 DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSource.java create mode 100644 DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSources.java create mode 100644 DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestDatasetGate.java create mode 100644 DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestLikePattern.java diff --git a/DatasetPresenter/bin/presenterInjectTemplates b/DatasetPresenter/bin/presenterInjectTemplates index b1d1e3e4..94744e6a 100644 --- a/DatasetPresenter/bin/presenterInjectTemplates +++ b/DatasetPresenter/bin/presenterInjectTemplates @@ -8,7 +8,14 @@ my $CLASSPATH = &FgpUtil::Util::CommandHelper::getJavaClasspath($GUS_HOME); my $sysProps = &FgpUtil::Util::CommandHelper::getSystemProps($GUS_HOME, 'presenterInjectTemplates'); my $args = &FgpUtil::Util::CommandHelper::getJavaArgs(@ARGV); -my $cmd = "java $sysProps -classpath $CLASSPATH org.apidb.apicommon.datasetPresenter.TemplatesInjector $args"; +# Forward GUSJVMOPTS so a caller can pass -D properties to this command, which is how the +# dataset gate is configured (-Dpresenter.dataset.gate=...). Two callers, two routes: +# rebuilder --gusjvmopts '-D...' parsed after rebuilder discards the environment, so its +# export survives into the build +# wb inherits GUSJVMOPTS from etc/setenv; wb does not scrub +my $jvmOpts = $ENV{GUSJVMOPTS} || ''; + +my $cmd = "java $jvmOpts $sysProps -classpath $CLASSPATH org.apidb.apicommon.datasetPresenter.TemplatesInjector $args"; system($cmd); diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AllDatasetsLoaded.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AllDatasetsLoaded.java new file mode 100644 index 00000000..781db506 --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AllDatasetsLoaded.java @@ -0,0 +1,19 @@ +package org.apidb.apicommon.datasetPresenter; + +/** + * The default {@link LoadedDatasetSource}: every presenter is treated as loaded and no + * database is contacted. This is the behavior the build has always had, and it is what + * runs unless a developer explicitly opts in to a dataset gate. + */ +public class AllDatasetsLoaded implements LoadedDatasetSource { + + @Override + public boolean isLoaded(String nameOrPattern) { + return true; + } + + @Override + public String describe() { + return "no dataset gate: every DatasetPresenter will be injected"; + } +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AlwaysLoadedDatasets.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AlwaysLoadedDatasets.java new file mode 100644 index 00000000..754d9491 --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AlwaysLoadedDatasets.java @@ -0,0 +1,53 @@ +package org.apidb.apicommon.datasetPresenter; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * A {@link LoadedDatasetSource} that treats a named set of datasets as loaded regardless of + * what the delegate says, and otherwise defers to it. + * + * Needed because some DatasetPresenters deliberately contribute only attributes, leaving + * their questions hardcoded in the anchor file — the custom expression presenters do exactly + * this, with a "setting these to false so we don't inject questions" comment. Skipping such a + * presenter leaves the hardcoded question referencing an attribute nobody generated, which + * fails model load. Excepting the dataset runs its injectors and supplies the attribute. + * + * Use sparingly: an exception asserts "inject this presenter even though the data is absent", + * so any template it injects that queries missing tables will fail later, in a less obvious + * place. It is the right tool only when the presenter's output is metadata rather than data. + */ +public class AlwaysLoadedDatasets implements LoadedDatasetSource { + + private final LoadedDatasetSource _delegate; + private final Set _alwaysLoaded; + + AlwaysLoadedDatasets(LoadedDatasetSource delegate, Set alwaysLoaded) { + _delegate = delegate; + _alwaysLoaded = Collections.unmodifiableSet(new LinkedHashSet(alwaysLoaded)); + } + + @Override + public boolean isLoaded(String nameOrPattern) { + if (nameOrPattern == null) return false; + + if (_alwaysLoaded.contains(nameOrPattern)) return true; + + // a pattern presenter counts as excepted when the pattern covers an excepted dataset + if (nameOrPattern.indexOf('%') >= 0) { + LikePattern pattern = new LikePattern(nameOrPattern); + for (String excepted : _alwaysLoaded) { + if (pattern.matches(excepted)) return true; + } + } + + return _delegate.isLoaded(nameOrPattern); + } + + @Override + public String describe() { + return _delegate.describe() + "; " + _alwaysLoaded.size() + + " dataset(s) always injected: " + String.join(", ", _alwaysLoaded); + } +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSet.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSet.java index 7aba5325..f1583cf4 100644 --- a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSet.java +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSet.java @@ -1,10 +1,14 @@ package org.apidb.apicommon.datasetPresenter; import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; /** @@ -26,6 +30,7 @@ public class DatasetPresenterSet { private Map> _propertiesFromFiles = new HashMap>(); private Set _duplicateDatasetNames = new HashSet(); + private Set _presentersNotLoaded = new LinkedHashSet(); /** * Add a DatasetPresenter to this set. @@ -175,6 +180,93 @@ void handleOverrides() { } } + /** + * Drop presenters whose dataset this instance has not loaded, according to the supplied + * source. The presenters XML directory is a legitimate superset of any one instance, so + * these are not errors — but they must go before anything walks the set, because + * constructing a DatasetInjector for an absent dataset fails on the props the dataset + * would have supplied. + * + * Filtering here, once, rather than at each point of use is deliberate: every consumer + * (dataset injectors, contact validation, model references, the loader) then inherits + * the gate without knowing it exists, and a consumer added later cannot forget it. + * + * Skipped presenters are always reported. A gate that dropped them silently would turn + * a loud build failure into a website that builds green with searches missing. + */ + void retainLoadedDatasets(LoadedDatasetSource loadedDatasets) { + Iterator> presenters = _presenters.entrySet().iterator(); + while (presenters.hasNext()) { + DatasetPresenter presenter = presenters.next().getValue(); + + // a presenter carries either a name or a datasetNamePattern; resolve the same way + // DatasetPresenterSetLoader.getPresenterValuesFromDatasourceTable does + String pattern = presenter.getDatasetNamePattern(); + String nameOrPattern = pattern == null ? presenter.getDatasetName() : pattern; + + if (!loadedDatasets.isLoaded(nameOrPattern)) { + _presentersNotLoaded.add(nameOrPattern); + presenters.remove(); + if (pattern != null) _namePatterns.remove(pattern); + } + } + + } + + /** + * @return where to list the skipped presenters, or null if GUS_HOME is not set + */ + static File skippedReportFile() { + String gusHome = System.getenv("GUS_HOME"); + return gusHome == null ? null : new File(gusHome + "/lib/wdk/presentersNotLoaded.txt"); + } + + /** + * Report the skipped presenters: the count on stderr, the names in a file. + * + * The count goes in the build log because a silent gate is worse than no gate — a build + * that quietly drops presenters looks successful while the website loses searches. The + * names go to a file because there can be thousands of them, and burying the rest of the + * build log to list them defeats the purpose of reporting at all. + */ + void reportPresentersNotLoaded(File reportFile) { + if (_presentersNotLoaded.isEmpty()) return; + + String summary = "Dataset gate: skipping " + _presentersNotLoaded.size() + + " DatasetPresenter(s) whose dataset is not loaded in this instance"; + + if (reportFile == null) { + // no GUS_HOME to write under; the names are all we can offer + System.err.println(summary + ":" + System.lineSeparator() + " " + + String.join(System.lineSeparator() + " ", _presentersNotLoaded)); + return; + } + + try { + File parent = reportFile.getParentFile(); + if (parent != null) parent.mkdirs(); + try (PrintWriter out = new PrintWriter(reportFile)) { + out.println("# DatasetPresenters skipped because their dataset is not in this"); + out.println("# instance's apidb.Datasource. Presenters are a superset by design;"); + out.println("# these are not errors."); + for (String name : _presentersNotLoaded) { + out.println(name); + } + } + System.err.println(summary + "; names listed in " + reportFile); + } + catch (IOException e) { + // reporting must not break the build, but it must not disappear either + System.err.println(summary + " (could not write " + reportFile + ": " + e.getMessage() + + "):" + System.lineSeparator() + " " + + String.join(System.lineSeparator() + " ", _presentersNotLoaded)); + } + } + + Set getPresentersNotLoaded() { + return Collections.unmodifiableSet(_presentersNotLoaded); + } + void addPropertiesFromFiles(Map> datasetNamesToProperties, Set duplicateDatasetNames) { for (DatasetPresenter datasetPresenter : _presenters.values()) { datasetPresenter.addPropertiesFromFile(datasetNamesToProperties, duplicateDatasetNames); @@ -200,6 +292,12 @@ void addIdentifierProperty() { // //////////////////// Static methods ////////////////// static DatasetPresenterSet createFromPresentersDir(String presentersDir, String globalXmlFile) { + return createFromPresentersDir(presentersDir, globalXmlFile, + LoadedDatasetSources.fromSiteConfig()); + } + + static DatasetPresenterSet createFromPresentersDir(String presentersDir, String globalXmlFile, + LoadedDatasetSource loadedDatasets) { File pres = new File(presentersDir); if (!pres.isDirectory()) throw new UserException("Presenters dir " + presentersDir @@ -208,7 +306,13 @@ static DatasetPresenterSet createFromPresentersDir(String presentersDir, String // get the presenters into memory DatasetPresenterParser dpp = new DatasetPresenterParser(); DatasetPresenterSet dps = dpp.parseDir(presentersDir, globalXmlFile); - + + // drop presenters for datasets this instance has not loaded, before anything walks + // the set. Off by default, so a normal build is unchanged. + System.err.println(loadedDatasets.describe()); + dps.retainLoadedDatasets(loadedDatasets); + dps.reportPresentersNotLoaded(skippedReportFile()); + // add properties from dataset prop files to presenters DatasetPropertiesParser propParser = new DatasetPropertiesParser(); diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasourceTableDatasets.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasourceTableDatasets.java new file mode 100644 index 00000000..bac3701b --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasourceTableDatasets.java @@ -0,0 +1,187 @@ +package org.apidb.apicommon.datasetPresenter; + +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +/** + * A {@link LoadedDatasetSource} backed by apidb.Datasource, which is what the workflow's + * root graph actually loaded. Opt-in only — see {@link LoadedDatasetSources}. + * + * Every dataset name is read once up front and matched in memory, rather than issuing a + * LIKE query per presenter, because presenters number in the thousands. + * + * Connection details come from the site's own model-config.xml (the database the model + * will run against), following the same precedence as WDK's ModelConfigDB: an explicit + * connectionUrl wins, then host + database name. LDAP resolution is not reimplemented + * here; a site configured only for LDAP should pass an explicit JDBC URL instead. + */ +public class DatasourceTableDatasets implements LoadedDatasetSource { + + private static final String DATASOURCE_SQL = "select name from apidb.datasource"; + + private final Set _loadedDatasetNames; + private final String _description; + + /** + * @param jdbcUrlOverride an explicit JDBC URL, or null to take one from model-config.xml + */ + DatasourceTableDatasets(String jdbcUrlOverride) { + File modelConfig = findModelConfig(); + Element appDb = parseAppDbElement(modelConfig); + + String url = jdbcUrlOverride != null ? jdbcUrlOverride : connectionUrlFrom(appDb, modelConfig); + String login = required(appDb, "login", modelConfig); + String password = required(appDb, "password", modelConfig); + + _loadedDatasetNames = readDatasetNames(url, login, password); + _description = "dataset gate on " + url + " as " + login + ": " + + _loadedDatasetNames.size() + " datasets in apidb.Datasource"; + } + + @Override + public boolean isLoaded(String nameOrPattern) { + if (nameOrPattern == null) return false; + + // Fast path: an exact hit is a match under LIKE semantics too, and most presenters + // carry a plain name. Only fall back to a scan when there is no exact match. + if (_loadedDatasetNames.contains(nameOrPattern)) return true; + + LikePattern pattern = new LikePattern(nameOrPattern); + for (String loadedName : _loadedDatasetNames) { + if (pattern.matches(loadedName)) return true; + } + return false; + } + + @Override + public String describe() { + return _description; + } + + private static Set readDatasetNames(String url, String login, String password) { + Set names = new HashSet(); + // A gate that was explicitly asked for must not degrade into "inject everything" when + // the database is unreachable; that would reproduce the failure it exists to prevent. + try (Connection connection = DriverManager.getConnection(url, login, password); + Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery(DATASOURCE_SQL)) { + while (rs.next()) { + names.add(rs.getString(1)); + } + } + catch (SQLException e) { + throw new UserException("Dataset gate is enabled but apidb.Datasource could not be" + + " read from " + url + " as " + login + ": " + e.getMessage(), e); + } + if (names.isEmpty()) + throw new UserException("Dataset gate is enabled but apidb.Datasource at " + url + + " holds no datasets; refusing to skip every DatasetPresenter"); + return names; + } + + /** + * Locate the site's model-config.xml under $GUS_HOME/config. + * + * Found by looking for the file rather than by composing a path from the project name, + * because the environment is not a dependable source here: rebuilder discards everything + * but a short whitelist, so WDK_MODEL is absent under rebuilder even though it is set for + * wb. WDK_MODEL and PROJECT are consulted only to disambiguate a gus_home that somehow + * configures more than one project. + */ + private static File findModelConfig() { + String gusHome = System.getenv("GUS_HOME"); + if (gusHome == null) + throw new UserException("Dataset gate is enabled but GUS_HOME is not set; it supplies" + + " the location of model-config.xml"); + + File configDir = new File(gusHome, "config"); + for (String hint : new String[] { System.getenv("WDK_MODEL"), System.getenv("PROJECT") }) { + if (hint == null || hint.trim().isEmpty()) continue; + File hinted = new File(configDir, hint.trim() + "/model-config.xml"); + if (hinted.isFile()) return hinted; + } + + List found = new ArrayList(); + File[] projectDirs = configDir.listFiles(); + if (projectDirs != null) { + for (File projectDir : projectDirs) { + File modelConfig = new File(projectDir, "model-config.xml"); + if (modelConfig.isFile()) found.add(modelConfig); + } + } + + if (found.size() == 1) return found.get(0); + + if (found.isEmpty()) + throw new UserException("Dataset gate is enabled but no model-config.xml was found" + + " under " + configDir + ". It is written by 'conifer configure', which rebuilder" + + " runs after this build step, so a from-scratch rebuilder cannot use the gate on" + + " its first pass."); + + throw new UserException("Dataset gate is enabled but " + configDir + " holds " + + found.size() + " model-config.xml files (" + found + "); set WDK_MODEL to name the" + + " project, or give the gate an explicit jdbc: URL"); + } + + private static Element parseAppDbElement(File modelConfig) { + try { + DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + Document document = builder.parse(modelConfig); + NodeList appDbNodes = document.getElementsByTagName("appDb"); + if (appDbNodes.getLength() == 0) + throw new UserException("No element in " + modelConfig); + return (Element) appDbNodes.item(0); + } + catch (UserException e) { + throw e; + } + catch (Exception e) { + throw new UserException("Could not read " + modelConfig + ": " + e.getMessage(), e); + } + } + + private static String connectionUrlFrom(Element appDb, File modelConfig) { + String connectionUrl = attribute(appDb, "connectionUrl"); + if (connectionUrl != null) return connectionUrl; + + String host = attribute(appDb, "dbHost"); + String dbName = attribute(appDb, "dbIdentifier"); + if (host != null && dbName != null) { + String port = attribute(appDb, "dbPort"); + return "jdbc:postgresql://" + host + (port == null ? "" : ":" + port) + "/" + dbName; + } + + throw new UserException("The in " + modelConfig + " supplies neither a" + + " connectionUrl nor dbHost + dbIdentifier. It is presumably configured for LDAP" + + " lookup, which the dataset gate does not perform; enable the gate with an" + + " explicit JDBC URL instead of 'on'."); + } + + private static String required(Element appDb, String name, File modelConfig) { + String value = attribute(appDb, name); + if (value == null) + throw new UserException("The in " + modelConfig + " has no " + name + + " attribute, which the dataset gate needs to connect"); + return value; + } + + private static String attribute(Element element, String name) { + String value = element.getAttribute(name); + return value == null || value.trim().isEmpty() ? null : value.trim(); + } +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LikePattern.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LikePattern.java new file mode 100644 index 00000000..b8bfdc11 --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LikePattern.java @@ -0,0 +1,58 @@ +package org.apidb.apicommon.datasetPresenter; + +import java.util.regex.Pattern; + +/** + * A SQL LIKE pattern, evaluated in memory. + * + * Deliberately mirrors the semantics of the "ds.NAME like ?" query in + * {@link DatasetPresenterSetLoader#getDatasourceTableStmt}, so that a dataset gate + * applied at build time can never disagree with the database about whether a + * DatasetPresenter matches a dataset: '%' spans any run of characters, '_' matches + * exactly one, and every other character is literal. + * + * Note that '_' remains a single-character wildcard even though dataset names contain + * literal underscores throughout. That is looser than most authors intend, but it is + * what the database does; tightening it here in isolation would let injection skip a + * presenter the loader matched. + */ +public class LikePattern { + + private final String _like; + private final Pattern _regex; + + LikePattern(String likePattern) { + _like = likePattern; + _regex = Pattern.compile(toRegex(likePattern)); + } + + /** + * Translate a SQL LIKE pattern into an equivalent regular expression. + */ + static String toRegex(String likePattern) { + StringBuilder regex = new StringBuilder(); + for (char c : likePattern.toCharArray()) { + switch (c) { + case '%': + regex.append(".*"); + break; + case '_': + regex.append('.'); + break; + default: + if ("\\.[]{}()*+-?^$|".indexOf(c) >= 0) regex.append('\\'); + regex.append(c); + } + } + return regex.toString(); + } + + boolean matches(String datasetName) { + return datasetName != null && _regex.matcher(datasetName).matches(); + } + + @Override + public String toString() { + return _like; + } +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSource.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSource.java new file mode 100644 index 00000000..5071120a --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSource.java @@ -0,0 +1,31 @@ +package org.apidb.apicommon.datasetPresenter; + +/** + * Answers "does this instance have the dataset a DatasetPresenter presents?". + * + * The presenters XML directory is deliberately a superset of what any one instance has + * loaded, so presenters for absent datasets must be skipped rather than injected. What + * counts as loaded is not knowable from the presenters or the dataset XML alone — the + * workflow's root graph decides, and apidb.Datasource is its output — so the answer is + * supplied by an implementation of this interface rather than computed inline. + * + * The default implementation, {@link AllDatasetsLoaded}, answers yes to everything and + * touches no database, which is required: the website build cannot make database calls in + * normal operation. A developer building against a partially loaded database opts in to + * {@link DatasourceTableDatasets} instead; see {@link LoadedDatasetSources}. + */ +public interface LoadedDatasetSource { + + /** + * @param nameOrPattern a DatasetPresenter's datasetNamePattern if it has one, else its + * dataset name. Interpreted with SQL LIKE semantics either way, matching how + * {@link DatasetPresenterSetLoader} resolves the same choice. + * @return whether at least one loaded dataset matches + */ + boolean isLoaded(String nameOrPattern); + + /** + * @return a short description of this source, for the build log + */ + String describe(); +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSources.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSources.java new file mode 100644 index 00000000..af02d948 --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSources.java @@ -0,0 +1,96 @@ +package org.apidb.apicommon.datasetPresenter; + +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Chooses the {@link LoadedDatasetSource} for this build. + * + * The gate is off unless a developer turns it on, so a normal build makes no database calls + * and injects exactly the presenters it always did. It is turned on with a JVM system + * property: + * + * 
+ *   -Dpresenter.dataset.gate=on                                # appDb from model-config.xml
+ *   -Dpresenter.dataset.gate=jdbc:postgresql://host:5432/mydb  # or name one explicitly
+ *   -Dpresenter.dataset.gate=off                               # (the default)
+ *
+ *   -Dpresenter.dataset.gate.always=ds_one_RSRC,ds_two_RSRC    # optional exceptions
+ *
+ * + * The exceptions list injects those presenters even though their data is absent, for + * presenters that deliberately contribute only attributes while their questions stay + * hardcoded in an anchor file; see {@link AlwaysLoadedDatasets}. + * + * A system property rather than an environment variable because the two ways a site is built + * do not share an environment, and only a property reaches both: + * + *
    + *
  • rebuilder discards the environment on startup (keeping only PWD, USER, LANG, + * SUDO_USER, HOME, SSH_AUTH_SOCK and GITHUB_*), but its --gusjvmopts flag is parsed + * afterwards, so rebuilder --gusjvmopts '-Dpresenter.dataset.gate=on' arrives + * intact.
    • + *
  • wb does not scrub, so GUSJVMOPTS exported from the site's etc/setenv reaches it.
    • + *
+ * + * Either way the value lands here via GUSJVMOPTS, which the presenterInjectTemplates wrapper + * forwards to the JVM. An environment variable read directly by this class was tried first: + * it worked under wb and silently did nothing under rebuilder, which is exactly the class of + * failure this gate exists to prevent. + * + * Credentials are deliberately not passed here: a command line is world-readable in ps + * output, so the appDb login and password come from model-config.xml. + */ +public class LoadedDatasetSources { + + public static final String GATE_PROPERTY = "presenter.dataset.gate"; + public static final String ALWAYS_PROPERTY = "presenter.dataset.gate.always"; + + private LoadedDatasetSources() {} + + static LoadedDatasetSource fromSiteConfig() { + return forProperties(System.getProperty(GATE_PROPERTY), System.getProperty(ALWAYS_PROPERTY)); + } + + /** + * @param gate value of presenter.dataset.gate + * @param always comma- or whitespace-separated dataset names to inject regardless + */ + static LoadedDatasetSource forProperties(String gate, String always) { + LoadedDatasetSource source = forSetting(gate); + + Set alwaysLoaded = parseAlways(always); + return alwaysLoaded.isEmpty() ? source : new AlwaysLoadedDatasets(source, alwaysLoaded); + } + + static Set parseAlways(String always) { + Set names = new LinkedHashSet(); + if (always == null) return names; + + for (String name : always.split("[,\\s]+")) { + if (!name.trim().isEmpty()) names.add(name.trim()); + } + return names; + } + + /** + * @param setting null, empty, "off" or "false" for no gate; "on" or "true" to gate on the + * appDb named in model-config.xml; or an explicit "jdbc:..." URL + */ + static LoadedDatasetSource forSetting(String setting) { + if (setting == null || setting.trim().isEmpty()) return new AllDatasetsLoaded(); + + String value = setting.trim(); + if (value.equalsIgnoreCase("off") || value.equalsIgnoreCase("false")) + return new AllDatasetsLoaded(); + + if (value.startsWith("jdbc:")) return new DatasourceTableDatasets(value); + + if (value.equalsIgnoreCase("on") || value.equalsIgnoreCase("true")) + return new DatasourceTableDatasets(null); + + throw new UserException("-D" + GATE_PROPERTY + " is '" + value + "', which is not" + + " understood. Use 'on' to gate on the appDb in model-config.xml, a 'jdbc:...' URL" + + " to name a database explicitly, or 'off' (or omit it) for no gate."); + } +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestDatasetGate.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestDatasetGate.java new file mode 100644 index 00000000..d866e151 --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestDatasetGate.java @@ -0,0 +1,145 @@ +package org.apidb.apicommon.datasetPresenter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.junit.Test; + +/** + * JUnit tests for the dataset gate's effect on a DatasetPresenterSet: which presenters + * survive, and how the skipped ones are reported. + */ +public class TestDatasetGate { + + /** a LoadedDatasetSource that knows a fixed set of loaded dataset names */ + private static class FakeLoadedDatasets implements LoadedDatasetSource { + + private final Set _loaded; + + FakeLoadedDatasets(String... loaded) { + _loaded = new HashSet(Arrays.asList(loaded)); + } + + @Override + public boolean isLoaded(String nameOrPattern) { + if (nameOrPattern == null) return false; + if (_loaded.contains(nameOrPattern)) return true; + LikePattern pattern = new LikePattern(nameOrPattern); + for (String name : _loaded) { + if (pattern.matches(name)) return true; + } + return false; + } + + @Override + public String describe() { + return "fake gate with " + _loaded.size() + " loaded datasets"; + } + } + + private static DatasetPresenter presenter(String name) { + DatasetPresenter presenter = new DatasetPresenter(); + presenter.setName(name); + return presenter; + } + + private static DatasetPresenter patternPresenter(String name, String pattern) { + DatasetPresenter presenter = presenter(name); + presenter.setDatasetNamePattern(pattern); + return presenter; + } + + @Test + public void keepsLoadedAndDropsUnloaded() { + DatasetPresenterSet set = new DatasetPresenterSet(); + set.addDatasetPresenter(presenter("lmajFriedlin_sanger_BACEnds_clonedInsertEnds_RSRC")); + set.addDatasetPresenter(presenter("tgonME49_Sanger_BAC_ends_clonedInsertEnds_RSRC")); + + set.retainLoadedDatasets( + new FakeLoadedDatasets("lmajFriedlin_sanger_BACEnds_clonedInsertEnds_RSRC")); + + assertEquals(1, set.getSize()); + assertTrue(set.getDatasetPresenters() + .containsKey("lmajFriedlin_sanger_BACEnds_clonedInsertEnds_RSRC")); + assertEquals(1, set.getPresentersNotLoaded().size()); + assertTrue(set.getPresentersNotLoaded() + .contains("tgonME49_Sanger_BAC_ends_clonedInsertEnds_RSRC")); + } + + @Test + public void patternPresenterSurvivesOnAnyMatch() { + DatasetPresenterSet set = new DatasetPresenterSet(); + set.addDatasetPresenter(patternPresenter("_massSpec_Phosphoproteome_RSRC", + "%_massSpec_Phosphoproteome_RSRC")); + + set.retainLoadedDatasets(new FakeLoadedDatasets("tgonME49_massSpec_Phosphoproteome_RSRC")); + + assertEquals(1, set.getSize()); + assertTrue(set.getPresentersNotLoaded().isEmpty()); + } + + @Test + public void patternPresenterIsDroppedWhenNothingMatches() { + DatasetPresenterSet set = new DatasetPresenterSet(); + set.addDatasetPresenter(patternPresenter("_massSpec_Phosphoproteome_RSRC", + "%_massSpec_Phosphoproteome_RSRC")); + + set.retainLoadedDatasets(new FakeLoadedDatasets("tgonME49_primary_genome_RSRC")); + + assertEquals(0, set.getSize()); + // the pattern, not the name, is what failed to match, so it is what gets reported + assertTrue(set.getPresentersNotLoaded().contains("%_massSpec_Phosphoproteome_RSRC")); + } + + @Test + public void defaultSourceKeepsEverything() { + DatasetPresenterSet set = new DatasetPresenterSet(); + set.addDatasetPresenter(presenter("anything_RSRC")); + set.addDatasetPresenter(presenter("anything_else_RSRC")); + + set.retainLoadedDatasets(new AllDatasetsLoaded()); + + assertEquals(2, set.getSize()); + assertTrue(set.getPresentersNotLoaded().isEmpty()); + } + + @Test + public void skippedNamesGoToAFileNotTheLog() throws Exception { + DatasetPresenterSet set = new DatasetPresenterSet(); + set.addDatasetPresenter(presenter("absent_one_RSRC")); + set.addDatasetPresenter(presenter("absent_two_RSRC")); + set.retainLoadedDatasets(new FakeLoadedDatasets("something_else_RSRC")); + + File report = new File(Files.createTempDirectory("gate").toFile(), "sub/report.txt"); + set.reportPresentersNotLoaded(report); + + assertTrue(report.isFile()); + List lines = Files.readAllLines(report.toPath(), StandardCharsets.UTF_8); + assertTrue(lines.contains("absent_one_RSRC")); + assertTrue(lines.contains("absent_two_RSRC")); + // comments explain why the file exists, so a reader does not mistake it for errors + assertTrue(lines.get(0).startsWith("#")); + } + + @Test + public void reportingSurvivesAnUnwritableTarget() { + DatasetPresenterSet set = new DatasetPresenterSet(); + set.addDatasetPresenter(presenter("absent_RSRC")); + set.retainLoadedDatasets(new FakeLoadedDatasets("present_RSRC")); + + // must not throw: a reporting failure cannot be allowed to break the build + set.reportPresentersNotLoaded(new File("/proc/cannot/write/here.txt")); + set.reportPresentersNotLoaded(null); + + assertFalse(set.getPresentersNotLoaded().isEmpty()); + } +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestLikePattern.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestLikePattern.java new file mode 100644 index 00000000..03362ef4 --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestLikePattern.java @@ -0,0 +1,153 @@ +package org.apidb.apicommon.datasetPresenter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.PrintWriter; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.junit.Test; + +/** + * JUnit tests for {@link LikePattern}, which must agree with SQL LIKE so that a build-time + * dataset gate and the database never disagree about whether a presenter matches. + */ +public class TestLikePattern { + + @Test + public void percentSpansAnyRun() { + LikePattern pattern = new LikePattern("%_massSpec_Boothroyd_Phosphoproteome_RSRC"); + assertTrue(pattern.matches("tgonME49_massSpec_Boothroyd_Phosphoproteome_RSRC")); + assertTrue(pattern.matches("tgonGT1_massSpec_Boothroyd_Phosphoproteome_RSRC")); + assertFalse(pattern.matches("tgonME49_massSpec_Someone_Else_RSRC")); + } + + @Test + public void percentMatchesEmptyRun() { + assertTrue(new LikePattern("%abc").matches("abc")); + assertTrue(new LikePattern("abc%").matches("abc")); + } + + @Test + public void underscoreMatchesExactlyOneCharacter() { + // '_' is a single-character wildcard in SQL LIKE, and stays one here on purpose + assertTrue(new LikePattern("a_c").matches("abc")); + assertTrue(new LikePattern("a_c").matches("axc")); + assertFalse(new LikePattern("a_c").matches("ac")); + assertFalse(new LikePattern("a_c").matches("abbc")); + } + + @Test + public void plainNameMatchesItself() { + String name = "lmajFriedlin_sanger_BACEnds_clonedInsertEnds_RSRC"; + assertTrue(new LikePattern(name).matches(name)); + assertFalse(new LikePattern(name).matches(name + "_extra")); + } + + @Test + public void anchorsWholeString() { + assertFalse(new LikePattern("abc").matches("xabcx")); + assertTrue(new LikePattern("%abc%").matches("xabcx")); + } + + @Test + public void regexMetacharactersAreLiteral() { + assertTrue(new LikePattern("a.c").matches("a.c")); + assertFalse(new LikePattern("a.c").matches("abc")); + + assertTrue(new LikePattern("tcruCLBrenerEsmeraldo-like_CHORI105_BACEnds_clonedInsertEnds_RSRC") + .matches("tcruCLBrenerEsmeraldo-like_CHORI105_BACEnds_clonedInsertEnds_RSRC")); + + assertTrue(new LikePattern("a+b").matches("a+b")); + assertFalse(new LikePattern("a+b").matches("aab")); + assertTrue(new LikePattern("a(b)c").matches("a(b)c")); + assertTrue(new LikePattern("a[bc]d").matches("a[bc]d")); + assertFalse(new LikePattern("a[bc]d").matches("abd")); + assertTrue(new LikePattern("a$b^c|d").matches("a$b^c|d")); + assertTrue(new LikePattern("a\\b").matches("a\\b")); + } + + @Test + public void translationIsReadable() { + assertEquals(".*_x_RSRC".replace("_", "."), LikePattern.toRegex("%_x_RSRC")); + } + + @Test + public void nullNameNeverMatches() { + assertFalse(new LikePattern("%").matches(null)); + } + + @Test + public void gateIsOffWhenUnset() { + assertTrue(LoadedDatasetSources.forSetting(null) instanceof AllDatasetsLoaded); + assertTrue(LoadedDatasetSources.forSetting("") instanceof AllDatasetsLoaded); + assertTrue(LoadedDatasetSources.forSetting(" ") instanceof AllDatasetsLoaded); + assertTrue(LoadedDatasetSources.forSetting("off") instanceof AllDatasetsLoaded); + assertTrue(LoadedDatasetSources.forSetting("FALSE") instanceof AllDatasetsLoaded); + } + + @Test(expected = UserException.class) + public void unrecognizedGateSettingIsRejected() { + LoadedDatasetSources.forSetting("yes-please"); + } + + @Test + public void gatePropertyIsParsed() { + assertTrue(LoadedDatasetSources.forProperties(null, null) instanceof AllDatasetsLoaded); + assertTrue(LoadedDatasetSources.forProperties("off", null) instanceof AllDatasetsLoaded); + assertTrue(LoadedDatasetSources.forProperties(" ", null) instanceof AllDatasetsLoaded); + } + + @Test + public void alwaysPropertyIsSplitOnCommasAndWhitespace() { + assertEquals(0, LoadedDatasetSources.parseAlways(null).size()); + assertEquals(0, LoadedDatasetSources.parseAlways(" ").size()); + + Set two = LoadedDatasetSources.parseAlways(" a_RSRC, b_RSRC "); + assertEquals(2, two.size()); + assertTrue(two.contains("a_RSRC")); + assertTrue(two.contains("b_RSRC")); + + assertEquals(3, LoadedDatasetSources.parseAlways("a_RSRC b_RSRC,c_RSRC").size()); + } + + @Test + public void exceptionsWrapTheGateEvenWhenItIsOff() { + // gate off + exceptions still yields a source that answers yes for the excepted names, + // which matters because "off" means "inject everything" rather than "inject nothing" + LoadedDatasetSource source = LoadedDatasetSources.forProperties("off", "a_RSRC"); + assertTrue(source instanceof AlwaysLoadedDatasets); + assertTrue(source.isLoaded("a_RSRC")); + assertTrue(source.isLoaded("anything_else_RSRC")); + } + + @Test + public void exceptedDatasetsCountAsLoaded() { + LoadedDatasetSource nothingLoaded = new LoadedDatasetSource() { + @Override public boolean isLoaded(String nameOrPattern) { return false; } + @Override public String describe() { return "nothing loaded"; } + }; + + Set excepted = new HashSet( + Arrays.asList("pfal3D7_microarrayExpression_Derisi_TimeSeries_RSRC")); + LoadedDatasetSource source = new AlwaysLoadedDatasets(nothingLoaded, excepted); + + assertTrue(source.isLoaded("pfal3D7_microarrayExpression_Derisi_TimeSeries_RSRC")); + assertFalse(source.isLoaded("some_other_RSRC")); + // a pattern presenter covering an excepted dataset is kept too + assertTrue(source.isLoaded("%_microarrayExpression_Derisi_TimeSeries_RSRC")); + assertFalse(source.isLoaded("%_nothing_like_it_RSRC")); + assertTrue(source.describe().contains("Derisi")); + } + + @Test + public void allDatasetsLoadedAcceptsEverything() { + LoadedDatasetSource source = new AllDatasetsLoaded(); + assertTrue(source.isLoaded("anything_RSRC")); + assertTrue(source.isLoaded("%_pattern_RSRC")); + } +}