diff --git a/DatasetPresenter/bin/presenterInjectTemplates b/DatasetPresenter/bin/presenterInjectTemplates index b1d1e3e4..94744e6a 100644 --- a/DatasetPresenter/bin/presenterInjectTemplates +++ b/DatasetPresenter/bin/presenterInjectTemplates @@ -8,7 +8,14 @@ my $CLASSPATH = &FgpUtil::Util::CommandHelper::getJavaClasspath($GUS_HOME); my $sysProps = &FgpUtil::Util::CommandHelper::getSystemProps($GUS_HOME, 'presenterInjectTemplates'); my $args = &FgpUtil::Util::CommandHelper::getJavaArgs(@ARGV); -my $cmd = "java $sysProps -classpath $CLASSPATH org.apidb.apicommon.datasetPresenter.TemplatesInjector $args"; +# Forward GUSJVMOPTS so a caller can pass -D properties to this command, which is how the +# dataset gate is configured (-Dpresenter.dataset.gate=...). Two callers, two routes: +# rebuilder --gusjvmopts '-D...' parsed after rebuilder discards the environment, so its +# export survives into the build +# wb inherits GUSJVMOPTS from etc/setenv; wb does not scrub +my $jvmOpts = $ENV{GUSJVMOPTS} || ''; + +my $cmd = "java $jvmOpts $sysProps -classpath $CLASSPATH org.apidb.apicommon.datasetPresenter.TemplatesInjector $args"; system($cmd); diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AllDatasetsLoaded.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AllDatasetsLoaded.java new file mode 100644 index 00000000..781db506 --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AllDatasetsLoaded.java @@ -0,0 +1,19 @@ +package org.apidb.apicommon.datasetPresenter; + +/** + * The default {@link LoadedDatasetSource}: every presenter is treated as loaded and no + * database is contacted. This is the behavior the build has always had, and it is what + * runs unless a developer explicitly opts in to a dataset gate. + */ +public class AllDatasetsLoaded implements LoadedDatasetSource { + + @Override + public boolean isLoaded(String nameOrPattern) { + return true; + } + + @Override + public String describe() { + return "no dataset gate: every DatasetPresenter will be injected"; + } +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AlwaysLoadedDatasets.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AlwaysLoadedDatasets.java new file mode 100644 index 00000000..754d9491 --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AlwaysLoadedDatasets.java @@ -0,0 +1,53 @@ +package org.apidb.apicommon.datasetPresenter; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * A {@link LoadedDatasetSource} that treats a named set of datasets as loaded regardless of + * what the delegate says, and otherwise defers to it. + * + * Needed because some DatasetPresenters deliberately contribute only attributes, leaving + * their questions hardcoded in the anchor file — the custom expression presenters do exactly + * this, with a "setting these to false so we don't inject questions" comment. Skipping such a + * presenter leaves the hardcoded question referencing an attribute nobody generated, which + * fails model load. Excepting the dataset runs its injectors and supplies the attribute. + * + * Use sparingly: an exception asserts "inject this presenter even though the data is absent", + * so any template it injects that queries missing tables will fail later, in a less obvious + * place. It is the right tool only when the presenter's output is metadata rather than data. + */ +public class AlwaysLoadedDatasets implements LoadedDatasetSource { + + private final LoadedDatasetSource _delegate; + private final Set _alwaysLoaded; + + AlwaysLoadedDatasets(LoadedDatasetSource delegate, Set alwaysLoaded) { + _delegate = delegate; + _alwaysLoaded = Collections.unmodifiableSet(new LinkedHashSet(alwaysLoaded)); + } + + @Override + public boolean isLoaded(String nameOrPattern) { + if (nameOrPattern == null) return false; + + if (_alwaysLoaded.contains(nameOrPattern)) return true; + + // a pattern presenter counts as excepted when the pattern covers an excepted dataset + if (nameOrPattern.indexOf('%') >= 0) { + LikePattern pattern = new LikePattern(nameOrPattern); + for (String excepted : _alwaysLoaded) { + if (pattern.matches(excepted)) return true; + } + } + + return _delegate.isLoaded(nameOrPattern); + } + + @Override + public String describe() { + return _delegate.describe() + "; " + _alwaysLoaded.size() + + " dataset(s) always injected: " + String.join(", ", _alwaysLoaded); + } +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSet.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSet.java index 7aba5325..f1583cf4 100644 --- a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSet.java +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSet.java @@ -1,10 +1,14 @@ package org.apidb.apicommon.datasetPresenter; import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; /** @@ -26,6 +30,7 @@ public class DatasetPresenterSet { private Map> _propertiesFromFiles = new HashMap>(); private Set _duplicateDatasetNames = new HashSet(); + private Set _presentersNotLoaded = new LinkedHashSet(); /** * Add a DatasetPresenter to this set. @@ -175,6 +180,93 @@ void handleOverrides() { } } + /** + * Drop presenters whose dataset this instance has not loaded, according to the supplied + * source. The presenters XML directory is a legitimate superset of any one instance, so + * these are not errors — but they must go before anything walks the set, because + * constructing a DatasetInjector for an absent dataset fails on the props the dataset + * would have supplied. + * + * Filtering here, once, rather than at each point of use is deliberate: every consumer + * (dataset injectors, contact validation, model references, the loader) then inherits + * the gate without knowing it exists, and a consumer added later cannot forget it. + * + * Skipped presenters are always reported. A gate that dropped them silently would turn + * a loud build failure into a website that builds green with searches missing. + */ + void retainLoadedDatasets(LoadedDatasetSource loadedDatasets) { + Iterator> presenters = _presenters.entrySet().iterator(); + while (presenters.hasNext()) { + DatasetPresenter presenter = presenters.next().getValue(); + + // a presenter carries either a name or a datasetNamePattern; resolve the same way + // DatasetPresenterSetLoader.getPresenterValuesFromDatasourceTable does + String pattern = presenter.getDatasetNamePattern(); + String nameOrPattern = pattern == null ? presenter.getDatasetName() : pattern; + + if (!loadedDatasets.isLoaded(nameOrPattern)) { + _presentersNotLoaded.add(nameOrPattern); + presenters.remove(); + if (pattern != null) _namePatterns.remove(pattern); + } + } + + } + + /** + * @return where to list the skipped presenters, or null if GUS_HOME is not set + */ + static File skippedReportFile() { + String gusHome = System.getenv("GUS_HOME"); + return gusHome == null ? null : new File(gusHome + "/lib/wdk/presentersNotLoaded.txt"); + } + + /** + * Report the skipped presenters: the count on stderr, the names in a file. + * + * The count goes in the build log because a silent gate is worse than no gate — a build + * that quietly drops presenters looks successful while the website loses searches. The + * names go to a file because there can be thousands of them, and burying the rest of the + * build log to list them defeats the purpose of reporting at all. + */ + void reportPresentersNotLoaded(File reportFile) { + if (_presentersNotLoaded.isEmpty()) return; + + String summary = "Dataset gate: skipping " + _presentersNotLoaded.size() + + " DatasetPresenter(s) whose dataset is not loaded in this instance"; + + if (reportFile == null) { + // no GUS_HOME to write under; the names are all we can offer + System.err.println(summary + ":" + System.lineSeparator() + " " + + String.join(System.lineSeparator() + " ", _presentersNotLoaded)); + return; + } + + try { + File parent = reportFile.getParentFile(); + if (parent != null) parent.mkdirs(); + try (PrintWriter out = new PrintWriter(reportFile)) { + out.println("# DatasetPresenters skipped because their dataset is not in this"); + out.println("# instance's apidb.Datasource. Presenters are a superset by design;"); + out.println("# these are not errors."); + for (String name : _presentersNotLoaded) { + out.println(name); + } + } + System.err.println(summary + "; names listed in " + reportFile); + } + catch (IOException e) { + // reporting must not break the build, but it must not disappear either + System.err.println(summary + " (could not write " + reportFile + ": " + e.getMessage() + + "):" + System.lineSeparator() + " " + + String.join(System.lineSeparator() + " ", _presentersNotLoaded)); + } + } + + Set getPresentersNotLoaded() { + return Collections.unmodifiableSet(_presentersNotLoaded); + } + void addPropertiesFromFiles(Map> datasetNamesToProperties, Set duplicateDatasetNames) { for (DatasetPresenter datasetPresenter : _presenters.values()) { datasetPresenter.addPropertiesFromFile(datasetNamesToProperties, duplicateDatasetNames); @@ -200,6 +292,12 @@ void addIdentifierProperty() { // //////////////////// Static methods ////////////////// static DatasetPresenterSet createFromPresentersDir(String presentersDir, String globalXmlFile) { + return createFromPresentersDir(presentersDir, globalXmlFile, + LoadedDatasetSources.fromSiteConfig()); + } + + static DatasetPresenterSet createFromPresentersDir(String presentersDir, String globalXmlFile, + LoadedDatasetSource loadedDatasets) { File pres = new File(presentersDir); if (!pres.isDirectory()) throw new UserException("Presenters dir " + presentersDir @@ -208,7 +306,13 @@ static DatasetPresenterSet createFromPresentersDir(String presentersDir, String // get the presenters into memory DatasetPresenterParser dpp = new DatasetPresenterParser(); DatasetPresenterSet dps = dpp.parseDir(presentersDir, globalXmlFile); - + + // drop presenters for datasets this instance has not loaded, before anything walks + // the set. Off by default, so a normal build is unchanged. + System.err.println(loadedDatasets.describe()); + dps.retainLoadedDatasets(loadedDatasets); + dps.reportPresentersNotLoaded(skippedReportFile()); + // add properties from dataset prop files to presenters DatasetPropertiesParser propParser = new DatasetPropertiesParser(); diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSetLoader.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSetLoader.java index d8b4ee71..e90e884c 100644 --- a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSetLoader.java +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSetLoader.java @@ -92,11 +92,13 @@ Set syncPresenterSetWithDatasourceTable() { for (DatasetPresenter datasetPresenter : dps.getDatasetPresenters().values()) { getPresenterValuesFromDatasourceTable(datasetPresenter, datasourceTableStmt, datasetNamesFoundInDb); - if (!datasetPresenter.getFoundInDb()) - presenterNamesNotInDb.add(datasetPresenter.getDatasetName()); - datasetPresenter.getContacts(allContacts); // validate contacts - datasetPresenter.getModelReferences(); // validate model references + if (datasetPresenter.getFoundInDb()) { + datasetPresenter.getContacts(allContacts); // validate contacts + datasetPresenter.getModelReferences(); // validate model references (runs dataset injectors) + } else { + presenterNamesNotInDb.add(datasetPresenter.getDatasetName()); + } } if (presenterNamesNotInDb.size() != 0) { diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasourceTableDatasets.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasourceTableDatasets.java new file mode 100644 index 00000000..bac3701b --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasourceTableDatasets.java @@ -0,0 +1,187 @@ +package org.apidb.apicommon.datasetPresenter; + +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +/** + * A {@link LoadedDatasetSource} backed by apidb.Datasource, which is what the workflow's + * root graph actually loaded. Opt-in only — see {@link LoadedDatasetSources}. + * + * Every dataset name is read once up front and matched in memory, rather than issuing a + * LIKE query per presenter, because presenters number in the thousands. + * + * Connection details come from the site's own model-config.xml (the database the model + * will run against), following the same precedence as WDK's ModelConfigDB: an explicit + * connectionUrl wins, then host + database name. LDAP resolution is not reimplemented + * here; a site configured only for LDAP should pass an explicit JDBC URL instead. + */ +public class DatasourceTableDatasets implements LoadedDatasetSource { + + private static final String DATASOURCE_SQL = "select name from apidb.datasource"; + + private final Set _loadedDatasetNames; + private final String _description; + + /** + * @param jdbcUrlOverride an explicit JDBC URL, or null to take one from model-config.xml + */ + DatasourceTableDatasets(String jdbcUrlOverride) { + File modelConfig = findModelConfig(); + Element appDb = parseAppDbElement(modelConfig); + + String url = jdbcUrlOverride != null ? jdbcUrlOverride : connectionUrlFrom(appDb, modelConfig); + String login = required(appDb, "login", modelConfig); + String password = required(appDb, "password", modelConfig); + + _loadedDatasetNames = readDatasetNames(url, login, password); + _description = "dataset gate on " + url + " as " + login + ": " + + _loadedDatasetNames.size() + " datasets in apidb.Datasource"; + } + + @Override + public boolean isLoaded(String nameOrPattern) { + if (nameOrPattern == null) return false; + + // Fast path: an exact hit is a match under LIKE semantics too, and most presenters + // carry a plain name. Only fall back to a scan when there is no exact match. + if (_loadedDatasetNames.contains(nameOrPattern)) return true; + + LikePattern pattern = new LikePattern(nameOrPattern); + for (String loadedName : _loadedDatasetNames) { + if (pattern.matches(loadedName)) return true; + } + return false; + } + + @Override + public String describe() { + return _description; + } + + private static Set readDatasetNames(String url, String login, String password) { + Set names = new HashSet(); + // A gate that was explicitly asked for must not degrade into "inject everything" when + // the database is unreachable; that would reproduce the failure it exists to prevent. + try (Connection connection = DriverManager.getConnection(url, login, password); + Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery(DATASOURCE_SQL)) { + while (rs.next()) { + names.add(rs.getString(1)); + } + } + catch (SQLException e) { + throw new UserException("Dataset gate is enabled but apidb.Datasource could not be" + + " read from " + url + " as " + login + ": " + e.getMessage(), e); + } + if (names.isEmpty()) + throw new UserException("Dataset gate is enabled but apidb.Datasource at " + url + + " holds no datasets; refusing to skip every DatasetPresenter"); + return names; + } + + /** + * Locate the site's model-config.xml under $GUS_HOME/config. + * + * Found by looking for the file rather than by composing a path from the project name, + * because the environment is not a dependable source here: rebuilder discards everything + * but a short whitelist, so WDK_MODEL is absent under rebuilder even though it is set for + * wb. WDK_MODEL and PROJECT are consulted only to disambiguate a gus_home that somehow + * configures more than one project. + */ + private static File findModelConfig() { + String gusHome = System.getenv("GUS_HOME"); + if (gusHome == null) + throw new UserException("Dataset gate is enabled but GUS_HOME is not set; it supplies" + + " the location of model-config.xml"); + + File configDir = new File(gusHome, "config"); + for (String hint : new String[] { System.getenv("WDK_MODEL"), System.getenv("PROJECT") }) { + if (hint == null || hint.trim().isEmpty()) continue; + File hinted = new File(configDir, hint.trim() + "/model-config.xml"); + if (hinted.isFile()) return hinted; + } + + List found = new ArrayList(); + File[] projectDirs = configDir.listFiles(); + if (projectDirs != null) { + for (File projectDir : projectDirs) { + File modelConfig = new File(projectDir, "model-config.xml"); + if (modelConfig.isFile()) found.add(modelConfig); + } + } + + if (found.size() == 1) return found.get(0); + + if (found.isEmpty()) + throw new UserException("Dataset gate is enabled but no model-config.xml was found" + + " under " + configDir + ". It is written by 'conifer configure', which rebuilder" + + " runs after this build step, so a from-scratch rebuilder cannot use the gate on" + + " its first pass."); + + throw new UserException("Dataset gate is enabled but " + configDir + " holds " + + found.size() + " model-config.xml files (" + found + "); set WDK_MODEL to name the" + + " project, or give the gate an explicit jdbc: URL"); + } + + private static Element parseAppDbElement(File modelConfig) { + try { + DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + Document document = builder.parse(modelConfig); + NodeList appDbNodes = document.getElementsByTagName("appDb"); + if (appDbNodes.getLength() == 0) + throw new UserException("No element in " + modelConfig); + return (Element) appDbNodes.item(0); + } + catch (UserException e) { + throw e; + } + catch (Exception e) { + throw new UserException("Could not read " + modelConfig + ": " + e.getMessage(), e); + } + } + + private static String connectionUrlFrom(Element appDb, File modelConfig) { + String connectionUrl = attribute(appDb, "connectionUrl"); + if (connectionUrl != null) return connectionUrl; + + String host = attribute(appDb, "dbHost"); + String dbName = attribute(appDb, "dbIdentifier"); + if (host != null && dbName != null) { + String port = attribute(appDb, "dbPort"); + return "jdbc:postgresql://" + host + (port == null ? "" : ":" + port) + "/" + dbName; + } + + throw new UserException("The in " + modelConfig + " supplies neither a" + + " connectionUrl nor dbHost + dbIdentifier. It is presumably configured for LDAP" + + " lookup, which the dataset gate does not perform; enable the gate with an" + + " explicit JDBC URL instead of 'on'."); + } + + private static String required(Element appDb, String name, File modelConfig) { + String value = attribute(appDb, name); + if (value == null) + throw new UserException("The in " + modelConfig + " has no " + name + + " attribute, which the dataset gate needs to connect"); + return value; + } + + private static String attribute(Element element, String name) { + String value = element.getAttribute(name); + return value == null || value.trim().isEmpty() ? null : value.trim(); + } +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LikePattern.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LikePattern.java new file mode 100644 index 00000000..b8bfdc11 --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LikePattern.java @@ -0,0 +1,58 @@ +package org.apidb.apicommon.datasetPresenter; + +import java.util.regex.Pattern; + +/** + * A SQL LIKE pattern, evaluated in memory. + * + * Deliberately mirrors the semantics of the "ds.NAME like ?" query in + * {@link DatasetPresenterSetLoader#getDatasourceTableStmt}, so that a dataset gate + * applied at build time can never disagree with the database about whether a + * DatasetPresenter matches a dataset: '%' spans any run of characters, '_' matches + * exactly one, and every other character is literal. + * + * Note that '_' remains a single-character wildcard even though dataset names contain + * literal underscores throughout. That is looser than most authors intend, but it is + * what the database does; tightening it here in isolation would let injection skip a + * presenter the loader matched. + */ +public class LikePattern { + + private final String _like; + private final Pattern _regex; + + LikePattern(String likePattern) { + _like = likePattern; + _regex = Pattern.compile(toRegex(likePattern)); + } + + /** + * Translate a SQL LIKE pattern into an equivalent regular expression. + */ + static String toRegex(String likePattern) { + StringBuilder regex = new StringBuilder(); + for (char c : likePattern.toCharArray()) { + switch (c) { + case '%': + regex.append(".*"); + break; + case '_': + regex.append('.'); + break; + default: + if ("\\.[]{}()*+-?^$|".indexOf(c) >= 0) regex.append('\\'); + regex.append(c); + } + } + return regex.toString(); + } + + boolean matches(String datasetName) { + return datasetName != null && _regex.matcher(datasetName).matches(); + } + + @Override + public String toString() { + return _like; + } +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSource.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSource.java new file mode 100644 index 00000000..5071120a --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSource.java @@ -0,0 +1,31 @@ +package org.apidb.apicommon.datasetPresenter; + +/** + * Answers "does this instance have the dataset a DatasetPresenter presents?". + * + * The presenters XML directory is deliberately a superset of what any one instance has + * loaded, so presenters for absent datasets must be skipped rather than injected. What + * counts as loaded is not knowable from the presenters or the dataset XML alone — the + * workflow's root graph decides, and apidb.Datasource is its output — so the answer is + * supplied by an implementation of this interface rather than computed inline. + * + * The default implementation, {@link AllDatasetsLoaded}, answers yes to everything and + * touches no database, which is required: the website build cannot make database calls in + * normal operation. A developer building against a partially loaded database opts in to + * {@link DatasourceTableDatasets} instead; see {@link LoadedDatasetSources}. + */ +public interface LoadedDatasetSource { + + /** + * @param nameOrPattern a DatasetPresenter's datasetNamePattern if it has one, else its + * dataset name. Interpreted with SQL LIKE semantics either way, matching how + * {@link DatasetPresenterSetLoader} resolves the same choice. + * @return whether at least one loaded dataset matches + */ + boolean isLoaded(String nameOrPattern); + + /** + * @return a short description of this source, for the build log + */ + String describe(); +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSources.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSources.java new file mode 100644 index 00000000..af02d948 --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/LoadedDatasetSources.java @@ -0,0 +1,96 @@ +package org.apidb.apicommon.datasetPresenter; + +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Chooses the {@link LoadedDatasetSource} for this build. + * + * The gate is off unless a developer turns it on, so a normal build makes no database calls + * and injects exactly the presenters it always did. It is turned on with a JVM system + * property: + * + * 
+ *   -Dpresenter.dataset.gate=on                                # appDb from model-config.xml
+ *   -Dpresenter.dataset.gate=jdbc:postgresql://host:5432/mydb  # or name one explicitly
+ *   -Dpresenter.dataset.gate=off                               # (the default)
+ *
+ *   -Dpresenter.dataset.gate.always=ds_one_RSRC,ds_two_RSRC    # optional exceptions
+ *
+ * + * The exceptions list injects those presenters even though their data is absent, for + * presenters that deliberately contribute only attributes while their questions stay + * hardcoded in an anchor file; see {@link AlwaysLoadedDatasets}. + * + * A system property rather than an environment variable because the two ways a site is built + * do not share an environment, and only a property reaches both: + * + *
    + *
  • rebuilder discards the environment on startup (keeping only PWD, USER, LANG, + * SUDO_USER, HOME, SSH_AUTH_SOCK and GITHUB_*), but its --gusjvmopts flag is parsed + * afterwards, so rebuilder --gusjvmopts '-Dpresenter.dataset.gate=on' arrives + * intact.
    • + *
  • wb does not scrub, so GUSJVMOPTS exported from the site's etc/setenv reaches it.
    • + *
+ * + * Either way the value lands here via GUSJVMOPTS, which the presenterInjectTemplates wrapper + * forwards to the JVM. An environment variable read directly by this class was tried first: + * it worked under wb and silently did nothing under rebuilder, which is exactly the class of + * failure this gate exists to prevent. + * + * Credentials are deliberately not passed here: a command line is world-readable in ps + * output, so the appDb login and password come from model-config.xml. + */ +public class LoadedDatasetSources { + + public static final String GATE_PROPERTY = "presenter.dataset.gate"; + public static final String ALWAYS_PROPERTY = "presenter.dataset.gate.always"; + + private LoadedDatasetSources() {} + + static LoadedDatasetSource fromSiteConfig() { + return forProperties(System.getProperty(GATE_PROPERTY), System.getProperty(ALWAYS_PROPERTY)); + } + + /** + * @param gate value of presenter.dataset.gate + * @param always comma- or whitespace-separated dataset names to inject regardless + */ + static LoadedDatasetSource forProperties(String gate, String always) { + LoadedDatasetSource source = forSetting(gate); + + Set alwaysLoaded = parseAlways(always); + return alwaysLoaded.isEmpty() ? source : new AlwaysLoadedDatasets(source, alwaysLoaded); + } + + static Set parseAlways(String always) { + Set names = new LinkedHashSet(); + if (always == null) return names; + + for (String name : always.split("[,\\s]+")) { + if (!name.trim().isEmpty()) names.add(name.trim()); + } + return names; + } + + /** + * @param setting null, empty, "off" or "false" for no gate; "on" or "true" to gate on the + * appDb named in model-config.xml; or an explicit "jdbc:..." URL + */ + static LoadedDatasetSource forSetting(String setting) { + if (setting == null || setting.trim().isEmpty()) return new AllDatasetsLoaded(); + + String value = setting.trim(); + if (value.equalsIgnoreCase("off") || value.equalsIgnoreCase("false")) + return new AllDatasetsLoaded(); + + if (value.startsWith("jdbc:")) return new DatasourceTableDatasets(value); + + if (value.equalsIgnoreCase("on") || value.equalsIgnoreCase("true")) + return new DatasourceTableDatasets(null); + + throw new UserException("-D" + GATE_PROPERTY + " is '" + value + "', which is not" + + " understood. Use 'on' to gate on the appDb in model-config.xml, a 'jdbc:...' URL" + + " to name a database explicitly, or 'off' (or omit it) for no gate."); + } +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestDatasetGate.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestDatasetGate.java new file mode 100644 index 00000000..d866e151 --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestDatasetGate.java @@ -0,0 +1,145 @@ +package org.apidb.apicommon.datasetPresenter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.junit.Test; + +/** + * JUnit tests for the dataset gate's effect on a DatasetPresenterSet: which presenters + * survive, and how the skipped ones are reported. + */ +public class TestDatasetGate { + + /** a LoadedDatasetSource that knows a fixed set of loaded dataset names */ + private static class FakeLoadedDatasets implements LoadedDatasetSource { + + private final Set _loaded; + + FakeLoadedDatasets(String... loaded) { + _loaded = new HashSet(Arrays.asList(loaded)); + } + + @Override + public boolean isLoaded(String nameOrPattern) { + if (nameOrPattern == null) return false; + if (_loaded.contains(nameOrPattern)) return true; + LikePattern pattern = new LikePattern(nameOrPattern); + for (String name : _loaded) { + if (pattern.matches(name)) return true; + } + return false; + } + + @Override + public String describe() { + return "fake gate with " + _loaded.size() + " loaded datasets"; + } + } + + private static DatasetPresenter presenter(String name) { + DatasetPresenter presenter = new DatasetPresenter(); + presenter.setName(name); + return presenter; + } + + private static DatasetPresenter patternPresenter(String name, String pattern) { + DatasetPresenter presenter = presenter(name); + presenter.setDatasetNamePattern(pattern); + return presenter; + } + + @Test + public void keepsLoadedAndDropsUnloaded() { + DatasetPresenterSet set = new DatasetPresenterSet(); + set.addDatasetPresenter(presenter("lmajFriedlin_sanger_BACEnds_clonedInsertEnds_RSRC")); + set.addDatasetPresenter(presenter("tgonME49_Sanger_BAC_ends_clonedInsertEnds_RSRC")); + + set.retainLoadedDatasets( + new FakeLoadedDatasets("lmajFriedlin_sanger_BACEnds_clonedInsertEnds_RSRC")); + + assertEquals(1, set.getSize()); + assertTrue(set.getDatasetPresenters() + .containsKey("lmajFriedlin_sanger_BACEnds_clonedInsertEnds_RSRC")); + assertEquals(1, set.getPresentersNotLoaded().size()); + assertTrue(set.getPresentersNotLoaded() + .contains("tgonME49_Sanger_BAC_ends_clonedInsertEnds_RSRC")); + } + + @Test + public void patternPresenterSurvivesOnAnyMatch() { + DatasetPresenterSet set = new DatasetPresenterSet(); + set.addDatasetPresenter(patternPresenter("_massSpec_Phosphoproteome_RSRC", + "%_massSpec_Phosphoproteome_RSRC")); + + set.retainLoadedDatasets(new FakeLoadedDatasets("tgonME49_massSpec_Phosphoproteome_RSRC")); + + assertEquals(1, set.getSize()); + assertTrue(set.getPresentersNotLoaded().isEmpty()); + } + + @Test + public void patternPresenterIsDroppedWhenNothingMatches() { + DatasetPresenterSet set = new DatasetPresenterSet(); + set.addDatasetPresenter(patternPresenter("_massSpec_Phosphoproteome_RSRC", + "%_massSpec_Phosphoproteome_RSRC")); + + set.retainLoadedDatasets(new FakeLoadedDatasets("tgonME49_primary_genome_RSRC")); + + assertEquals(0, set.getSize()); + // the pattern, not the name, is what failed to match, so it is what gets reported + assertTrue(set.getPresentersNotLoaded().contains("%_massSpec_Phosphoproteome_RSRC")); + } + + @Test + public void defaultSourceKeepsEverything() { + DatasetPresenterSet set = new DatasetPresenterSet(); + set.addDatasetPresenter(presenter("anything_RSRC")); + set.addDatasetPresenter(presenter("anything_else_RSRC")); + + set.retainLoadedDatasets(new AllDatasetsLoaded()); + + assertEquals(2, set.getSize()); + assertTrue(set.getPresentersNotLoaded().isEmpty()); + } + + @Test + public void skippedNamesGoToAFileNotTheLog() throws Exception { + DatasetPresenterSet set = new DatasetPresenterSet(); + set.addDatasetPresenter(presenter("absent_one_RSRC")); + set.addDatasetPresenter(presenter("absent_two_RSRC")); + set.retainLoadedDatasets(new FakeLoadedDatasets("something_else_RSRC")); + + File report = new File(Files.createTempDirectory("gate").toFile(), "sub/report.txt"); + set.reportPresentersNotLoaded(report); + + assertTrue(report.isFile()); + List lines = Files.readAllLines(report.toPath(), StandardCharsets.UTF_8); + assertTrue(lines.contains("absent_one_RSRC")); + assertTrue(lines.contains("absent_two_RSRC")); + // comments explain why the file exists, so a reader does not mistake it for errors + assertTrue(lines.get(0).startsWith("#")); + } + + @Test + public void reportingSurvivesAnUnwritableTarget() { + DatasetPresenterSet set = new DatasetPresenterSet(); + set.addDatasetPresenter(presenter("absent_RSRC")); + set.retainLoadedDatasets(new FakeLoadedDatasets("present_RSRC")); + + // must not throw: a reporting failure cannot be allowed to break the build + set.reportPresentersNotLoaded(new File("/proc/cannot/write/here.txt")); + set.reportPresentersNotLoaded(null); + + assertFalse(set.getPresentersNotLoaded().isEmpty()); + } +} diff --git a/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestLikePattern.java b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestLikePattern.java new file mode 100644 index 00000000..03362ef4 --- /dev/null +++ b/DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/TestLikePattern.java @@ -0,0 +1,153 @@ +package org.apidb.apicommon.datasetPresenter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.PrintWriter; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.junit.Test; + +/** + * JUnit tests for {@link LikePattern}, which must agree with SQL LIKE so that a build-time + * dataset gate and the database never disagree about whether a presenter matches. + */ +public class TestLikePattern { + + @Test + public void percentSpansAnyRun() { + LikePattern pattern = new LikePattern("%_massSpec_Boothroyd_Phosphoproteome_RSRC"); + assertTrue(pattern.matches("tgonME49_massSpec_Boothroyd_Phosphoproteome_RSRC")); + assertTrue(pattern.matches("tgonGT1_massSpec_Boothroyd_Phosphoproteome_RSRC")); + assertFalse(pattern.matches("tgonME49_massSpec_Someone_Else_RSRC")); + } + + @Test + public void percentMatchesEmptyRun() { + assertTrue(new LikePattern("%abc").matches("abc")); + assertTrue(new LikePattern("abc%").matches("abc")); + } + + @Test + public void underscoreMatchesExactlyOneCharacter() { + // '_' is a single-character wildcard in SQL LIKE, and stays one here on purpose + assertTrue(new LikePattern("a_c").matches("abc")); + assertTrue(new LikePattern("a_c").matches("axc")); + assertFalse(new LikePattern("a_c").matches("ac")); + assertFalse(new LikePattern("a_c").matches("abbc")); + } + + @Test + public void plainNameMatchesItself() { + String name = "lmajFriedlin_sanger_BACEnds_clonedInsertEnds_RSRC"; + assertTrue(new LikePattern(name).matches(name)); + assertFalse(new LikePattern(name).matches(name + "_extra")); + } + + @Test + public void anchorsWholeString() { + assertFalse(new LikePattern("abc").matches("xabcx")); + assertTrue(new LikePattern("%abc%").matches("xabcx")); + } + + @Test + public void regexMetacharactersAreLiteral() { + assertTrue(new LikePattern("a.c").matches("a.c")); + assertFalse(new LikePattern("a.c").matches("abc")); + + assertTrue(new LikePattern("tcruCLBrenerEsmeraldo-like_CHORI105_BACEnds_clonedInsertEnds_RSRC") + .matches("tcruCLBrenerEsmeraldo-like_CHORI105_BACEnds_clonedInsertEnds_RSRC")); + + assertTrue(new LikePattern("a+b").matches("a+b")); + assertFalse(new LikePattern("a+b").matches("aab")); + assertTrue(new LikePattern("a(b)c").matches("a(b)c")); + assertTrue(new LikePattern("a[bc]d").matches("a[bc]d")); + assertFalse(new LikePattern("a[bc]d").matches("abd")); + assertTrue(new LikePattern("a$b^c|d").matches("a$b^c|d")); + assertTrue(new LikePattern("a\\b").matches("a\\b")); + } + + @Test + public void translationIsReadable() { + assertEquals(".*_x_RSRC".replace("_", "."), LikePattern.toRegex("%_x_RSRC")); + } + + @Test + public void nullNameNeverMatches() { + assertFalse(new LikePattern("%").matches(null)); + } + + @Test + public void gateIsOffWhenUnset() { + assertTrue(LoadedDatasetSources.forSetting(null) instanceof AllDatasetsLoaded); + assertTrue(LoadedDatasetSources.forSetting("") instanceof AllDatasetsLoaded); + assertTrue(LoadedDatasetSources.forSetting(" ") instanceof AllDatasetsLoaded); + assertTrue(LoadedDatasetSources.forSetting("off") instanceof AllDatasetsLoaded); + assertTrue(LoadedDatasetSources.forSetting("FALSE") instanceof AllDatasetsLoaded); + } + + @Test(expected = UserException.class) + public void unrecognizedGateSettingIsRejected() { + LoadedDatasetSources.forSetting("yes-please"); + } + + @Test + public void gatePropertyIsParsed() { + assertTrue(LoadedDatasetSources.forProperties(null, null) instanceof AllDatasetsLoaded); + assertTrue(LoadedDatasetSources.forProperties("off", null) instanceof AllDatasetsLoaded); + assertTrue(LoadedDatasetSources.forProperties(" ", null) instanceof AllDatasetsLoaded); + } + + @Test + public void alwaysPropertyIsSplitOnCommasAndWhitespace() { + assertEquals(0, LoadedDatasetSources.parseAlways(null).size()); + assertEquals(0, LoadedDatasetSources.parseAlways(" ").size()); + + Set two = LoadedDatasetSources.parseAlways(" a_RSRC, b_RSRC "); + assertEquals(2, two.size()); + assertTrue(two.contains("a_RSRC")); + assertTrue(two.contains("b_RSRC")); + + assertEquals(3, LoadedDatasetSources.parseAlways("a_RSRC b_RSRC,c_RSRC").size()); + } + + @Test + public void exceptionsWrapTheGateEvenWhenItIsOff() { + // gate off + exceptions still yields a source that answers yes for the excepted names, + // which matters because "off" means "inject everything" rather than "inject nothing" + LoadedDatasetSource source = LoadedDatasetSources.forProperties("off", "a_RSRC"); + assertTrue(source instanceof AlwaysLoadedDatasets); + assertTrue(source.isLoaded("a_RSRC")); + assertTrue(source.isLoaded("anything_else_RSRC")); + } + + @Test + public void exceptedDatasetsCountAsLoaded() { + LoadedDatasetSource nothingLoaded = new LoadedDatasetSource() { + @Override public boolean isLoaded(String nameOrPattern) { return false; } + @Override public String describe() { return "nothing loaded"; } + }; + + Set excepted = new HashSet( + Arrays.asList("pfal3D7_microarrayExpression_Derisi_TimeSeries_RSRC")); + LoadedDatasetSource source = new AlwaysLoadedDatasets(nothingLoaded, excepted); + + assertTrue(source.isLoaded("pfal3D7_microarrayExpression_Derisi_TimeSeries_RSRC")); + assertFalse(source.isLoaded("some_other_RSRC")); + // a pattern presenter covering an excepted dataset is kept too + assertTrue(source.isLoaded("%_microarrayExpression_Derisi_TimeSeries_RSRC")); + assertFalse(source.isLoaded("%_nothing_like_it_RSRC")); + assertTrue(source.describe().contains("Derisi")); + } + + @Test + public void allDatasetsLoadedAcceptsEverything() { + LoadedDatasetSource source = new AllDatasetsLoaded(); + assertTrue(source.isLoaded("anything_RSRC")); + assertTrue(source.isLoaded("%_pattern_RSRC")); + } +}