Optional dataset gate for presenter template injection (partial-appDb builds)

DatasetPresenter/bin/presenterInjectTemplates

@@ -8,7 +8,14 @@ my $CLASSPATH = &FgpUtil::Util::CommandHelper::getJavaClasspath($GUS_HOME);
 my $sysProps = &FgpUtil::Util::CommandHelper::getSystemProps($GUS_HOME, 'presenterInjectTemplates');
 my $args = &FgpUtil::Util::CommandHelper::getJavaArgs(@ARGV);
 
-my $cmd = "java $sysProps -classpath $CLASSPATH org.apidb.apicommon.datasetPresenter.TemplatesInjector $args";
+# Forward GUSJVMOPTS so a caller can pass -D properties to this command, which is how the
+# dataset gate is configured (-Dpresenter.dataset.gate=...). Two callers, two routes:
+#   rebuilder --gusjvmopts '-D...'   parsed after rebuilder discards the environment, so its
+#                                    export survives into the build
+#   wb                               inherits GUSJVMOPTS from etc/setenv; wb does not scrub
+my $jvmOpts = $ENV{GUSJVMOPTS} || '';
+
+my $cmd = "java $jvmOpts $sysProps -classpath $CLASSPATH org.apidb.apicommon.datasetPresenter.TemplatesInjector $args";
 
 system($cmd);
 

DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AllDatasetsLoaded.java

@@ -0,0 +1,19 @@
+package org.apidb.apicommon.datasetPresenter;
+
+/**
+ * The default {@link LoadedDatasetSource}: every presenter is treated as loaded and no
+ * database is contacted. This is the behavior the build has always had, and it is what
+ * runs unless a developer explicitly opts in to a dataset gate.
+ */
+public class AllDatasetsLoaded implements LoadedDatasetSource {
+
+  @Override
+  public boolean isLoaded(String nameOrPattern) {
+    return true;
+  }
+
+  @Override
+  public String describe() {
+    return "no dataset gate: every DatasetPresenter will be injected";
+  }
+}

DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/AlwaysLoadedDatasets.java

@@ -0,0 +1,53 @@
+package org.apidb.apicommon.datasetPresenter;
+
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * A {@link LoadedDatasetSource} that treats a named set of datasets as loaded regardless of
+ * what the delegate says, and otherwise defers to it.
+ *
+ * Needed because some DatasetPresenters deliberately contribute only attributes, leaving
+ * their questions hardcoded in the anchor file — the custom expression presenters do exactly
+ * this, with a "setting these to false so we don't inject questions" comment. Skipping such a
+ * presenter leaves the hardcoded question referencing an attribute nobody generated, which
+ * fails model load. Excepting the dataset runs its injectors and supplies the attribute.
+ *
+ * Use sparingly: an exception asserts "inject this presenter even though the data is absent",
+ * so any template it injects that queries missing tables will fail later, in a less obvious
+ * place. It is the right tool only when the presenter's output is metadata rather than data.
+ */
+public class AlwaysLoadedDatasets implements LoadedDatasetSource {
+
+  private final LoadedDatasetSource _delegate;
+  private final Set<String> _alwaysLoaded;
+
+  AlwaysLoadedDatasets(LoadedDatasetSource delegate, Set<String> alwaysLoaded) {
+    _delegate = delegate;
+    _alwaysLoaded = Collections.unmodifiableSet(new LinkedHashSet<String>(alwaysLoaded));
+  }
+
+  @Override
+  public boolean isLoaded(String nameOrPattern) {
+    if (nameOrPattern == null) return false;
+
+    if (_alwaysLoaded.contains(nameOrPattern)) return true;
+
+    // a pattern presenter counts as excepted when the pattern covers an excepted dataset
+    if (nameOrPattern.indexOf('%') >= 0) {
+      LikePattern pattern = new LikePattern(nameOrPattern);
+      for (String excepted : _alwaysLoaded) {
+        if (pattern.matches(excepted)) return true;
+      }
+    }
+
+    return _delegate.isLoaded(nameOrPattern);
+  }
+
+  @Override
+  public String describe() {
+    return _delegate.describe() + "; " + _alwaysLoaded.size()
+        + " dataset(s) always injected: " + String.join(", ", _alwaysLoaded);
+  }
+}

DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSet.java

@@ -1,10 +1,14 @@
 package org.apidb.apicommon.datasetPresenter;
 
 import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Set;
 /**
@@ -26,6 +30,7 @@ public class DatasetPresenterSet {
 
   private Map<String,Map<String,String>> _propertiesFromFiles = new HashMap<String,Map<String,String>>();
   private Set<String> _duplicateDatasetNames = new HashSet<String>();
+  private Set<String> _presentersNotLoaded = new LinkedHashSet<String>();
 
   /**
    * Add a DatasetPresenter to this set.
@@ -175,6 +180,93 @@ void handleOverrides() {
     }        
   }
 
+  /**
+   * Drop presenters whose dataset this instance has not loaded, according to the supplied
+   * source. The presenters XML directory is a legitimate superset of any one instance, so
+   * these are not errors — but they must go before anything walks the set, because
+   * constructing a DatasetInjector for an absent dataset fails on the props the dataset
+   * would have supplied.
+   *
+   * Filtering here, once, rather than at each point of use is deliberate: every consumer
+   * (dataset injectors, contact validation, model references, the loader) then inherits
+   * the gate without knowing it exists, and a consumer added later cannot forget it.
+   *
+   * Skipped presenters are always reported. A gate that dropped them silently would turn
+   * a loud build failure into a website that builds green with searches missing.
+   */
+  void retainLoadedDatasets(LoadedDatasetSource loadedDatasets) {
+    Iterator<Map.Entry<String, DatasetPresenter>> presenters = _presenters.entrySet().iterator();
+    while (presenters.hasNext()) {
+      DatasetPresenter presenter = presenters.next().getValue();
+
+      // a presenter carries either a name or a datasetNamePattern; resolve the same way
+      // DatasetPresenterSetLoader.getPresenterValuesFromDatasourceTable does
+      String pattern = presenter.getDatasetNamePattern();
+      String nameOrPattern = pattern == null ? presenter.getDatasetName() : pattern;
+
+      if (!loadedDatasets.isLoaded(nameOrPattern)) {
+        _presentersNotLoaded.add(nameOrPattern);
+        presenters.remove();
+        if (pattern != null) _namePatterns.remove(pattern);
+      }
+    }
+
+  }
+
+  /**
+   * @return where to list the skipped presenters, or null if GUS_HOME is not set
+   */
+  static File skippedReportFile() {
+    String gusHome = System.getenv("GUS_HOME");
+    return gusHome == null ? null : new File(gusHome + "/lib/wdk/presentersNotLoaded.txt");
+  }
+
+  /**
+   * Report the skipped presenters: the count on stderr, the names in a file.
+   *
+   * The count goes in the build log because a silent gate is worse than no gate — a build
+   * that quietly drops presenters looks successful while the website loses searches. The
+   * names go to a file because there can be thousands of them, and burying the rest of the
+   * build log to list them defeats the purpose of reporting at all.
+   */
+  void reportPresentersNotLoaded(File reportFile) {
+    if (_presentersNotLoaded.isEmpty()) return;
+
+    String summary = "Dataset gate: skipping " + _presentersNotLoaded.size()
+        + " DatasetPresenter(s) whose dataset is not loaded in this instance";
+
+    if (reportFile == null) {
+      // no GUS_HOME to write under; the names are all we can offer
+      System.err.println(summary + ":" + System.lineSeparator() + "  "
+          + String.join(System.lineSeparator() + "  ", _presentersNotLoaded));
+      return;
+    }
+
+    try {
+      File parent = reportFile.getParentFile();
+      if (parent != null) parent.mkdirs();
+      try (PrintWriter out = new PrintWriter(reportFile)) {
+        out.println("# DatasetPresenters skipped because their dataset is not in this");
+        out.println("# instance's apidb.Datasource. Presenters are a superset by design;");
+        out.println("# these are not errors.");
+        for (String name : _presentersNotLoaded) {
+          out.println(name);
+        }
+      }
+      System.err.println(summary + "; names listed in " + reportFile);
+    }
+    catch (IOException e) {
+      // reporting must not break the build, but it must not disappear either
+      System.err.println(summary + " (could not write " + reportFile + ": " + e.getMessage()
+          + "):" + System.lineSeparator() + "  "
+          + String.join(System.lineSeparator() + "  ", _presentersNotLoaded));
+    }
+  }
+
+  Set<String> getPresentersNotLoaded() {
+    return Collections.unmodifiableSet(_presentersNotLoaded);
+  }
+
   void addPropertiesFromFiles(Map<String,Map<String,String>> datasetNamesToProperties, Set<String> duplicateDatasetNames) {
     for (DatasetPresenter datasetPresenter : _presenters.values()) {
       datasetPresenter.addPropertiesFromFile(datasetNamesToProperties, duplicateDatasetNames);
@@ -200,6 +292,12 @@ void addIdentifierProperty() {
   // //////////////////// Static methods //////////////////
 
   static DatasetPresenterSet createFromPresentersDir(String presentersDir, String globalXmlFile) {
+    return createFromPresentersDir(presentersDir, globalXmlFile,
+        LoadedDatasetSources.fromSiteConfig());
+  }
+
+  static DatasetPresenterSet createFromPresentersDir(String presentersDir, String globalXmlFile,
+      LoadedDatasetSource loadedDatasets) {
     File pres = new File(presentersDir);
     if (!pres.isDirectory())
       throw new UserException("Presenters dir " + presentersDir
@@ -208,7 +306,13 @@ static DatasetPresenterSet createFromPresentersDir(String presentersDir, String
     // get the presenters into memory
     DatasetPresenterParser dpp = new DatasetPresenterParser();
     DatasetPresenterSet dps = dpp.parseDir(presentersDir, globalXmlFile);
-
+
+    // drop presenters for datasets this instance has not loaded, before anything walks
+    // the set. Off by default, so a normal build is unchanged.
+    System.err.println(loadedDatasets.describe());
+    dps.retainLoadedDatasets(loadedDatasets);
+    dps.reportPresentersNotLoaded(skippedReportFile());
+
     // add properties from dataset prop files to presenters
 
     DatasetPropertiesParser propParser = new DatasetPropertiesParser();

DatasetPresenter/src/main/java/org/apidb/apicommon/datasetPresenter/DatasetPresenterSetLoader.java

@@ -92,11 +92,13 @@ Set<String> syncPresenterSetWithDatasourceTable() {
       for (DatasetPresenter datasetPresenter : dps.getDatasetPresenters().values()) {
         getPresenterValuesFromDatasourceTable(datasetPresenter, datasourceTableStmt,
             datasetNamesFoundInDb);
-        if (!datasetPresenter.getFoundInDb())
-          presenterNamesNotInDb.add(datasetPresenter.getDatasetName());
 
-        datasetPresenter.getContacts(allContacts); // validate contacts
-        datasetPresenter.getModelReferences(); // validate model references
+        if (datasetPresenter.getFoundInDb()) {
+          datasetPresenter.getContacts(allContacts); // validate contacts
+          datasetPresenter.getModelReferences(); // validate model references (runs dataset injectors)
+        } else {
+          presenterNamesNotInDb.add(datasetPresenter.getDatasetName());
+        }
       }
 
       if (presenterNamesNotInDb.size() != 0) {