diff --git a/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MLContext.java b/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MLContext.java index 534f873e5..cf799a23a 100644 --- a/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MLContext.java +++ b/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MLContext.java @@ -63,8 +63,8 @@ public void execute(final WayangPlan wayangPlan, final String... udfJars) { final Optional originalOption = config.getOptionalStringProperty("wayang.ml.experience.original"); final OneHotMappings mappings = new OneHotMappings(); - final TreeEncoder encoder = new TreeEncoder(mappings); - final String original = originalOption.orElse(encoder.encode(wayangPlan, wayangJob.getOptimizationContext(), false).toString()); + final TreeEncoder encoder = new TreeEncoder(mappings, config); + final String original = originalOption.orElse(encoder.encode(wayangPlan, wayangJob.getOptimizationContext()).toString()); final Optional choicesOption = config .getOptionalStringProperty("wayang.ml.experience.with-platforms"); diff --git a/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MachineLearning.java b/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MachineLearning.java index 80ebc1f67..4c2313c2d 100644 --- a/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MachineLearning.java +++ b/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MachineLearning.java @@ -18,6 +18,7 @@ package org.apache.wayang.ml; +import java.io.InputStream; import java.util.Collection; import java.util.Collections; @@ -26,6 +27,7 @@ import org.apache.wayang.core.optimizer.channels.ChannelConversion; import org.apache.wayang.core.platform.Platform; import org.apache.wayang.core.plugin.Plugin; +import org.apache.wayang.core.util.ReflectionUtils; import org.apache.wayang.java.platform.JavaPlatform; import org.apache.wayang.spark.platform.SparkPlatform; @@ -34,6 +36,8 @@ */ public class MachineLearning { + private static final String DEFAULT_CONFIG_FILE = "wayang-ml-defaults.properties"; + /** * Enables use with the {@link JavaPlatform} and {@link SparkPlatform}. */ @@ -56,9 +60,47 @@ public Collection getChannelConversions() { @Override public void setProperties(final Configuration configuration) { + try (final InputStream is = ReflectionUtils.loadResource(DEFAULT_CONFIG_FILE)) { + if (is != null) { + final java.util.Properties properties = new java.util.Properties(); + properties.load(is); + for (final java.util.Map.Entry propertyEntry : properties.entrySet()) { + final String key = propertyEntry.getKey().toString(); + final String value = propertyEntry.getValue().toString(); + if (isConfigured(configuration, key)) { + continue; + } + configuration.setProperty(key, value); + } + } + } catch (java.io.IOException e) { + throw new org.apache.wayang.core.api.exception.WayangException("Could not load ML default configuration.", e); + } } }; + private static boolean isConfigured(final Configuration configuration, final String key) { + return isExplicitlyConfigured(configuration, key); + } + + private static boolean isExplicitlyConfigured(final Configuration configuration, final String key) { + if (configuration == null) { + return false; + } + if (configuration == Configuration.getDefaultConfiguration()) { + return configuration.getProperties() != null + && configuration.getProperties().provideLocally(key) != null; + } + Configuration current = configuration; + while (current != null && current != Configuration.getDefaultConfiguration()) { + if (current.getProperties() != null && current.getProperties().provideLocally(key) != null) { + return true; + } + current = current.getParent(); + } + return false; + } + /** * Retrieve a {@link Plugin} to use xxx on the * diff --git a/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/encoding/TreeEncoder.java b/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/encoding/TreeEncoder.java index cff4430f2..edc3a5f60 100644 --- a/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/encoding/TreeEncoder.java +++ b/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/encoding/TreeEncoder.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Queue; +import org.apache.wayang.core.api.Configuration; import org.apache.wayang.core.api.exception.WayangException; import org.apache.wayang.core.optimizer.OptimizationContext; import org.apache.wayang.core.optimizer.enumeration.PlanImplementation; @@ -40,17 +41,93 @@ import org.apache.wayang.core.platform.Junction; public class TreeEncoder { + + public static final String ENCODE_IDS_PROPERTY = "wayang.ml.encoding.encode-ids"; + + public static final String ENCODE_IDS_SHORTHAND_PROPERTY = "wayang.ml.encode-ids"; + private final OneHotMappings mappings; + private Configuration configuration; + public TreeEncoder(final OneHotMappings mappings) { + this(mappings, null); + } + + public TreeEncoder(final OneHotMappings mappings, final Configuration configuration) { this.mappings = mappings; + this.configuration = configuration; } public OneHotMappings getMappings() { return this.mappings; } + public Configuration getConfiguration() { + return this.configuration; + } + + public void setConfiguration(final Configuration configuration) { + this.configuration = configuration; + } + + /** + * Resolves whether operator IDs should be encoded. The configuration hierarchy is walked from the given + * (most specific, e.g. job-level) configuration up to its ancestors; the closest level that sets either + * {@link #ENCODE_IDS_PROPERTY} or {@link #ENCODE_IDS_SHORTHAND_PROPERTY} wins, so a child can override an + * inherited value even if it uses the other spelling. If both are set on the very same level, a spelling + * that merely mirrors the inherited value (e.g., cached by an earlier read through the child) loses + * against the other one; otherwise the canonical property takes precedence. Defaults to {@code false}. + */ + public static boolean resolveEncodeIds(final Configuration config) { + for (Configuration level = config; level != null; level = level.getParent()) { + final String canonical = level.getProperties().provideLocally(ENCODE_IDS_PROPERTY); + final String shorthand = level.getProperties().provideLocally(ENCODE_IDS_SHORTHAND_PROPERTY); + if (canonical == null && shorthand == null) { + continue; + } + if (canonical == null) { + return Boolean.parseBoolean(shorthand); + } + if (shorthand == null) { + return Boolean.parseBoolean(canonical); + } + final Configuration parent = level.getParent(); + final boolean canonicalMirrorsParent = canonical.equals(inheritedValue(parent, ENCODE_IDS_PROPERTY)); + final boolean shorthandMirrorsParent = shorthand.equals(inheritedValue(parent, ENCODE_IDS_SHORTHAND_PROPERTY)); + return canonicalMirrorsParent && !shorthandMirrorsParent + ? Boolean.parseBoolean(shorthand) + : Boolean.parseBoolean(canonical); + } + return false; + } + + /** + * Looks up the value that {@code level} or its closest ancestor defines for {@code key} without touching + * any property caches. + */ + private static String inheritedValue(final Configuration level, final String key) { + for (Configuration current = level; current != null; current = current.getParent()) { + final String value = current.getProperties().provideLocally(key); + if (value != null) { + return value; + } + } + return null; + } + + private boolean resolveEncodeIds(final OptimizationContext optimizationContext) { + final Configuration config = this.configuration != null + ? this.configuration + : (optimizationContext != null ? optimizationContext.getConfiguration() : null); + return resolveEncodeIds(config); + } + public TreeNode encode(final PlanImplementation plan) { + return this.encode(plan, this.resolveEncodeIds(plan.getOptimizationContext())); + } + + public TreeNode encode(final PlanImplementation plan, final boolean encodeIds) { final List result = new ArrayList(); final HashMap> tree = new HashMap<>(); @@ -58,9 +135,6 @@ public TreeNode encode(final PlanImplementation plan) { final Map, Junction> junctions = plan.getJunctions(); - // TODO: convert to config - final boolean encodeIds = false; - for (final Operator sink : sinks) { final TreeNode sinkNode = traversePIOperator(sink, plan.getOptimizationContext(), encodeIds, junctions, tree); @@ -77,6 +151,10 @@ public TreeNode encode(final PlanImplementation plan) { return resultNode; } + public TreeNode encode(final WayangPlan plan, final OptimizationContext optimizationContext) { + return this.encode(plan, optimizationContext, this.resolveEncodeIds(optimizationContext)); + } + public TreeNode encode(final WayangPlan plan, final OptimizationContext optimizationContext, final boolean encodeIds) { final List result = new ArrayList(); diff --git a/wayang-plugins/wayang-ml/src/main/resources/wayang-ml-defaults.properties b/wayang-plugins/wayang-ml/src/main/resources/wayang-ml-defaults.properties index 822169eb8..7cbc683f6 100644 --- a/wayang-plugins/wayang-ml/src/main/resources/wayang-ml-defaults.properties +++ b/wayang-plugins/wayang-ml/src/main/resources/wayang-ml-defaults.properties @@ -73,4 +73,7 @@ wayang.ml.experience.enabled = false wayang.ml.executions.file = /var/www/html/data/executions.txt wayang.ml.optimizations.file = /var/www/html/data/optmizations.txt wayang.ml.experience.file = /var/www/html/data/experience/experience-vae.txt +# Not set on purpose (TreeEncoder defaults to false); a default here would shadow inherited settings. +# Use wayang.ml.encoding.encode-ids or the shorthand wayang.ml.encode-ids. +# wayang.ml.encoding.encode-ids = false org.apache.logging.log4j.level = INFO diff --git a/wayang-plugins/wayang-ml/src/test/java/org/apache/wayang/ml/test/TreeEncoderTest.java b/wayang-plugins/wayang-ml/src/test/java/org/apache/wayang/ml/test/TreeEncoderTest.java index 31ea7de01..beaa15c9a 100644 --- a/wayang-plugins/wayang-ml/src/test/java/org/apache/wayang/ml/test/TreeEncoderTest.java +++ b/wayang-plugins/wayang-ml/src/test/java/org/apache/wayang/ml/test/TreeEncoderTest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -28,6 +29,7 @@ import org.apache.wayang.core.api.Configuration; import org.apache.wayang.core.api.Job; import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.optimizer.enumeration.PlanImplementation; import org.apache.wayang.core.plan.executionplan.ExecutionPlan; import org.apache.wayang.core.plan.wayangplan.WayangPlan; import org.apache.wayang.java.Java; @@ -57,5 +59,354 @@ public void testTreeEncoding() throws IOException, URISyntaxException { Assertions.assertNotNull(exPlan); Assertions.assertNotNull(encoded); + Assertions.assertEquals(0L, encoded.encoded[0]); + } + + @Test + public void testEncodePlanImplementationWithDefaultConfig() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration config = new Configuration(); + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final WayangContext wayangContext = new WayangContext(config); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + final Collection planImplementations = buildPlanImplementations(wayangPlan, wayangContext); + Assertions.assertFalse(planImplementations.isEmpty()); + + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings()); + for (final PlanImplementation planImplementation : planImplementations) { + final TreeNode encoded = encoder.encode(planImplementation); + Assertions.assertNotNull(encoded); + Assertions.assertEquals(0L, encoded.encoded[0]); + } + } + + @Test + public void testEncodePlanImplementationWithEncodeIdsEnabled() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration config = new Configuration(); + config.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final WayangContext wayangContext = new WayangContext(config); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + final Collection planImplementations = buildPlanImplementations(wayangPlan, wayangContext); + Assertions.assertFalse(planImplementations.isEmpty()); + + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings()); + for (final PlanImplementation planImplementation : planImplementations) { + final TreeNode encoded = encoder.encode(planImplementation); + Assertions.assertNotNull(encoded); + Assertions.assertNotEquals(0L, encoded.encoded[0]); + } + } + + @Test + public void testEncodeWayangPlanWithConfiguration() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration config = new Configuration(); + config.setProperty("wayang.ml.encode-ids", "true"); + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final WayangContext wayangContext = new WayangContext(config); + final Job wayangJob = wayangContext.createJob("", wayangPlan, ""); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + wayangJob.buildInitialExecutionPlan(); + + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings()); + final TreeNode encoded = encoder.encode(wayangPlan, wayangJob.getOptimizationContext()); + + Assertions.assertNotNull(encoded); + Assertions.assertNotEquals(0L, encoded.encoded[0]); + } + + @Test + public void testTreeEncoderConstructorWithConfiguration() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration config = new Configuration(); + config.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final WayangContext wayangContext = new WayangContext(config); + final Job wayangJob = wayangContext.createJob("", wayangPlan, ""); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + wayangJob.buildInitialExecutionPlan(); + + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings(), config); + Assertions.assertEquals(config, encoder.getConfiguration()); + + final TreeNode encoded = encoder.encode(wayangPlan, wayangJob.getOptimizationContext()); + Assertions.assertNotNull(encoded); + Assertions.assertNotEquals(0L, encoded.encoded[0]); + } + + @Test + public void testMachineLearningPluginSetProperties() { + final Configuration config = new Configuration(); + org.apache.wayang.ml.MachineLearning.plugin().setProperties(config); + Assertions.assertFalse(config.getOptionalBooleanProperty(TreeEncoder.ENCODE_IDS_PROPERTY).isPresent()); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(config)); + Assertions.assertEquals(100L, config.getLongProperty("wayang.ml.tuple.average-size")); + } + + @Test + public void testPluginRegistrationPreservesUserSettings() { + final Configuration config = new Configuration(); + config.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + config.setProperty("wayang.ml.model.file", "/custom/model/path.onnx"); + config.setProperty("wayang.ml.experience.enabled", "true"); + config.setProperty("wayang.ml.experience.file", "/custom/experience.txt"); + config.setProperty("wayang.core.optimizer.reoptimize", "true"); + config.setProperty("wayang.core.optimizer.reoptimize.proactive", "true"); + + final WayangContext wayangContext = new WayangContext(config); + wayangContext.register(org.apache.wayang.ml.MachineLearning.plugin()); + final Configuration contextConfig = wayangContext.getConfiguration(); + + // User settings must remain intact + Assertions.assertTrue(contextConfig.getBooleanProperty(TreeEncoder.ENCODE_IDS_PROPERTY)); + Assertions.assertEquals("/custom/model/path.onnx", contextConfig.getStringProperty("wayang.ml.model.file")); + Assertions.assertTrue(contextConfig.getBooleanProperty("wayang.ml.experience.enabled")); + Assertions.assertEquals("/custom/experience.txt", contextConfig.getStringProperty("wayang.ml.experience.file")); + Assertions.assertTrue(contextConfig.getBooleanProperty("wayang.core.optimizer.reoptimize")); + Assertions.assertTrue(contextConfig.getBooleanProperty("wayang.core.optimizer.reoptimize.proactive")); + + // Unconfigured defaults should still be populated + Assertions.assertEquals(100L, contextConfig.getLongProperty("wayang.ml.tuple.average-size")); + Assertions.assertEquals("/var/www/html/data/executions.txt", contextConfig.getStringProperty("wayang.ml.executions.file")); + } + + @Test + public void testPluginRegistrationPreservesShorthandEncodeIds() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration config = new Configuration(); + config.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + + final WayangContext wayangContext = new WayangContext(config); + wayangContext.register(org.apache.wayang.ml.MachineLearning.plugin()); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + final Configuration contextConfig = wayangContext.getConfiguration(); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(contextConfig)); + + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final Job wayangJob = wayangContext.createJob("", wayangPlan, ""); + wayangJob.buildInitialExecutionPlan(); + + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings()); + final TreeNode encoded = encoder.encode(wayangPlan, wayangJob.getOptimizationContext()); + Assertions.assertNotNull(encoded); + Assertions.assertNotEquals(0L, encoded.encoded[0]); + } + + @Test + public void testShorthandEncodeIdsAfterPluginRegistration() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration config = new Configuration(); + + final WayangContext wayangContext = new WayangContext(config); + wayangContext.register(org.apache.wayang.ml.MachineLearning.plugin()); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + final Configuration contextConfig = wayangContext.getConfiguration(); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(contextConfig)); + + // Setting shorthand after registration must not be shadowed by canonical default false + contextConfig.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(contextConfig)); + + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final Job wayangJob = wayangContext.createJob("", wayangPlan, ""); + wayangJob.buildInitialExecutionPlan(); + + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings()); + final TreeNode encoded = encoder.encode(wayangPlan, wayangJob.getOptimizationContext()); + Assertions.assertNotNull(encoded); + Assertions.assertNotEquals(0L, encoded.encoded[0]); + } + + @Test + public void testPluginRegistrationDirectSetPropertiesPreservesUserSettings() { + final Configuration config = new Configuration(); + config.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + config.setProperty("wayang.ml.model.file", "/my/custom/model.onnx"); + config.setProperty("wayang.ml.experience.enabled", "true"); + config.setProperty("wayang.core.optimizer.reoptimize", "true"); + + org.apache.wayang.ml.MachineLearning.plugin().setProperties(config); + + Assertions.assertEquals("/my/custom/model.onnx", config.getStringProperty("wayang.ml.model.file")); + Assertions.assertTrue(config.getBooleanProperty("wayang.ml.experience.enabled")); + Assertions.assertTrue(config.getBooleanProperty("wayang.core.optimizer.reoptimize")); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(config)); + Assertions.assertEquals(100L, config.getLongProperty("wayang.ml.tuple.average-size")); + } + + @Test + public void testResolveEncodeIdsMatrix() { + // 1. Both absent + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(new Configuration())); + + // 2. Canonical true, shorthand absent + final Configuration config1 = new Configuration(); + config1.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(config1)); + + // 3. Canonical false, shorthand absent + final Configuration config2 = new Configuration(); + config2.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(config2)); + + // 4. Canonical absent, shorthand true + final Configuration config3 = new Configuration(); + config3.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(config3)); + + // 5. Canonical absent, shorthand false + final Configuration config4 = new Configuration(); + config4.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(config4)); + + // 6. Canonical false, shorthand true on the same level: canonical takes precedence + final Configuration config5 = new Configuration(); + config5.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + config5.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(config5)); + + // 7. Canonical true, shorthand false + final Configuration config6 = new Configuration(); + config6.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + config6.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "false"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(config6)); + + // 8. Canonical true, shorthand true + final Configuration config7 = new Configuration(); + config7.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + config7.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(config7)); + + // 9. Canonical false, shorthand false + final Configuration config8 = new Configuration(); + config8.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + config8.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(config8)); + } + + @Test + public void testChildShorthandFalseOverridesInheritedCanonicalTrue() { + final Configuration parent = new Configuration(); + parent.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + final Configuration child = parent.fork(); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + + child.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(child)); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(parent)); + } + + @Test + public void testChildCanonicalFalseOverridesInheritedShorthandTrue() { + final Configuration parent = new Configuration(); + parent.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + final Configuration child = parent.fork(); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + + child.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(child)); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(parent)); + } + + @Test + public void testChildTrueOverridesInheritedFalseAcrossSpellings() { + final Configuration parent = new Configuration(); + parent.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + final Configuration child = parent.fork(); + child.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + + final Configuration parent2 = new Configuration(); + parent2.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "false"); + final Configuration child2 = parent2.fork(); + child2.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child2)); + } + + @Test + public void testChildOverrideAfterCachedParentRead() { + final Configuration parent = new Configuration(); + parent.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + final Configuration child = parent.fork(); + + // Reading through the child caches the inherited value locally; this must not look like an override. + Assertions.assertTrue(child.getBooleanProperty(TreeEncoder.ENCODE_IDS_PROPERTY)); + child.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(child)); + + final Configuration parent2 = new Configuration(); + parent2.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + final Configuration child2 = parent2.fork(); + Assertions.assertTrue(child2.getBooleanProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY)); + child2.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(child2)); + } + + @Test + public void testChildInheritsFromGrandparentWhenNotOverridden() { + final Configuration grandparent = new Configuration(); + grandparent.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + final Configuration parent = grandparent.fork(); + final Configuration child = parent.fork(); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + + parent.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(child)); + + child.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + } + + @Test + public void testJobLevelOverrideAfterPluginRegistration() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration userConfig = new Configuration(); + userConfig.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + + final WayangContext wayangContext = new WayangContext(userConfig); + wayangContext.register(org.apache.wayang.ml.MachineLearning.plugin()); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + // The plugin must not inject a default that shadows the user's inherited shorthand. + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(wayangContext.getConfiguration())); + + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final Job inheritingJob = wayangContext.createJob("", wayangPlan, ""); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(inheritingJob.getConfiguration())); + + final WayangPlan overridingPlan = createWayangPlan(filePath, collector); + final Job overridingJob = wayangContext.createJob("", overridingPlan, ""); + overridingJob.getConfiguration().setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(overridingJob.getConfiguration())); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(wayangContext.getConfiguration())); + + overridingJob.buildInitialExecutionPlan(); + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings(), overridingJob.getConfiguration()); + final TreeNode encoded = encoder.encode(overridingPlan, overridingJob.getOptimizationContext()); + Assertions.assertNotNull(encoded); + Assertions.assertEquals(0L, encoded.encoded[0]); } } +