Add profiles to x/config/cli

x/config/cli/README.md

@@ -147,4 +147,7 @@ takes a string.
 - [`Options`](options.go): env var prefixes, base config, default config path.
 - [`New` / `Binder.Register`](binder.go): registration, on the root or a subcommand. Flags are
   persistent, so subcommands take them too.
+- [`Profile` / `ProfileWithSelector`](profile.go): build a struct's or a section's defaults with a
+  function picked by one of its top-level fields; see [`examples/profiles`](examples/profiles) and
+  [`examples/nested_profiles`](examples/nested_profiles).
 - [`examples/`](examples): runnable programs.

x/config/cli/binder.go

@@ -58,8 +58,8 @@ func New(opts Options) (*Binder, error) {
 	return b, nil
 }
 
-// Register attaches target to cmd. Before cmd or a subcommand runs, target is filled from flags, env vars, and config
-// files, then its `validate` tags are checked.
+// Register attaches target to cmd. Before cmd or a subcommand runs, target is filled from flags, env vars, config
+// files, and any [Profile], then its `validate` tags are checked.
 //
 // Scalars, durations, []byte, [encoding.TextUnmarshaler] types, and lists and maps of those get a persistent flag and
 // env vars; other fields are config file only. A list of pointers or interfaces, a map of them, and a map whose keys

x/config/cli/entries.go

@@ -256,48 +256,93 @@ func overlap(a, b, placeholder, sep string) (string, bool) {
 
 // entrySet is one value a source sets inside a map, such as rpc in entry mainnet.
 type entrySet struct {
-	leaf   *entryLeaf
-	keys   []reflect.Value
+	leaf *entryLeaf
+	keys []reflect.Value
+	// texts holds the leading keys still to resolve, named by env vars; keys holds the rest, already parsed.
+	texts  []string
 	value  reflect.Value
 	source string
 	// rank is the env var prefix's index, as the first prefix set wins.
 	rank int
 }
 
+// Keys named by an env var resolve against the map the entry layers over, which holds every source below env vars.
+func (s entrySet) resolve(m reflect.Value) ([]reflect.Value, error) {
+	if s.texts == nil {
+		return s.keys, nil
+	}
+
+	keys := make([]reflect.Value, 0, len(s.texts)+len(s.keys))
+	for i, text := range s.texts {
+		keyText, err := envKeyText(entryMap(m, s.leaf, keys), text)
+		if err != nil {
+			return nil, fmt.Errorf("%s: %w", s.source, err)
+		}
+
+		key, err := mapKey(s.leaf.keyTypes[i], keyText, map[any]string{})
+		if err != nil {
+			return nil, fmt.Errorf("%s: %w", s.source, err)
+		}
+
+		keys = append(keys, key)
+	}
+
+	return append(keys, s.keys...), nil
+}
+
 // decodeEntries layers a map's sources lowest first, each setting whole entries or values inside them, so
-// --chains.mainnet.rpc changes only rpc in the file's mainnet.
-func (e *typedEntry[T]) decodeEntries(k leafKey, cc commandConfig, dst reflect.Value) error {
+// --chains.mainnet.rpc changes only rpc in the file's mainnet. It returns how the sources layer over a map, nil if
+// none set it.
+func (e *typedEntry[T]) decodeEntries(k leafKey, cc commandConfig, dst reflect.Value) (sourceLayer, error) {
 	t := commentparsing.DerefType(k.goType)
-	m := mergeMaps(t, readField(dst, k.goPath))
-	changed := false
+	var file reflect.Value
 	if raw, ok := cc.keys.lookup(cc.fileValues, k.fileKey); ok {
-		file, err := fromFile(t, raw)
-		if err != nil {
-			return err
+		var err error
+		if file, err = fromFile(t, raw); err != nil {
+			return nil, err
 		}
-
-		m, changed = mergeMaps(t, m, file), true
 	}
 
-	envSets, err := e.envEntrySets(k, m)
+	envSets, err := e.envEntrySets(k)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	for _, layer := range [][]entrySet{envSets, e.flagEntrySets(k, cc.flags)} {
-		if layer, err = oneValueEach(layer); err != nil {
-			return err
-		}
+	layers := [][]entrySet{envSets, e.flagEntrySets(k, cc.flags)}
+	if !file.IsValid() && len(layers[0]) == 0 && len(layers[1]) == 0 {
+		return nil, nil
+	}
 
-		for _, s := range layer {
-			setEntry(m, s.leaf, s.keys, 0, s.value)
+	layer := func(under reflect.Value) (reflect.Value, error) {
+		m := mergeMaps(t, under)
+		if file.IsValid() {
+			m = mergeMaps(t, m, file)
 		}
 
-		changed = changed || len(layer) > 0
-	}
+		for _, sets := range layers {
+			// Resolved against m, before any set entry can add a key another set's name would then match.
+			resolved := make([]entrySet, 0, len(sets))
+			for _, s := range sets {
+				keys, kerr := s.resolve(m)
+				if kerr != nil {
+					return reflect.Value{}, kerr
+				}
 
-	if !changed {
-		return nil
+				s.keys, s.texts = keys, nil
+				resolved = append(resolved, s)
+			}
+
+			unique, uerr := oneValueEach(resolved)
+			if uerr != nil {
+				return reflect.Value{}, uerr
+			}
+
+			for _, s := range unique {
+				setEntry(m, s.leaf, s.keys, 0, s.value)
+			}
+		}
+
+		return m, nil
 	}
 
 	f := allocateField(dst, k.goPath)
@@ -309,12 +354,22 @@ func (e *typedEntry[T]) decodeEntries(k leafKey, cc commandConfig, dst reflect.V
 		f = f.Elem()
 	}
 
-	f.Set(m)
-	return nil
+	// A profile replaces its section's maps, so the keys an env var names are resolved against the profile's map
+	// when it applies this layer, not the one here.
+	if slices.ContainsFunc(e.profileSelectors, func(s leafKey) bool { return inScope(k, s.goPath[:len(s.goPath)-1]) }) {
+		return layer, nil
+	}
+
+	v, err := layer(readField(dst, k.goPath))
+	if err != nil {
+		return nil, err
+	}
+
+	f.Set(v)
+	return layer, nil
 }
 
-// Keys in env var names are matched against base's, which holds every source below env vars.
-func (e *typedEntry[T]) envEntrySets(k leafKey, base reflect.Value) ([]entrySet, error) {
+func (e *typedEntry[T]) envEntrySets(k leafKey) ([]entrySet, error) {
 	var sets []entrySet
 	if k.flag != nil {
 		for rank, name := range e.envVars(k) {
@@ -324,7 +379,7 @@ func (e *typedEntry[T]) envEntrySets(k leafKey, base reflect.Value) ([]entrySet,
 					return nil, fmt.Errorf("invalid value %q for %s: %w", s, name, err)
 				}
 
-				sets = splitEntries(k.each, nil, val, name, rank)
+				sets = splitEntries(k.each, nil, nil, val, name, rank)
 				break
 			}
 		}
@@ -343,24 +398,12 @@ func (e *typedEntry[T]) envEntrySets(k leafKey, base reflect.Value) ([]entrySet,
 			continue
 		}
 
-		keys := make([]reflect.Value, len(texts))
-		for i, text := range texts {
-			keyText, err := envKeyText(entryMap(base, leaf, keys[:i]), text)
-			if err != nil {
-				return nil, fmt.Errorf("%s: %w", name, err)
-			}
-
-			if keys[i], err = mapKey(leaf.keyTypes[i], keyText, map[any]string{}); err != nil {
-				return nil, fmt.Errorf("%s: %w", name, err)
-			}
-		}
-
 		val, err := parseText(leaf.goType, s)
 		if err != nil {
 			return nil, fmt.Errorf("invalid value %q for %s: %w", s, name, err)
 		}
 
-		sets = append(sets, entrySets(leaf, keys, val, name, rank)...)
+		sets = append(sets, entrySets(leaf, nil, texts, val, name, rank)...)
 	}
 
 	return sets, nil
@@ -381,12 +424,12 @@ func (e *typedEntry[T]) envEntry(k leafKey, name string) (*entryLeaf, int, []str
 func (e *typedEntry[T]) flagEntrySets(k leafKey, flags *pflag.FlagSet) []entrySet {
 	var sets []entrySet
 	if k.flag != nil && k.flag.Changed {
-		sets = splitEntries(k.each, nil, k.value(), "--"+k.flagName, 0)
+		sets = splitEntries(k.each, nil, nil, k.value(), "--"+k.flagName, 0)
 	}
 
 	flags.Visit(func(f *pflag.Flag) {
 		if ef, ok := e.b.entryFlags[f]; ok && slices.Contains(k.entries, ef.leaf) {
-			sets = append(sets, entrySets(ef.leaf, ef.keys, ef.value(), "--"+f.Name, 0)...)
+			sets = append(sets, entrySets(ef.leaf, ef.keys, nil, ef.value(), "--"+f.Name, 0)...)
 		}
 	})
 
@@ -439,22 +482,22 @@ func readField(v reflect.Value, path []string) reflect.Value {
 
 // Splits a whole map into its entries: --labels a=1,b=2 becomes the same as --labels.a 1 --labels.b 2, so giving
 // --labels a=1 --labels.a 3 is caught as setting it twice.
-func splitEntries(each *entryLeaf, keys []reflect.Value, m reflect.Value, source string, rank int) []entrySet {
+func splitEntries(each *entryLeaf, keys []reflect.Value, texts []string, m reflect.Value, source string, rank int) []entrySet {
 	var sets []entrySet
 	for key, val := range m.Seq2() {
 		entrySource := fmt.Sprintf("%s (key %q)", source, textOf(key))
-		sets = append(sets, entrySets(each, append(slices.Clone(keys), key), val, entrySource, rank)...)
+		sets = append(sets, entrySets(each, append(slices.Clone(keys), key), texts, val, entrySource, rank)...)
 	}
 
 	return sets
 }
 
-func entrySets(leaf *entryLeaf, keys []reflect.Value, val reflect.Value, source string, rank int) []entrySet {
+func entrySets(leaf *entryLeaf, keys []reflect.Value, texts []string, val reflect.Value, source string, rank int) []entrySet {
 	if leaf.each != nil && val.Kind() == reflect.Map {
-		return splitEntries(leaf.each, keys, val, source, rank)
+		return splitEntries(leaf.each, keys, texts, val, source, rank)
 	}
 
-	return []entrySet{{leaf, keys, val, source, rank}}
+	return []entrySet{{leaf: leaf, keys: keys, texts: texts, value: val, source: source, rank: rank}}
 }
 
 // Within one source, two values for one entry are an error, as two keys for one are. Across env var prefixes the

x/config/cli/entry.go

@@ -61,6 +61,9 @@ type leafKey struct {
 	// entries set values inside a map's entries, such as chains.<key>.rpc; each is the one that sets a whole entry.
 	entries []*entryLeaf
 	each    *entryLeaf
+
+	// required is a `required` rule on the field, which a profile's help leaves to the sources.
+	required bool
 }
 
 type typedEntry[T any] struct {
@@ -72,6 +75,13 @@ type typedEntry[T any] struct {
 	// ns, if set, is the first segment of every key.
 	ns string
 
+	// profiles run after the sources are decoded, given the leaves a source set.
+	profiles []func(supplied map[string]sourceLayer) error
+
+	// profileSelectors holds each registered profile's selector, so a new one overlapping its section fails Register:
+	// each profile rebuilds its whole section, so two on one section would drop each other's values.
+	profileSelectors []leafKey
+
 	undocumentedKeys []string
 }
 
@@ -226,14 +236,25 @@ func (e *typedEntry[T]) parentType(structNS string) reflect.Type {
 	return t
 }
 
+// sourceLayer puts a map's source values over under, key by key, since maps merge by key at every source. A value that
+// isn't a map has none: its source replaces it whole. Keys an env var names resolve against under, which can hold keys
+// a profile added, so applying one can fail.
+type sourceLayer func(under reflect.Value) (reflect.Value, error)
+
 func (e *typedEntry[T]) decode(cc commandConfig) error {
 	dst := reflect.ValueOf(e.dst).Elem()
+	supplied := map[string]sourceLayer{}
 	for _, k := range e.leaves {
 		if len(k.entries) > 0 {
-			if err := e.decodeEntries(k, cc, dst); err != nil {
+			layer, err := e.decodeEntries(k, cc, dst)
+			if err != nil {
 				return fmt.Errorf("%s: %w", k.key, err)
 			}
 
+			if layer != nil {
+				supplied[k.key] = layer
+			}
+
 			continue
 		}
 
@@ -257,14 +278,29 @@ func (e *typedEntry[T]) decode(cc commandConfig) error {
 
 		if mergesByKey(f.Type(), e.b.opts.Markup) {
 			// The struct's own entries first, then sources lowest precedence first, so the highest wins a key.
-			vals = append(vals, f)
 			slices.Reverse(vals)
-			f.Set(mergeMaps(f.Type(), vals...))
+			t := f.Type()
+			supplied[k.key] = func(under reflect.Value) (reflect.Value, error) {
+				return mergeMaps(t, append([]reflect.Value{under}, vals...)...), nil
+			}
+			v, err := supplied[k.key](f)
+			if err != nil {
+				return fmt.Errorf("%s: %w", k.key, err)
+			}
+
+			f.Set(v)
 		} else {
+			supplied[k.key] = nil
 			f.Set(vals[0])
 		}
 	}
 
+	for _, apply := range e.profiles {
+		if err := apply(supplied); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

x/config/cli/examples/nested_profiles/README.md

@@ -0,0 +1,23 @@
+# Nested profiles
+
+[`profiles`](../profiles) registers the chain on its own, so its profile builds the whole struct. Here the chain is one
+section of a larger `Config`, registered whole, so a config file holds both `LogLevel` and `[Chain]`.
+`cli.ProfileWithSelector` picks the `Chain` section, and `Chain.ID` (a top-level field of that section) picks a function
+from `chains` in `main.go`. The profiles are the same as in `profiles`: they set the chain's `BlockTime`, and for chains 1
+and 137 its `RPC`, and only chain 137 replaces the default `Finality` of 64.
+
+A profile only builds its section: `LogLevel` comes from its default and the sources, whatever the chain.
+
+```sh
+# chain 1 and its profile; log-level keeps its default
+go run . --chain.id 1
+
+# a source outside the section, alongside a profile
+go run . --chain.id 137 --log-level debug
+
+# TOML values: log level and chain 137, with its own RPC
+go run . --config example.toml
+
+# each profile and its values
+go run . --help
+```

x/config/cli/examples/nested_profiles/appconfig/config.go

@@ -0,0 +1,27 @@
+package appconfig
+
+//go:generate go run ./gen
+
+import "github.com/smartcontractkit/chainlink-common/pkg/config"
+
+type ChainConfig struct {
+	// ID is the chain to connect to. A built-in chain fills in the fields below.
+	ID uint64 `validate:"required"`
+
+	// RPC is the node to dial.
+	RPC string `validate:"required"`
+
+	// BlockTime is how often the chain produces a block.
+	BlockTime config.Duration
+
+	// Finality is how many blocks until a block is final.
+	Finality uint32
+}
+
+type Config struct {
+	// LogLevel is how much to log. No profile touches it.
+	LogLevel string
+
+	// Chain is the chain configuration.
+	Chain ChainConfig
+}