feat: add forge normalize command (FORGE-1)

Cargo.toml

@@ -8,9 +8,10 @@ edition = "2024"
 
 [dependencies]
 clap = { version = "4.6.7", features = ["derive"], optional = true }
+memchr = { version = "2.8.3", optional = true }
 rayon = "1.12.0"
 serde = { version = "1.0.229", features = ["derive"] }
-serde_json = { version = "1.0.151", optional = true }
+serde_json = { version = "1.0.151", features = ["preserve_order"], optional = true }
 wasmtime = { version = "49.0.2", default-features = false, features = ["runtime", "cranelift", "component-model", "parallel-compilation", "std"], optional = true }
 wasmtime-wasi = { version = "49.0.2", default-features = false, features = ["p2"], optional = true }
 
@@ -26,7 +27,7 @@ harness = false
 default = ["cli"]
 # The `forge` command. Library users can turn it off with
 # `default-features = false`.
-cli = ["dep:clap", "dep:serde_json"]
+cli = ["dep:clap", "dep:memchr", "dep:serde_json"]
 # `forge::wasm`, which runs plugins compiled to WebAssembly components.
 wasm = ["dep:wasmtime", "dep:wasmtime-wasi"]
 

packages/forge/README.md

@@ -16,6 +16,77 @@ npx @cloudflare/forge openapi.json --target ts --out sdk
 Pass `-` instead of a path to read the document from standard input. Run
 `forge --help` for every option.
 
+## Normalizing a document
+
+`forge normalize` rewrites metadata extensions from other tools into Forge's
+`x-forge-*` vocabulary. This command does not change which extensions Forge
+reads; adopting the native vocabulary in the parser is a separate change.
+
+```sh
+# Print the document with its keys rewritten. Nothing is changed.
+npx forge normalize openapi.json
+
+# Rewrite the files in place.
+npx forge normalize --write specs/*.json
+
+# Rewrite only Fern compatibility keys. --from may be repeated.
+npx forge normalize --write --from fern specs/*.json
+
+# Keep the old keys beside the native ones, for what still reads them.
+npx forge normalize --write --leave-old-extensions openapi.json
+
+# Write nothing, and fail if anything would be rewritten: for a check in CI.
+npx forge normalize --check specs/*.json
+
+# Read from standard input and write to standard output.
+cat openapi.json | npx forge normalize -
+```
+
+Without `--write` it changes nothing and prints one document to standard
+output, so several files need `--write` or `--check`. A file with nothing to
+rewrite is left as it is, a link is followed and stays a link, and a read-only
+file is left as it is, with an error. `-v` says how many times each key was
+rewritten. Documents have to be JSON: YAML is not supported yet.
+
+By default every compatibility source is normalized. Use `--from cloudflare`,
+`--from deprecated`, `--from fern`, or `--from stainless` to select only that
+source; repeat `--from` to select more than one. For example,
+`--from fern --from cloudflare` leaves Stainless extensions untouched.
+`deprecated` is the migration path for old Forge keys after they are renamed;
+it currently has no keys.
+
+What is rewritten:
+
+- A group path written as an array becomes the names joined by dots
+  (`["a", "b"]` becomes `"a.b"`), unless a name has a dot in it.
+- A skip for only some targets, written as a list, has no native form, since
+  `x-forge-skip` is for every target: it is left as it is. A skip that is plainly
+  true or false is rewritten.
+- Native keys always win: when a document says the same thing twice, the native
+  key is kept.
+
+<details>
+<summary>Known limitations</summary>
+
+`forge normalize` reads a document as JSON and writes it again, so a file that
+has a key to rewrite is not a minimal edit. The order of its keys is kept.
+
+- **A key written twice** keeps only its last copy, with no error or warning.
+- **Numbers** that do not fit in 64 bits are rounded (`100000000000000000001`
+  becomes `1e+20`), a decimal of more than about 17 digits is shortened, and
+  `1E5`, `0.10` and `-0` are written `100000.0`, `0.1` and `-0.0`.
+- **Formatting** is the writer's: the indentation of the first indented line is
+  used for the whole file, arrays and objects on one line are written on
+  several, blank lines, spaces at the end of a line and anything before the
+  first `{` are dropped, `\r\n` becomes `\n`, and escapes such as `\u00e9` and
+  `\/` are written as the characters.
+- **Errors**: JSON with comments, a byte order mark, and a document nested
+  deeper than 128 levels are not read.
+- **Memory**: a document is held in memory at about 6.5 times its size, one file
+  at a time.
+
+</details>
+
 ## Plugins
 
 A plugin generates an SDK, or anything else, from the document: forge parses

src/bin/forge/main.rs

@@ -3,6 +3,7 @@
 //! ```text
 //! forge openapi.json --target ts --out sdk
 //! forge openapi.json --target wasm://plugin.wasm --out sdk
+//! forge normalize specs/*.json
 //! ```
 
 use std::fs;
@@ -13,21 +14,37 @@ use std::process::ExitCode;
 use clap::Parser;
 use forge::{Err, Forge};
 
+mod normalize;
 mod ts;
 
 /// Generates an SDK from an OpenAPI document.
 #[derive(Parser)]
-#[command(version)]
+#[command(version, args_conflicts_with_subcommands = true)]
 struct Args {
+    #[command(subcommand)]
+    command: Option<Command>,
     /// The OpenAPI JSON document, or - to read it from standard input.
-    spec: PathBuf,
+    #[arg(required = true)]
+    spec: Option<PathBuf>,
     /// The language of the SDK: `ts` for TypeScript, or `wasm://PATH` for
     /// the plugin compiled to a WebAssembly component at PATH.
-    #[arg(long, value_parser = Target::parse)]
-    target: Target,
+    #[arg(long, value_parser = Target::parse, required = true)]
+    target: Option<Target>,
     /// The directory to write the SDK to.
-    #[arg(long)]
-    out: PathBuf,
+    #[arg(long, required = true)]
+    out: Option<PathBuf>,
+}
+
+/// What `forge` does other than generate an SDK.
+#[derive(clap::Subcommand)]
+enum Command {
+    /// Rewrite other tools' extensions into the native x-forge-* ones.
+    ///
+    /// One document is printed to standard output, and nothing is changed,
+    /// unless --write is given, which rewrites the files in place. A document
+    /// that has none is left as it is. A native key is never replaced: when a
+    /// document says the same thing twice, the native key is kept.
+    Normalize(normalize::Args),
 }
 
 /// What forge generates an SDK with.
@@ -68,20 +85,32 @@ fn main() -> ExitCode {
     }
 }
 
-/// Generates the SDK that `args` ask for.
+/// Does what `args` ask for.
 fn run(args: Args) -> Result<(), Err> {
-    match args.target {
+    if let Some(Command::Normalize(args)) = args.command {
+        return normalize::run(args);
+    }
+    // They are required, which a command is the only way around.
+    let (Some(spec), Some(target), Some(out)) = (args.spec, args.target, args.out) else {
+        unreachable!("clap asks for a spec, a target and an out when there is no command");
+    };
+    generate(&spec, target, &out)
+}
+
+/// Generates the SDK for the document at `spec`, and writes it to `out`.
+fn generate(spec: &Path, target: Target, out: &Path) -> Result<(), Err> {
+    match target {
         Target::Ts => {
-            let forge = read(&args.spec)?;
-            forge::write(&args.out, forge::run(&forge, &ts::Generator)?)
+            let forge = read(spec)?;
+            forge::write(out, forge::run(&forge, &ts::Generator)?)
         }
         #[cfg(feature = "wasm")]
         Target::Wasm(path) => {
             // Compiling the plugin and reading the document are both slow,
             // so they run at the same time.
-            let (plugin, forge) = rayon::join(|| forge::wasm::WasmPlugin::from_file(&path), || read(&args.spec));
+            let (plugin, forge) = rayon::join(|| forge::wasm::WasmPlugin::from_file(&path), || read(spec));
             let (plugin, forge) = (plugin?, forge?);
-            forge::write(&args.out, forge::run(&forge, &plugin)?)
+            forge::write(out, forge::run(&forge, &plugin)?)
         }
     }
 }
@@ -104,11 +133,32 @@ fn read(path: &Path) -> Result<Forge, Err> {
 #[cfg(test)]
 mod tests {
     use clap::CommandFactory;
+    use clap::Parser;
+    use std::path::Path;
 
     use super::Args;
+    use super::Command;
 
     #[test]
     fn args() {
         Args::command().debug_assert();
     }
+
+    #[test]
+    fn a_command_and_a_document_are_told_apart() {
+        let parse = |args: &[&str]| Args::try_parse_from(std::iter::once("forge").chain(args.iter().copied()));
+
+        let command = parse(&["normalize", "--check", "a.json"]).unwrap();
+        assert!(matches!(command.command, Some(Command::Normalize(_))) && command.spec.is_none());
+
+        let document = parse(&["./normalize", "--target", "ts", "--out", "sdk"]).unwrap();
+        assert!(document.command.is_none() && document.spec.as_deref() == Some(Path::new("./normalize")));
+
+        // What a command or a document needs is not waived for the other.
+        assert!(parse(&["normalize"]).is_err());
+        assert!(parse(&["a.json"]).is_err());
+        assert!(parse(&["normalize", "--target", "ts", "a.json"]).is_err());
+        assert!(parse(&["--target", "ts", "normalize", "a.json"]).is_err());
+        assert!(parse(&[]).is_err());
+    }
 }