From 26df53f7619c81ebc2f5035176085f4da437e704 Mon Sep 17 00:00:00 2001 From: Niels Swimberghe Date: Fri, 30 Oct 2026 14:11:18 -0400 Subject: [PATCH 1/2] feat: add normalize command --- Cargo.lock | 2 + Cargo.toml | 5 +- packages/forge/README.md | 71 +++ src/bin/forge/main.rs | 74 ++- src/bin/forge/normalize.rs | 363 +++++++++++++++ src/lib.rs | 2 + src/normalize/mod.rs | 228 ++++++++++ src/normalize/vendor/README.md | 55 +++ src/normalize/vendor/cloudflare.rs | 6 + src/normalize/vendor/deprecated.rs | 15 + src/normalize/vendor/fern.rs | 25 ++ src/normalize/vendor/mod.rs | 95 ++++ src/normalize/vendor/stainless.rs | 9 + src/normalize/vendor/tests.rs | 100 +++++ src/normalize/walk.rs | 226 ++++++++++ tests/normalize/known_gaps.rs | 142 ++++++ tests/normalize/main.rs | 9 + tests/normalize/normalize_cli.rs | 692 +++++++++++++++++++++++++++++ 18 files changed, 2105 insertions(+), 14 deletions(-) create mode 100644 src/bin/forge/normalize.rs create mode 100644 src/normalize/mod.rs create mode 100644 src/normalize/vendor/README.md create mode 100644 src/normalize/vendor/cloudflare.rs create mode 100644 src/normalize/vendor/deprecated.rs create mode 100644 src/normalize/vendor/fern.rs create mode 100644 src/normalize/vendor/mod.rs create mode 100644 src/normalize/vendor/stainless.rs create mode 100644 src/normalize/vendor/tests.rs create mode 100644 src/normalize/walk.rs create mode 100644 tests/normalize/known_gaps.rs create mode 100644 tests/normalize/main.rs create mode 100644 tests/normalize/normalize_cli.rs diff --git a/Cargo.lock b/Cargo.lock index db4f44398..51d2bfcaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -574,6 +574,7 @@ version = "0.1.0" dependencies = [ "clap", "divan", + "memchr", "rayon", "serde", "serde_json", @@ -1228,6 +1229,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap", "itoa", "memchr", "serde", diff --git a/Cargo.toml b/Cargo.toml index 184531c7a..936278255 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,9 +8,10 @@ edition = "2024" [dependencies] clap = { version = "4.6.7", features = ["derive"], optional = true } +memchr = { version = "2.8.3", optional = true } rayon = "1.12.0" serde = { version = "1.0.229", features = ["derive"] } -serde_json = { version = "1.0.151", optional = true } +serde_json = { version = "1.0.151", features = ["preserve_order"], optional = true } wasmtime = { version = "49.0.2", default-features = false, features = ["runtime", "cranelift", "component-model", "parallel-compilation", "std"], optional = true } wasmtime-wasi = { version = "49.0.2", default-features = false, features = ["p2"], optional = true } @@ -26,7 +27,7 @@ harness = false default = ["cli"] # The `forge` command. Library users can turn it off with # `default-features = false`. -cli = ["dep:clap", "dep:serde_json"] +cli = ["dep:clap", "dep:memchr", "dep:serde_json"] # `forge::wasm`, which runs plugins compiled to WebAssembly components. wasm = ["dep:wasmtime", "dep:wasmtime-wasi"] diff --git a/packages/forge/README.md b/packages/forge/README.md index 39bc2163f..b50b84bab 100644 --- a/packages/forge/README.md +++ b/packages/forge/README.md @@ -16,6 +16,77 @@ npx @cloudflare/forge openapi.json --target ts --out sdk Pass `-` instead of a path to read the document from standard input. Run `forge --help` for every option. +## Normalizing a document + +`forge normalize` rewrites metadata extensions from other tools into Forge's +`x-forge-*` vocabulary. This command does not change which extensions Forge +reads; adopting the native vocabulary in the parser is a separate change. + +```sh +# Print the document with its keys rewritten. Nothing is changed. +npx forge normalize openapi.json + +# Rewrite the files in place. +npx forge normalize --write specs/*.json + +# Rewrite only Fern compatibility keys. --from may be repeated. +npx forge normalize --write --from fern specs/*.json + +# Keep the old keys beside the native ones, for what still reads them. +npx forge normalize --write --leave-old-extensions openapi.json + +# Write nothing, and fail if anything would be rewritten: for a check in CI. +npx forge normalize --check specs/*.json + +# Read from standard input and write to standard output. +cat openapi.json | npx forge normalize - +``` + +Without `--write` it changes nothing and prints one document to standard +output, so several files need `--write` or `--check`. A file with nothing to +rewrite is left as it is, a link is followed and stays a link, and a read-only +file is left as it is, with an error. `-v` says how many times each key was +rewritten. Documents have to be JSON: YAML is not supported yet. + +By default every compatibility source is normalized. Use `--from cloudflare`, +`--from deprecated`, `--from fern`, or `--from stainless` to select only that +source; repeat `--from` to select more than one. For example, +`--from fern --from cloudflare` leaves Stainless extensions untouched. +`deprecated` is the migration path for old Forge keys after they are renamed; +it currently has no keys. + +What is rewritten: + +- A group path written as an array becomes the names joined by dots + (`["a", "b"]` becomes `"a.b"`), unless a name has a dot in it. +- A skip for only some targets, written as a list, has no native form, since + `x-forge-skip` is for every target: it is left as it is. A skip that is plainly + true or false is rewritten. +- Native keys always win: when a document says the same thing twice, the native + key is kept. + +
+Known limitations + +`forge normalize` reads a document as JSON and writes it again, so a file that +has a key to rewrite is not a minimal edit. The order of its keys is kept. + +- **A key written twice** keeps only its last copy, with no error or warning. +- **Numbers** that do not fit in 64 bits are rounded (`100000000000000000001` + becomes `1e+20`), a decimal of more than about 17 digits is shortened, and + `1E5`, `0.10` and `-0` are written `100000.0`, `0.1` and `-0.0`. +- **Formatting** is the writer's: the indentation of the first indented line is + used for the whole file, arrays and objects on one line are written on + several, blank lines, spaces at the end of a line and anything before the + first `{` are dropped, `\r\n` becomes `\n`, and escapes such as `\u00e9` and + `\/` are written as the characters. +- **Errors**: JSON with comments, a byte order mark, and a document nested + deeper than 128 levels are not read. +- **Memory**: a document is held in memory at about 6.5 times its size, one file + at a time. + +
+ ## Plugins A plugin generates an SDK, or anything else, from the document: forge parses diff --git a/src/bin/forge/main.rs b/src/bin/forge/main.rs index 6781dd8ea..b036b487a 100644 --- a/src/bin/forge/main.rs +++ b/src/bin/forge/main.rs @@ -3,6 +3,7 @@ //! ```text //! forge openapi.json --target ts --out sdk //! forge openapi.json --target wasm://plugin.wasm --out sdk +//! forge normalize specs/*.json //! ``` use std::fs; @@ -13,21 +14,37 @@ use std::process::ExitCode; use clap::Parser; use forge::{Err, Forge}; +mod normalize; mod ts; /// Generates an SDK from an OpenAPI document. #[derive(Parser)] -#[command(version)] +#[command(version, args_conflicts_with_subcommands = true)] struct Args { + #[command(subcommand)] + command: Option, /// The OpenAPI JSON document, or - to read it from standard input. - spec: PathBuf, + #[arg(required = true)] + spec: Option, /// The language of the SDK: `ts` for TypeScript, or `wasm://PATH` for /// the plugin compiled to a WebAssembly component at PATH. - #[arg(long, value_parser = Target::parse)] - target: Target, + #[arg(long, value_parser = Target::parse, required = true)] + target: Option, /// The directory to write the SDK to. - #[arg(long)] - out: PathBuf, + #[arg(long, required = true)] + out: Option, +} + +/// What `forge` does other than generate an SDK. +#[derive(clap::Subcommand)] +enum Command { + /// Rewrite other tools' extensions into the native x-forge-* ones. + /// + /// One document is printed to standard output, and nothing is changed, + /// unless --write is given, which rewrites the files in place. A document + /// that has none is left as it is. A native key is never replaced: when a + /// document says the same thing twice, the native key is kept. + Normalize(normalize::Args), } /// What forge generates an SDK with. @@ -68,20 +85,32 @@ fn main() -> ExitCode { } } -/// Generates the SDK that `args` ask for. +/// Does what `args` ask for. fn run(args: Args) -> Result<(), Err> { - match args.target { + if let Some(Command::Normalize(args)) = args.command { + return normalize::run(args); + } + // They are required, which a command is the only way around. + let (Some(spec), Some(target), Some(out)) = (args.spec, args.target, args.out) else { + unreachable!("clap asks for a spec, a target and an out when there is no command"); + }; + generate(&spec, target, &out) +} + +/// Generates the SDK for the document at `spec`, and writes it to `out`. +fn generate(spec: &Path, target: Target, out: &Path) -> Result<(), Err> { + match target { Target::Ts => { - let forge = read(&args.spec)?; - forge::write(&args.out, forge::run(&forge, &ts::Generator)?) + let forge = read(spec)?; + forge::write(out, forge::run(&forge, &ts::Generator)?) } #[cfg(feature = "wasm")] Target::Wasm(path) => { // Compiling the plugin and reading the document are both slow, // so they run at the same time. - let (plugin, forge) = rayon::join(|| forge::wasm::WasmPlugin::from_file(&path), || read(&args.spec)); + let (plugin, forge) = rayon::join(|| forge::wasm::WasmPlugin::from_file(&path), || read(spec)); let (plugin, forge) = (plugin?, forge?); - forge::write(&args.out, forge::run(&forge, &plugin)?) + forge::write(out, forge::run(&forge, &plugin)?) } } } @@ -104,11 +133,32 @@ fn read(path: &Path) -> Result { #[cfg(test)] mod tests { use clap::CommandFactory; + use clap::Parser; + use std::path::Path; use super::Args; + use super::Command; #[test] fn args() { Args::command().debug_assert(); } + + #[test] + fn a_command_and_a_document_are_told_apart() { + let parse = |args: &[&str]| Args::try_parse_from(std::iter::once("forge").chain(args.iter().copied())); + + let command = parse(&["normalize", "--check", "a.json"]).unwrap(); + assert!(matches!(command.command, Some(Command::Normalize(_))) && command.spec.is_none()); + + let document = parse(&["./normalize", "--target", "ts", "--out", "sdk"]).unwrap(); + assert!(document.command.is_none() && document.spec.as_deref() == Some(Path::new("./normalize"))); + + // What a command or a document needs is not waived for the other. + assert!(parse(&["normalize"]).is_err()); + assert!(parse(&["a.json"]).is_err()); + assert!(parse(&["normalize", "--target", "ts", "a.json"]).is_err()); + assert!(parse(&["--target", "ts", "normalize", "a.json"]).is_err()); + assert!(parse(&[]).is_err()); + } } diff --git a/src/bin/forge/normalize.rs b/src/bin/forge/normalize.rs new file mode 100644 index 000000000..a799f5868 --- /dev/null +++ b/src/bin/forge/normalize.rs @@ -0,0 +1,363 @@ +//! `forge normalize`, which rewrites the extensions other tools' metadata is +//! written with into the native ones, in the files it is given. + +use std::collections::HashSet; +use std::fs; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; + +use forge::Err; +use forge::normalize::{Normalized, Options, Report, Source, Sources, normalize}; + +/// A compatibility source named on the command line. +#[derive(Clone, Copy, clap::ValueEnum)] +enum SourceArg { + Cloudflare, + Deprecated, + Fern, + Stainless, +} + +impl From for Source { + fn from(source: SourceArg) -> Source { + match source { + SourceArg::Cloudflare => Source::Cloudflare, + SourceArg::Deprecated => Source::Deprecated, + SourceArg::Fern => Source::Fern, + SourceArg::Stainless => Source::Stainless, + } + } +} + +/// What `forge normalize` is given, which says what it does in `main.rs`. +#[derive(clap::Args)] +pub struct Args { + /// The OpenAPI JSON documents, as paths. A pattern such as `specs/*.json` + /// is expanded by the shell. One document is printed to standard output, + /// with nothing changed, unless `--write` is given. Or - to read one from + /// standard input and print it. + #[arg(required = true)] + inputs: Vec, + /// Rewrite the files in place, each only if there is something to rewrite + /// in it. Without it nothing is changed, and one document is printed. + #[arg(short, long, conflicts_with = "check")] + write: bool, + /// Keep each key that is rewritten, beside the native key that replaces + /// it, for what still reads the old one. + #[arg(long)] + leave_old_extensions: bool, + /// Rewrite only keys from this compatibility source. May be repeated. + /// Without it, keys from every source are rewritten. + #[arg(long = "from", value_enum)] + from: Vec, + /// Say what would be rewritten, and write nothing. It fails if there is + /// anything to rewrite, to check that documents are normalized. + #[arg(long)] + check: bool, + /// Say how many times each key was rewritten. + #[arg(short, long)] + verbose: bool, +} + +/// A file to rewrite. +struct Input { + /// The path it was given by, which is how it is named. + shown: PathBuf, + /// The file it is, with every link followed, which is what is read and + /// written: a link stays a link, and a file given twice is rewritten once. + real: PathBuf, +} + +/// How a run says what it did. +#[derive(Clone, Copy)] +struct Say { + check: bool, + verbose: bool, + /// Whether the document is printed to standard output, and no file is + /// rewritten, so that is what is said. + printed: bool, +} + +pub fn run(args: Args) -> Result<(), Err> { + let sources = if args.from.is_empty() { + Sources::ALL + } else { + args.from.into_iter().map(Source::from).collect() + }; + let options = Options { + leave_old_extensions: args.leave_old_extensions, + sources, + }; + let say = Say { + check: args.check, + verbose: args.verbose, + printed: !args.write && !args.check, + }; + if args.inputs.iter().any(|input| input == "-") { + if args.inputs.len() > 1 { + return Err("- cannot be given with other inputs".into()); + } + if args.write { + return Err("- cannot be given with --write: standard input is not a file to rewrite".into()); + } + return filter(&options, say); + } + + let files = unique_files(&args.inputs)?; + if !args.write && !args.check { + // Nothing is changed, so there is one document to print. + let [file] = &files[..] else { + return Err("several files are only rewritten with --write, or checked with --check".into()); + }; + return print_file(file, &options, say); + } + + let (mut rewritten, mut unchanged, mut failed) = (0, 0, 0); + // One file at a time: each is held as a tree of several times its size. + for input in files { + let name = input.shown.display().to_string(); + let result = rewrite(&input.real, &options, say).map_err(|err| format!("{name}: {err}")); + match result { + Ok(report) => { + if report.changed() { + rewritten += 1; + } else { + unchanged += 1; + } + print(&name, &report, say); + } + Err(err) => { + failed += 1; + eprintln!("forge: {err}"); + } + } + } + let total = rewritten + unchanged; + if failed > 0 { + return Err(format!("{failed} {} could not be processed", plural(failed, "file", "files")).into()); + } + if say.check { + if rewritten > 0 { + return Err(format!( + "{rewritten} of {total} {} would be rewritten", + plural(total, "file", "files") + ) + .into()); + } + eprintln!( + "forge: {total} {} checked, none would be rewritten", + plural(total, "file", "files") + ); + } else { + eprintln!( + "forge: {total} {}: {rewritten} rewritten, {unchanged} unchanged", + plural(total, "file", "files") + ); + } + Ok(()) +} + +/// Returns `one` if `n` is 1, and `many` if it is not. +fn plural<'a>(n: usize, one: &'a str, many: &'a str) -> &'a str { + if n == 1 { one } else { many } +} + +/// Rewrites the document read from standard input to standard output. +fn filter(options: &Options, say: Say) -> Result<(), Err> { + let mut json = Vec::new(); + io::stdin().read_to_end(&mut json)?; + document(&json, "stdin", options, say) +} + +/// Prints the document in the file to standard output, rewritten, and leaves +/// the file as it is. +fn print_file(input: &Input, options: &Options, say: Say) -> Result<(), Err> { + let name = input.shown.display().to_string(); + reject_yaml(&input.real).map_err(|err| format!("{name}: {err}"))?; + let json = fs::read(&input.real).map_err(|err| format!("{name}: {err}"))?; + document(&json, &name, options, say) +} + +/// Rewrites `json`, the document called `name`, to standard output. +fn document(json: &[u8], name: &str, options: &Options, say: Say) -> Result<(), Err> { + let normalized = normalize(json, options).map_err(|err| format!("{name}: {err}"))?; + if say.check { + print(name, &normalized.report, say); + if normalized.report.changed() { + return Err("the document would be rewritten".into()); + } + return Ok(()); + } + let mut out = io::stdout().lock(); + match &normalized.json { + Some(edited) => out.write_all(edited)?, + None => out.write_all(json)?, + } + // Standard output is held until a line is whole, and what is left at the + // end, which is all of a document that is one line, is not otherwise + // known to have been written. + out.flush()?; + print(name, &normalized.report, say); + Ok(()) +} + +/// Returns the files `inputs` name, each once. A path that is not there is +/// left for reading it to say so, and a pattern such as `specs/*.json` is for +/// the shell to expand, so one that was not is a path that is not there. +fn unique_files(inputs: &[String]) -> Result, Err> { + let mut seen = HashSet::new(); + let mut files = Vec::new(); + for input in inputs { + if Path::new(input).is_dir() { + return Err(format!("{input}: is a folder, not a file").into()); + } + let shown = PathBuf::from(input); + let real = fs::canonicalize(&shown).unwrap_or_else(|_| shown.clone()); + if seen.insert(Identity::of(&real)) { + files.push(Input { shown, real }); + } + } + Ok(files) +} + +/// Which file a path is, so that a file with more than one name, which a hard +/// link is, is found to be one and rewritten once. +#[derive(PartialEq, Eq, Hash)] +enum Identity { + /// The device it is on, and where on it. + #[cfg(unix)] + Inode(u64, u64), + Path(PathBuf), +} + +impl Identity { + fn of(real: &Path) -> Identity { + #[cfg(unix)] + if let Ok(metadata) = fs::metadata(real) { + use std::os::unix::fs::MetadataExt; + + return Identity::Inode(metadata.dev(), metadata.ino()); + } + Identity::Path(real.to_owned()) + } +} + +/// Refuses the file at `path` if it is YAML, which is not supported yet. +fn reject_yaml(path: &Path) -> Result<(), Err> { + let yaml = path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("yml") || extension.eq_ignore_ascii_case("yaml")); + if yaml { + return Err("YAML is not supported yet: convert the document to JSON first".into()); + } + Ok(()) +} + +/// Rewrites the file at `path`, if it has a key to rewrite, and says what it +/// found, which is all it does when it is only checking. +fn rewrite(path: &Path, options: &Options, say: Say) -> Result { + reject_yaml(path)?; + let json = fs::read(path)?; + let Normalized { json, report } = normalize(&json, options)?; + if let Some(edited) = &json + && !say.check + { + if fs::metadata(path)?.permissions().readonly() { + return Err("the file is read-only, so it is not rewritten".into()); + } + write(path, edited)?; + } + Ok(report) +} + +/// Writes `json` to the file at `path`. It is written beside it first, and put +/// in its place, so that it is never left half written. Where a new file cannot +/// take the place of this one, because it has other links, which would be left +/// with the old one, or because its folder cannot be written to, it is written +/// in place. +fn write(path: &Path, edited: &[u8]) -> Result<(), Err> { + if linked(path) { + return Ok(write_in_place(path, edited)?); + } + match write_atomically(path, edited) { + Err(err) if err.kind() == io::ErrorKind::PermissionDenied => Ok(write_in_place(path, edited)?), + written => Ok(written?), + } +} + +fn write_in_place(path: &Path, edited: &[u8]) -> io::Result<()> { + fs::File::create(path)?.write_all(edited) +} + +/// Returns whether the file at `path` has other names. +#[cfg(unix)] +fn linked(path: &Path) -> bool { + use std::os::unix::fs::MetadataExt; + + fs::metadata(path).is_ok_and(|metadata| metadata.nlink() > 1) +} + +#[cfg(not(unix))] +fn linked(_: &Path) -> bool { + false +} + +fn write_atomically(path: &Path, edited: &[u8]) -> io::Result<()> { + let name = path.file_name().map(|name| name.to_string_lossy()).unwrap_or_default(); + let temporary = path.with_file_name(format!(".{name}.forge-normalize-{}", std::process::id())); + let permissions = fs::metadata(path)?.permissions(); + let written = create_private(&temporary) + .and_then(|mut file| { + // Before it has anything in it, so that it is not readable by more + // than the file it replaces is, at any time. + file.set_permissions(permissions)?; + file.write_all(edited) + }) + .and_then(|()| fs::rename(&temporary, path)); + if written.is_err() { + let _ = fs::remove_file(&temporary); + } + written +} + +/// Creates the file at `path`, which is not there, and is not a link to +/// another file that is, and that only its owner can read. +fn create_private(path: &Path) -> io::Result { + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + + options.mode(0o600); + } + options.open(path) +} + +/// Says what was done to the document called `name`, and what was not. +fn print(name: &str, report: &Report, say: Say) { + if report.changed() { + let total: usize = report.migrated.values().sum(); + let keys = plural(total, "key", "keys"); + if say.check { + eprintln!("{name}: would rewrite {total} {keys}"); + } else if say.printed { + eprintln!("{name}: {total} {keys} rewritten in the output"); + } else { + eprintln!("{name}: rewrote {total} {keys}"); + } + if say.verbose { + for (key, count) in &report.migrated { + eprintln!(" {key}: {count}"); + } + if report.joined > 0 { + eprintln!( + " {} group {} written as a dotted string", + report.joined, + plural(report.joined, "path", "paths") + ); + } + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 55e1611d5..7fbd30c40 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,6 +7,8 @@ mod json; mod link; mod method; mod name; +#[cfg(feature = "cli")] +pub mod normalize; mod plugin; mod types; pub use json::Json; diff --git a/src/normalize/mod.rs b/src/normalize/mod.rs new file mode 100644 index 000000000..53f4062ff --- /dev/null +++ b/src/normalize/mod.rs @@ -0,0 +1,228 @@ +//! Rewrites the extensions other tools' metadata is written with into the +//! native `x-forge-*` ones, in a JSON document. +//! +//! The key a tool writes is found with the tables of [`vendor`], at the kinds +//! of OpenAPI object where its native replacement belongs. A key elsewhere is +//! left alone, since rewriting it would give it a meaning it did not have. +//! +//! A native key is never replaced. When a document says the same thing with a +//! native key and another, the native one is kept. +//! +//! The document is read with `serde_json` and written again, in the order its +//! keys were in, with the indentation of its first indented line. That is +//! simple, and it is not a minimal edit: what is lost is listed in the README +//! of the package under "Known limitations", and each one is pinned by a test in +//! `tests/normalize/known_gaps.rs`. + +mod vendor; +mod walk; + +use std::collections::BTreeMap; +use std::fmt; + +use serde::Serialize; +use serde_json::Value; +use serde_json::ser::PrettyFormatter; + +/// A source of compatibility keys that [`normalize`] can rewrite. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Source { + /// Cloudflare's compatibility keys. + Cloudflare, + /// Deprecated Forge keys that have a current replacement. + Deprecated, + /// Fern's compatibility keys. + Fern, + /// Stainless's compatibility keys. + Stainless, +} + +impl Source { + const fn bit(self) -> u8 { + match self { + Source::Cloudflare => 1, + Source::Deprecated => 2, + Source::Fern => 4, + Source::Stainless => 8, + } + } +} + +/// The sources of compatibility keys that [`normalize`] rewrites. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Sources(u8); + +impl Sources { + /// Every compatibility source. + pub const ALL: Sources = Sources(1 | 2 | 4 | 8); + + /// Returns only `source`. + pub const fn only(source: Source) -> Sources { + Sources(source.bit()) + } + + /// Returns whether `source` is included. + pub const fn contains(self, source: Source) -> bool { + self.0 & source.bit() != 0 + } +} + +impl FromIterator for Sources { + fn from_iter>(sources: T) -> Sources { + Sources(sources.into_iter().fold(0, |bits, source| bits | source.bit())) + } +} + +/// What [`normalize`] does. +#[derive(Clone, Copy, Debug)] +pub struct Options { + /// Keeps each key that is rewritten, beside the native key that replaces + /// it, for what still reads the old one. + pub leave_old_extensions: bool, + /// The compatibility sources to rewrite. + pub sources: Sources, +} + +impl Default for Options { + fn default() -> Options { + Options { + leave_old_extensions: false, + sources: Sources::ALL, + } + } +} + +/// What [`normalize`] did. +#[derive(Debug, Default)] +pub struct Report { + /// How many keys were rewritten, or removed for a native key that + /// already said it, by the name of the key. + pub migrated: BTreeMap, + /// How many group paths were written again, as the names joined by dots, + /// which is the one value that is not copied as it was. + pub joined: usize, +} + +impl Report { + /// Returns whether the document is different from what it was. + pub fn changed(&self) -> bool { + !self.migrated.is_empty() + } +} + +/// A document after [`normalize`]. +#[derive(Debug)] +pub struct Normalized { + /// The document, if it is different from what was read. + pub json: Option>, + /// What was done to it. + pub report: Report, +} + +/// Why a document could not be normalized. +#[derive(Debug)] +pub struct Error(String); + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for Error {} + +impl From for Error { + fn from(err: serde_json::Error) -> Error { + Error(err.to_string()) + } +} + +/// Rewrites the keys of the document `json` that a native key replaces. +pub fn normalize(json: &[u8], options: &Options) -> Result { + let mut document: Value = serde_json::from_slice(json)?; + let report = walk::walk(&mut document, options); + if !report.changed() { + return Ok(Normalized { json: None, report }); + } + Ok(Normalized { + json: Some(write(&document, json)), + report, + }) +} + +/// Writes `document` as `original` was written, as far as it is known: with +/// its indentation, which is that of the first line that is indented, on one +/// line if it was, and with a newline at the end if it had one. +fn write(document: &Value, original: &[u8]) -> Vec { + let mut out = Vec::with_capacity(original.len() + original.len() / 16); + match indentation(original) { + Some(indent) => { + let mut serializer = serde_json::Serializer::with_formatter(&mut out, PrettyFormatter::with_indent(indent)); + document + .serialize(&mut serializer) + .expect("a document is written as JSON to the memory"); + } + None => serde_json::to_writer(&mut out, document).expect("a document is written as JSON to the memory"), + } + if original.ends_with(b"\n") { + out.push(b'\n'); + } + out +} + +/// Returns what `original` is indented with, or none if it is on one line. +fn indentation(original: &[u8]) -> Option<&[u8]> { + let first = memchr::memchr(b'\n', original)?; + for line in original[first + 1..].split(|byte| *byte == b'\n') { + let blanks = line.iter().take_while(|byte| matches!(byte, b' ' | b'\t')).count(); + if blanks > 0 && blanks < line.len() { + return Some(&line[..blanks]); + } + } + Some(b"") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn all_sources_includes_every_compatibility_adapter() { + for source in [Source::Cloudflare, Source::Deprecated, Source::Fern, Source::Stainless] { + assert!(Sources::ALL.contains(source), "{source:?}"); + } + } + + #[test] + fn the_indentation_is_that_of_the_first_line_that_is_indented() { + assert_eq!(indentation(b"{\n \"a\": {\n \"b\": 1\n }\n}"), Some(&b" "[..])); + assert_eq!(indentation(b"{\n\t\"a\": 1\n}"), Some(&b"\t"[..])); + assert_eq!(indentation(b"{\n \"a\": 1\n}"), Some(&b" "[..])); + } + + #[test] + fn a_document_on_one_line_is_written_on_one_line() { + assert_eq!(indentation(br#"{"a":1}"#), None); + assert_eq!( + write(&serde_json::json!({"a": 1, "b": [1, 2]}), br#"{"a":1}"#), + br#"{"a":1,"b":[1,2]}"# + ); + } + + #[test] + fn a_document_with_no_indented_line_is_written_with_none() { + assert_eq!(indentation(b"{\n\"a\": 1\n}\n"), Some(&b""[..])); + } + + #[test] + fn a_blank_line_with_spaces_is_not_the_indentation() { + assert_eq!(indentation(b"{\n \n \"a\": 1\n}"), Some(&b" "[..])); + } + + #[test] + fn the_newline_at_the_end_is_kept_if_there_was_one() { + let document = serde_json::json!({"a": 1}); + assert!(write(&document, b"{\n \"a\": 1\n}\n").ends_with(b"}\n")); + assert!(write(&document, b"{\n \"a\": 1\n}").ends_with(b"}")); + } +} diff --git a/src/normalize/vendor/README.md b/src/normalize/vendor/README.md new file mode 100644 index 000000000..85e841baa --- /dev/null +++ b/src/normalize/vendor/README.md @@ -0,0 +1,55 @@ +# The vendor tables + +`forge` reads one vocabulary: the native `x-forge-*` extensions, in the readers +of `src/link/`. The extensions other tools write +(Cloudflare's `x-sensitive`, Fern's `x-fern-*`, and Stainless's +`x-stainless-*`) are ordinary extensions to it, kept for plugins like any +other it does not read. This folder is the only place that knows them, their +source, and what they say: `forge normalize` uses it to rewrite them into +native keys, before `forge` reads the document. + +`deprecated` is also treated as a vendor here, though it is not literally one: +its adapter holds old Forge keys that have a current replacement. This keeps +renames out of the parser by giving them the same explicit migration path. + +## Rules + +- **Only this folder names a vendor extension.** That includes the readers, + the linker, the plugin host, the WIT, the `packages/forge` npm package, and + their comments and docs. `tests/boundary.rs` fails, with file and line, if + one appears elsewhere (`just boundary`; it also runs with `cargo test`). + The tests of vendor-keyed documents live in `tests/normalize/`. +- **Mapping, not semantics.** A table says which native key a key becomes, + and whether its value is written as the native key's is, as data. + Anything structural, such as how aliases become methods, belongs to the + builder, not here. The one value that is written again is a group path, which + becomes the names joined by dots (`walk.rs`). +- **One adapter per source.** `--from` selects the `cloudflare`, `deprecated`, + `fern`, and `stainless` adapters, and no `--from` selects them all. The + temporary in-memory compatibility bridge always uses all adapters, so it + preserves what the old parser read. +- **A key is read only at the level `forge` reads its native key at**: an + operation or an alias, a parameter, or a schema. A key of an object `forge` + does not read it at is left alone, since rewriting it would give it a + meaning it did not have. +- **Native wins.** When a document has the native key, the vendor one is + dropped, whatever order they appear in. When a vendor key has no native form, it is left as it is written: on a + property, `x-stainless-skip: true` and `false` become `x-forge-skip` with the + same value, but a list of targets such as `["terraform"]` says "skip for + these targets", which `x-forge-skip` cannot, as it is for every target. On an + operation, `x-stainless-skip` is not read at all. + +## Adding a vendor key + +1. Find the native key it stands for in the readers of `src/link/`. If there + is none, add a native `x-forge-*` key to the reader first: a vendor key + never introduces a primitive. +2. Map it in the table of the adapter for its source, or add an adapter next to + `cloudflare.rs`, `deprecated.rs`, `fern.rs` and `stainless.rs` and ask it + from `mod.rs`. A test fails if the table names a key that `forge` does not + read at that kind of object. A renamed Forge key goes in `deprecated.rs` + and its exact old name goes in the table for that object kind. +3. Add a test in `tests/normalize/` showing that the vendor-keyed document, + after `forge normalize`, builds the same `Forge` as the native-keyed one. + +`forge` itself needs no change: it never learns the key. diff --git a/src/normalize/vendor/cloudflare.rs b/src/normalize/vendor/cloudflare.rs new file mode 100644 index 000000000..3c75c6841 --- /dev/null +++ b/src/normalize/vendor/cloudflare.rs @@ -0,0 +1,6 @@ +//! The adapter for Cloudflare's compatibility extensions. + +use super::Key; + +/// The keys of a Schema Object. +pub(super) const SCHEMA: [(&str, Key); 1] = [("x-sensitive", Key::same("x-forge-sensitive"))]; diff --git a/src/normalize/vendor/deprecated.rs b/src/normalize/vendor/deprecated.rs new file mode 100644 index 000000000..451d04d77 --- /dev/null +++ b/src/normalize/vendor/deprecated.rs @@ -0,0 +1,15 @@ +//! The adapter for deprecated Forge extensions that have a current name. +//! +//! It starts empty. A rename adds the old key to the table for its kind of +//! object and maps it to the current native key. + +use super::Key; + +/// Deprecated keys of an operation or alias. +pub(super) const DEFINITION: [(&str, Key); 0] = []; + +/// Deprecated keys of a Parameter Object. +pub(super) const PARAMETER: [(&str, Key); 0] = []; + +/// Deprecated keys of a Schema Object. +pub(super) const SCHEMA: [(&str, Key); 0] = []; diff --git a/src/normalize/vendor/fern.rs b/src/normalize/vendor/fern.rs new file mode 100644 index 000000000..a9a59f1b4 --- /dev/null +++ b/src/normalize/vendor/fern.rs @@ -0,0 +1,25 @@ +//! The adapter for Fern's `x-fern-*` extensions. +//! +//! A group path written as one string is split on dots, as for +//! `x-forge-group-name`, which Fern itself does not do: Fern reads a string as a +//! single group. A document written for Fern uses the array, which means the +//! same in both. + +use super::Key; + +/// The keys of an operation or an alias. +pub(super) const DEFINITION: [(&str, Key); 4] = [ + ("x-fern-sdk-method-name", Key::same("x-forge-method-name")), + ("x-fern-sdk-group-name", Key::group("x-forge-group-name")), + ("x-fern-ignore", Key::same("x-forge-skip")), + ("x-fern-availability", Key::same("x-forge-availability")), +]; + +/// The keys of a Parameter Object. +pub(super) const PARAMETER: [(&str, Key); 1] = [("x-fern-parameter-name", Key::same("x-forge-parameter-name"))]; + +/// The keys of a Schema Object. +pub(super) const SCHEMA: [(&str, Key); 2] = [ + ("x-fern-type-name", Key::same("x-forge-model-name")), + ("x-fern-property-name", Key::same("x-forge-property-name")), +]; diff --git a/src/normalize/vendor/mod.rs b/src/normalize/vendor/mod.rs new file mode 100644 index 000000000..b6fa80984 --- /dev/null +++ b/src/normalize/vendor/mod.rs @@ -0,0 +1,95 @@ +//! The keys other tools' metadata is written with, and the native extension +//! that each says the same as. This is the only code that holds them: `forge` +//! normalization keeps isolated from the rest of Forge. +//! +//! Each adapter, [`cloudflare`], [`deprecated`], [`fern`] and [`stainless`], is +//! a table per kind of object that maps its compatibility keys onto native +//! ones, and says whether the value is written as the native key's is. +//! +//! A key has to be in the table of the kind of object `forge` reads its native +//! key at, so that a rewrite does not give a key a meaning it did not have. An +//! extension that says something `forge` has no primitive for is not here. + +mod cloudflare; +mod deprecated; +mod fern; +mod stainless; +#[cfg(test)] +mod tests; + +use super::{Source, Sources}; + +/// What a key of another tool becomes. +#[derive(Clone, Copy)] +pub(super) struct Key { + /// The native key that says the same. + pub(super) native: &'static str, + /// How the value is written when it is written as the native key's is. + pub(super) form: Form, +} + +/// How the value of a key is written as the native key's is. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum Form { + /// As it is. + Same, + /// A group path: an array of names is the names joined by dots. + Group, + /// A boolean, or a list of the targets, which a native key does not say. + BoolOrTargets, +} + +impl Key { + /// Returns a key whose value is written as the native key's is. + const fn same(native: &'static str) -> Key { + Key { + native, + form: Form::Same, + } + } + + /// Returns a key whose value is a group path. + const fn group(native: &'static str) -> Key { + Key { + native, + form: Form::Group, + } + } + + /// Returns a key whose value is a boolean, or a list of targets. + const fn bool_or_targets(native: &'static str) -> Key { + Key { + native, + form: Form::BoolOrTargets, + } + } +} + +/// Returns what `key` becomes on an operation or an alias, if an adapter reads +/// it there. +pub(super) fn definition(key: &str, sources: Sources) -> Option { + find(Source::Deprecated, sources, &deprecated::DEFINITION, key) + .or_else(|| find(Source::Fern, sources, &fern::DEFINITION, key)) +} + +/// Returns what `key` becomes on a Parameter Object, if an adapter reads it +/// there. +pub(super) fn parameter(key: &str, sources: Sources) -> Option { + find(Source::Deprecated, sources, &deprecated::PARAMETER, key) + .or_else(|| find(Source::Fern, sources, &fern::PARAMETER, key)) +} + +/// Returns what `key` becomes on a Schema Object, if an adapter reads it there. +pub(super) fn schema(key: &str, sources: Sources) -> Option { + find(Source::Deprecated, sources, &deprecated::SCHEMA, key) + .or_else(|| find(Source::Cloudflare, sources, &cloudflare::SCHEMA, key)) + .or_else(|| find(Source::Fern, sources, &fern::SCHEMA, key)) + .or_else(|| find(Source::Stainless, sources, &stainless::SCHEMA, key)) +} + +fn find(source: Source, sources: Sources, table: &[(&str, Key)], key: &str) -> Option { + if !sources.contains(source) { + return None; + } + table.iter().find(|(name, _)| *name == key).map(|(_, key)| *key) +} diff --git a/src/normalize/vendor/stainless.rs b/src/normalize/vendor/stainless.rs new file mode 100644 index 000000000..144cb2e1b --- /dev/null +++ b/src/normalize/vendor/stainless.rs @@ -0,0 +1,9 @@ +//! The adapter for Stainless's `x-stainless-*` extensions. +//! +//! `x-stainless-skip` may be a list of the targets to skip a property for, +//! which a native skip, that is for every target, does not say. + +use super::Key; + +/// The keys of a Schema Object. +pub(super) const SCHEMA: [(&str, Key); 1] = [("x-stainless-skip", Key::bool_or_targets("x-forge-skip"))]; diff --git a/src/normalize/vendor/tests.rs b/src/normalize/vendor/tests.rs new file mode 100644 index 000000000..08bdc66ae --- /dev/null +++ b/src/normalize/vendor/tests.rs @@ -0,0 +1,100 @@ +//! The vendor tables. They are here, and not in `tests/normalize/`, as they are +//! of what is private to `normalize`, and name the keys of other tools, which +//! only this module may. + +use super::*; + +/// A kind of object that has keys of other tools. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Level { + Definition, + Parameter, + Schema, +} + +/// Every key of the tables, with the kind of object it is read at. +fn tables() -> Vec<(Source, Level, &'static str, Key)> { + let table = |source: Source, level: Level, table: &[(&'static str, Key)]| -> Vec<_> { + table.iter().map(|&(name, key)| (source, level, name, key)).collect() + }; + [ + table(Source::Cloudflare, Level::Schema, &cloudflare::SCHEMA), + table(Source::Deprecated, Level::Definition, &deprecated::DEFINITION), + table(Source::Deprecated, Level::Parameter, &deprecated::PARAMETER), + table(Source::Deprecated, Level::Schema, &deprecated::SCHEMA), + table(Source::Fern, Level::Definition, &fern::DEFINITION), + table(Source::Fern, Level::Parameter, &fern::PARAMETER), + table(Source::Fern, Level::Schema, &fern::SCHEMA), + table(Source::Stainless, Level::Schema, &stainless::SCHEMA), + ] + .concat() +} + +fn read(level: Level, key: &str, sources: Sources) -> Option { + match level { + Level::Definition => definition(key, sources), + Level::Parameter => parameter(key, sources), + Level::Schema => schema(key, sources), + } +} + +#[test] +fn a_key_is_read_only_at_its_level() { + for (_, level, name, _) in tables() { + for other in [Level::Definition, Level::Parameter, Level::Schema] { + assert_eq!( + read(other, name, Sources::ALL).is_some(), + other == level, + "{name} at {other:?}" + ); + } + } +} + +#[test] +fn a_key_is_read_only_when_its_source_is_selected() { + let sources = [Source::Cloudflare, Source::Deprecated, Source::Fern, Source::Stainless]; + for (source, level, name, _) in tables() { + for selected in sources { + assert_eq!( + read(level, name, Sources::only(selected)).is_some(), + selected == source, + "{name} from {source:?}, with {selected:?} selected" + ); + } + } +} + +#[test] +fn what_a_key_becomes_is_a_native_key_and_no_key_of_another_tool_is_one() { + for (_, _, name, key) in tables() { + assert!(key.native.starts_with("x-forge-"), "{name} becomes {}", key.native); + assert!(!name.starts_with("x-forge-"), "{name}"); + } +} + +#[test] +fn only_the_skip_of_a_schema_that_stainless_writes_may_be_a_list_of_targets() { + for (_, _, name, key) in tables() { + assert_eq!(key.form == Form::BoolOrTargets, name == "x-stainless-skip", "{name}"); + } +} + +#[test] +fn the_other_extensions_of_these_tools_are_not_read() { + for key in [ + "x-fern-enum", + "x-fern-audiences", + "x-fern-pagination", + "x-fern-sdk-variables", + "x-stainless-terraform-configurability", + "x-stainless-deprecation-message", + // The native keys, which are not for a rewrite. + "x-forge-skip", + "x-forge-method-name", + ] { + assert!(definition(key, Sources::ALL).is_none(), "{key}"); + assert!(parameter(key, Sources::ALL).is_none(), "{key}"); + assert!(schema(key, Sources::ALL).is_none(), "{key}"); + } +} diff --git a/src/normalize/walk.rs b/src/normalize/walk.rs new file mode 100644 index 000000000..da00b587f --- /dev/null +++ b/src/normalize/walk.rs @@ -0,0 +1,226 @@ +//! What a document means, which is where a key is to be rewritten. +//! +//! The walk follows the document as `forge` reads it: down from `paths` and +//! `components` to the operations, the parameters and the schemas, which are +//! the objects that have keys to rewrite, and no further. Everything else is +//! passed over. What is below an object is found by what kind of object it is +//! and what the key is, in [`child`], which is the one place that says where +//! `forge` reads. + +use serde_json::{Map, Value}; + +use super::vendor::{self, Form, Key}; +use super::{Options, Report}; + +/// Returns the names of an array joined by dots, if each is a name that a +/// dotted string reads back as it is. +fn dotted(names: &[Value]) -> Option { + let mut path = String::new(); + for (i, name) in names.iter().enumerate() { + let name = name.as_str().filter(|name| !name.is_empty() && !name.contains('.'))?; + if i > 0 { + path.push('.'); + } + path.push_str(name); + } + Some(path) +} + +/// A kind of object that is read. +#[derive(Clone, Copy)] +enum Kind { + Document, + Components, + PathItem, + Operation, + /// An alias of an operation, which has the keys of one. + Alias, + Parameter, + /// A request body or a response, which has nothing but a `content` to read. + Body, + /// A media type, which has nothing but a `schema` to read. + Media, + Schema, +} + +/// How what is read is held by the key that has it. +#[derive(Clone, Copy)] +enum Shape { + /// The value is one object. + One(Kind), + /// The value is an object, and each of its members is. + Values(Kind), + /// The value is an object, and each member that is not an extension is. + Entries(Kind), + /// The value is an array, and each of its elements is. + Elements(Kind), +} + +/// Returns what the member `key` of an object of the kind `kind` holds, if +/// `forge` reads it. +fn child(kind: Kind, key: &str, value: &Value) -> Option { + use Kind::*; + use Shape::*; + Some(match (kind, key) { + (Document, "paths") => Entries(PathItem), + (Document, "components") => One(Components), + (Components, "schemas") => Values(Schema), + (Components, "parameters") => Values(Parameter), + (Components, "requestBodies" | "responses") => Values(Body), + (Components, "pathItems") => Values(PathItem), + (PathItem, "parameters") => Elements(Parameter), + ( + PathItem, + "get" | "put" | "post" | "delete" | "options" | "head" | "patch" | "trace" | "query" | "connect", + ) => One(Operation), + (PathItem, "additionalOperations") => Values(Operation), + (Operation, "parameters") => Elements(Parameter), + (Operation, "requestBody") => One(Body), + (Operation, "responses") => Entries(Body), + (Operation, "x-forge-aliases") => Elements(Alias), + (Parameter, "schema") => One(Schema), + (Parameter, "content") => Values(Media), + (Body, "content") => Values(Media), + (Media, "schema") => One(Schema), + (Schema, "properties" | "$defs") => Values(Schema), + (Schema, "additionalProperties" | "unevaluatedProperties") => One(Schema), + (Schema, "items") if value.is_object() => One(Schema), + (Schema, "items" | "allOf" | "anyOf" | "oneOf") => Elements(Schema), + _ => return None, + }) +} + +/// What is done to a member of an object. +enum Action { + /// The key is written as the native key, with its value, or with this one. + Rename(&'static str, Option), + /// The member is removed, since a native key says what it says. + Remove, + /// The native key is written before it, with its value, or with this one, + /// and it stays. + Insert(&'static str, Option), +} + +/// Walks a document, which is changed, and returns what was done. +pub(super) fn walk(document: &mut Value, options: &Options) -> Report { + let mut walk = Walk { + options, + report: Report::default(), + }; + walk.visit(Kind::Document, document); + walk.report +} + +struct Walk<'o> { + options: &'o Options, + report: Report, +} + +impl Walk<'_> { + /// Reads an object of the kind `kind`: its keys that are to be rewritten + /// are, and then what is in it that is read. + fn visit(&mut self, kind: Kind, value: &mut Value) { + let Some(members) = value.as_object_mut() else { + return; + }; + match kind { + Kind::Operation | Kind::Alias => self.rewrite(members, vendor::definition), + Kind::Parameter => self.rewrite(members, vendor::parameter), + Kind::Schema => self.rewrite(members, vendor::schema), + Kind::Document | Kind::Components | Kind::PathItem | Kind::Body | Kind::Media => {} + } + for (key, member) in members.iter_mut() { + let Some(shape) = child(kind, key, member) else { + continue; + }; + match shape { + Shape::One(kind) => self.visit(kind, member), + Shape::Values(kind) => { + if let Some(members) = member.as_object_mut() { + for member in members.values_mut() { + self.visit(kind, member); + } + } + } + Shape::Entries(kind) => { + if let Some(members) = member.as_object_mut() { + for (key, member) in members { + if !key.starts_with("x-") { + self.visit(kind, member); + } + } + } + } + Shape::Elements(kind) => { + if let Some(elements) = member.as_array_mut() { + for element in elements.iter_mut() { + self.visit(kind, element); + } + } + } + } + } + } + + /// Decides what to do to the keys of the object that a native key replaces, + /// as `read` says which they are for the kind of object it is, and does it. + fn rewrite(&mut self, members: &mut Map, read: fn(&str, super::Sources) -> Option) { + let mut actions: Vec<(String, Action)> = Vec::new(); + for (key, value) in members.iter() { + if !key.starts_with("x-") { + continue; + } + let Some(hit) = read(key, self.options.sources) else { + continue; + }; + // A list of targets is something no native key says: it is left + // as it is written, and not for a rewrite to say some other way + // or to drop. + if hit.form == Form::BoolOrTargets && value.is_array() { + continue; + } + // The native key that is beside is the one that decides. + let writes = !members.contains_key(hit.native); + let written = match value { + Value::Array(names) if writes && hit.form == Form::Group => dotted(names).map(Value::String), + _ => None, + }; + if written.is_some() { + self.report.joined += 1; + } + let action = match (self.options.leave_old_extensions, writes) { + (false, true) => Action::Rename(hit.native, written), + (false, false) => Action::Remove, + (true, true) => Action::Insert(hit.native, written), + (true, false) => continue, + }; + *self.report.migrated.entry(key.clone()).or_insert(0) += 1; + actions.push((key.clone(), action)); + } + if actions.is_empty() { + return; + } + + // The members are written again in the order they were in. If two keys + // of other tools said the same native field, the last would win here, + // where a rewrite of bytes would have written the native key twice, for + // `forge` to reject. No table has two keys for a field today. + for (key, value) in std::mem::take(members) { + match actions.iter().position(|(name, _)| *name == key) { + None => { + members.insert(key, value); + } + Some(at) => match actions.swap_remove(at).1 { + Action::Rename(native, written) => { + members.insert(native.to_owned(), written.unwrap_or(value)); + } + Action::Remove => {} + Action::Insert(native, written) => { + members.insert(native.to_owned(), written.unwrap_or_else(|| value.clone())); + members.insert(key, value); + } + }, + } + } + } +} diff --git a/tests/normalize/known_gaps.rs b/tests/normalize/known_gaps.rs new file mode 100644 index 000000000..3883525b3 --- /dev/null +++ b/tests/normalize/known_gaps.rs @@ -0,0 +1,142 @@ +//! What `forge normalize` gets wrong because a document is read with +//! `serde_json` and written again, as it happens today. +//! +//! Each test says what a document becomes, which is not what it should be, and +//! passes. When one of them fails, a gap was fixed: move it out of here and +//! out of "Known limitations" in the README of the package, if it is in it. + +use forge::normalize::{Options, normalize}; + +/// A document with one key to rewrite, so that it is written again, and +/// `extra` as more of the members of the operation. +fn with_a_key_to_rewrite(extra: &str) -> String { + format!( + "{{\n \"openapi\": \"3.0.3\",\n \"info\": {{\"title\": \"t\", \"version\": \"1\"}},\n \"paths\": {{\"/a\": {{\"get\": {{\n \"x-fern-sdk-method-name\": \"get\",\n {extra}\n \"responses\": {{\"204\": {{\"description\": \"ok\"}}}}}}}}}}\n}}\n" + ) +} + +#[track_caller] +fn rewritten(document: &str) -> String { + let normalized = normalize(document.as_bytes(), &Options::default()).unwrap(); + String::from_utf8(normalized.json.expect("there is a key to rewrite")).unwrap() +} + +/// GAP: a key written twice is not an error, and not left: the first of the +/// two is dropped, without a word. `forge` itself says `duplicate field`. +#[test] +fn a_key_written_twice_loses_the_first_copy_without_a_word() { + let text = rewritten(&with_a_key_to_rewrite( + r#""description": "first", "description": "second","#, + )); + + assert!(text.contains("second") && !text.contains("first"), "{text}"); +} + +/// GAP: a number is read as a 64-bit float, or an integer that fits in 64 +/// bits, so a longer one is rounded, and one written as `1E5` is written as +/// `100000.0`. `forge` reads them as floats too, but the document is what is +/// rewritten, and what is read by every other tool. +#[test] +fn a_number_that_does_not_fit_is_rounded_and_the_others_are_written_their_own_way() { + let text = rewritten(&with_a_key_to_rewrite( + r#""x-ids": [100000000000000000001, 3.14159265358979323846, 1E5, 0.10, 2.50],"#, + )); + + assert!(text.contains("1e+20"), "{text}"); + assert!( + text.contains("3.141592653589793") && !text.contains("3.1415926535897932"), + "{text}" + ); + assert!(text.contains("100000.0") && !text.contains("1E5"), "{text}"); + assert!(text.contains("0.1") && !text.contains("0.10"), "{text}"); + assert!(text.contains("2.5") && !text.contains("2.50"), "{text}"); +} + +/// GAP: line endings are not kept: a document with `\r\n` is written with +/// `\n`, so every line of it is in the diff. +#[test] +fn windows_line_endings_are_written_as_unix_ones() { + let text = rewritten(&with_a_key_to_rewrite("").replace('\n', "\r\n")); + + assert!(!text.contains('\r'), "{text:?}"); +} + +/// GAP: an array or an object that is written on one line is written on +/// several, which is a diff of every one of them in a document that is +/// formatted that way, as most formatters of JSON do. +#[test] +fn an_array_on_one_line_is_written_on_several() { + let text = rewritten(&with_a_key_to_rewrite(r#""tags": ["a", "b"],"#)); + + assert!(!text.contains(r#"["a", "b"]"#), "{text}"); + assert!(text.contains("\"tags\": [\n"), "{text}"); +} + +/// GAP: a document on one line is written on one line, with no space after a +/// comma or a colon, which is not how it was written if it had them. +#[test] +fn spaces_in_a_document_on_one_line_are_not_kept() { + let document = r#"{"openapi": "3.0.3", "info": {"title": "t", "version": "1"}, "paths": {"/a": {"get": {"x-fern-ignore": true, "responses": {"204": {"description": "ok"}}}}}}"#; + + let text = rewritten(document); + + assert!(!text.contains(", ") && !text.contains(": "), "{text}"); +} + +/// GAP: an escape is written as the character, so `\u00e9`, `\/` and `\u003c` +/// are not what they were, and a document with them is a diff of every line +/// that has one. +#[test] +fn escapes_in_strings_are_written_as_the_characters() { + let text = rewritten(&with_a_key_to_rewrite( + r#""description": "caf\u00e9 a\/b \u003ctag\u003e","#, + )); + + assert!(text.contains("café a/b "), "{text}"); +} + +/// GAP: JSON with comments, which some call JSONC, is not JSON, and is an error. +#[test] +fn json_with_comments_is_an_error() { + let err = normalize(b"// api\n{\"openapi\": \"3.0.3\"}", &Options::default()).unwrap_err(); + + assert!(!err.to_string().is_empty()); +} + +/// GAP: a document nested deeper than `serde_json` allows, 128 levels, is an +/// error, wherever it is nested. +#[test] +fn a_document_nested_deeper_than_128_levels_is_an_error() { + let document = format!( + r#"{{"x-fern-ignore": true, "x-other": {}{}}}"#, + "[".repeat(200), + "]".repeat(200) + ); + + let err = normalize(document.as_bytes(), &Options::default()).unwrap_err(); + + assert!(err.to_string().contains("recursion limit"), "{err}"); +} + +/// GAP: the whitespace outside strings is the writer's: one indentation for +/// the file, no blank lines, no spaces at the end of a line, nothing before the +/// first `{`, and one newline at the end. +#[test] +fn whitespace_is_the_writers() { + let document = "\n\n {\n \"openapi\": \"3.0.3\", \n\t\"paths\": {\"/a\": {\"get\": {\n\n \"x-fern-ignore\": true}}}\n} \n\n\n"; + + let text = rewritten(document); + + assert_eq!( + text, + "{\n \"openapi\": \"3.0.3\",\n \"paths\": {\n \"/a\": {\n \"get\": {\n \"x-forge-skip\": true\n }\n }\n }\n}\n" + ); +} + +/// GAP: `-0` is read as a float, and written as one. +#[test] +fn minus_zero_is_written_as_a_float() { + let text = rewritten(&with_a_key_to_rewrite(r#""x-n": -0,"#)); + + assert!(text.contains("-0.0"), "{text}"); +} diff --git a/tests/normalize/main.rs b/tests/normalize/main.rs new file mode 100644 index 000000000..fcf070fed --- /dev/null +++ b/tests/normalize/main.rs @@ -0,0 +1,9 @@ +//! Tests for `forge normalize`: the keys other tools write, rewritten into the +//! native extensions that say the same things. What the rewrite gets wrong, +//! because a document is read and written again, is in `known_gaps`. + +#![cfg(feature = "cli")] + +mod known_gaps; +#[cfg(feature = "cli")] +mod normalize_cli; diff --git a/tests/normalize/normalize_cli.rs b/tests/normalize/normalize_cli.rs new file mode 100644 index 000000000..14848580d --- /dev/null +++ b/tests/normalize/normalize_cli.rs @@ -0,0 +1,692 @@ +//! `forge normalize`, the command, on files and on standard input. + +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; + +use serde_json::Value; + +/// A document with one operation, written on one line, with no final newline. +const FERN: &str = r#"{"openapi":"3.0.3","info":{"title":"t","version":"1"},"paths":{"/a":{"get":{"x-fern-sdk-group-name":["a"],"x-fern-sdk-method-name":"get","responses":{"204":{"description":"ok"}}}}}}"#; + +/// The same document, written native. +const NATIVE: &str = r#"{"openapi":"3.0.3","info":{"title":"t","version":"1"},"paths":{"/a":{"get":{"x-forge-group-name":"a","x-forge-method-name":"get","responses":{"204":{"description":"ok"}}}}}}"#; + +/// A document with a key from each compatibility source. +const SOURCES: &str = r##"{"openapi":"3.0.3","info":{"title":"t","version":"1"},"paths":{"/a":{"get":{"x-fern-sdk-group-name":["a"],"x-fern-sdk-method-name":"get","responses":{"200":{"description":"ok","content":{"application/json":{"schema":{"$ref":"#/components/schemas/S"}}}}}}}},"components":{"schemas":{"S":{"type":"object","properties":{"secret":{"type":"string","x-sensitive":true},"hidden":{"type":"string","x-stainless-skip":true}}}}}}"##; + +/// A folder of its own to write files in, which is empty. +fn dir(name: &str) -> PathBuf { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("normalize").join(name); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir +} + +/// Runs `forge normalize` with `args`, writing `stdin` to it. +fn normalize(args: &[&str], stdin: &str) -> Output { + forge(&[&["normalize"], args].concat(), stdin) +} + +/// Runs `forge normalize --write` with `args`, which rewrites the files in place. +fn rewrite(args: &[&str], stdin: &str) -> Output { + normalize(&[&["--write"], args].concat(), stdin) +} + +/// Runs `forge` with `args`, writing `stdin` to it. +fn forge(args: &[&str], stdin: &str) -> Output { + let mut child = Command::new(env!("CARGO_BIN_EXE_forge")) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + // Beside the output being read: a child that has written more than a pipe + // holds is not reading its input until that is read. + let mut input = child.stdin.take().unwrap(); + let stdin = stdin.to_owned(); + // A child that exits before it has read it all is not a failure to write. + let writer = std::thread::spawn(move || input.write_all(stdin.as_bytes())); + let output = child.wait_with_output().unwrap(); + let _ = writer.join().unwrap(); + output +} + +fn text(bytes: &[u8]) -> &str { + std::str::from_utf8(bytes).unwrap() +} + +#[track_caller] +fn succeeded(out: &Output) { + assert!(out.status.success(), "forge failed: {}", text(&out.stderr)); +} + +#[track_caller] +fn failed(out: &Output) -> &str { + assert!(!out.status.success(), "forge succeeded"); + text(&out.stderr) +} + +#[test] +fn a_file_is_rewritten_in_place_as_it_was_written() { + let dir = dir("in-place"); + let file = dir.join("spec.json"); + fs::write(&file, FERN).unwrap(); + + let out = rewrite(&[file.to_str().unwrap()], ""); + + succeeded(&out); + assert_eq!( + fs::read_to_string(&file).unwrap(), + NATIVE, + "still on one line, with no final newline" + ); + let err = text(&out.stderr); + assert!(err.contains("rewrote 2 keys"), "{err}"); + assert!( + !err.contains("x-fern-sdk-method-name: 1"), + "the keys are listed only with -v: {err}" + ); + assert!(err.contains("1 file: 1 rewritten, 0 unchanged"), "{err}"); + let leftovers: Vec<_> = fs::read_dir(&dir).unwrap().collect(); + assert_eq!(leftovers.len(), 1, "no file is left beside it"); +} + +#[test] +fn a_file_that_needs_nothing_is_left_as_it_is() { + let dir = dir("native"); + let file = dir.join("spec.json"); + let written = format!("{NATIVE}\n\n"); + fs::write(&file, &written).unwrap(); + + let out = rewrite(&[file.to_str().unwrap()], ""); + + succeeded(&out); + assert_eq!(fs::read_to_string(&file).unwrap(), written); + assert!( + text(&out.stderr).contains("1 file: 0 rewritten, 1 unchanged"), + "{}", + text(&out.stderr) + ); +} + +#[test] +fn the_old_keys_can_be_left() { + let dir = dir("leave"); + let file = dir.join("spec.json"); + fs::write(&file, FERN).unwrap(); + + succeeded(&rewrite(&["--leave-old-extensions", file.to_str().unwrap()], "")); + + let value: Value = serde_json::from_str(&fs::read_to_string(&file).unwrap()).unwrap(); + let operation = &value["paths"]["/a"]["get"]; + assert_eq!(operation["x-fern-sdk-method-name"], "get"); + assert_eq!(operation["x-forge-method-name"], "get"); +} + +#[test] +fn standard_input_is_written_to_standard_output() { + let out = normalize(&["-"], FERN); + + succeeded(&out); + assert_eq!(text(&out.stdout), NATIVE); +} + +#[test] +fn from_rewrites_only_the_selected_compatibility_sources_and_may_be_repeated() { + let out = normalize(&["--from", "fern", "--from", "cloudflare", "-"], SOURCES); + + succeeded(&out); + let value: Value = serde_json::from_slice(&out.stdout).unwrap(); + let operation = &value["paths"]["/a"]["get"]; + assert_eq!(operation["x-forge-method-name"], "get"); + assert!(operation.get("x-fern-sdk-method-name").is_none()); + let properties = &value["components"]["schemas"]["S"]["properties"]; + assert_eq!(properties["secret"]["x-forge-sensitive"], true); + assert!(properties["secret"].get("x-sensitive").is_none()); + assert_eq!(properties["hidden"]["x-stainless-skip"], true); + assert!(properties["hidden"].get("x-forge-skip").is_none()); +} + +#[test] +fn deprecated_is_a_compatibility_source_and_currently_has_no_keys() { + let out = normalize(&["--from", "deprecated", "-"], SOURCES); + + succeeded(&out); + assert_eq!(text(&out.stdout), SOURCES); +} + +#[test] +fn without_from_every_compatibility_source_is_rewritten() { + let out = normalize(&["-"], SOURCES); + + succeeded(&out); + let text = text(&out.stdout); + for old in ["x-fern-", "x-sensitive", "x-stainless-"] { + assert!(!text.contains(old), "{old} is left in {text}"); + } +} + +#[test] +fn standard_input_that_needs_nothing_comes_out_as_it_went_in() { + let document = format!(" {NATIVE} \n"); + + let out = normalize(&["-"], &document); + + succeeded(&out); + assert_eq!(text(&out.stdout), document); +} + +#[test] +fn standard_input_cannot_be_given_with_files() { + let out = normalize(&["-", "spec.json"], ""); + + let err = failed(&out); + + assert!(err.contains("- cannot be given with other inputs"), "{err}"); +} + +#[test] +fn a_file_that_is_missing_is_an_error_that_names_it() { + let out = rewrite(&["missing.json"], ""); + + let err = failed(&out); + assert!(err.contains("forge: missing.json: "), "{err}"); +} + +#[test] +fn a_file_that_is_not_json_is_an_error_and_the_others_are_still_rewritten() { + let dir = dir("invalid"); + fs::write(dir.join("a.json"), FERN).unwrap(); + // It has to look like it might change, or it is not parsed. + fs::write(dir.join("b.json"), r#"{"x-fern-ignore": "#).unwrap(); + + let (a, b) = (dir.join("a.json"), dir.join("b.json")); + + let out = rewrite(&[a.to_str().unwrap(), b.to_str().unwrap()], ""); + + let err = failed(&out); + assert!(err.contains("b.json"), "{err}"); + assert!(err.contains("1 file could not be processed"), "{err}"); + assert_eq!(fs::read_to_string(dir.join("a.json")).unwrap(), NATIVE); + assert_eq!(fs::read_to_string(dir.join("b.json")).unwrap(), r#"{"x-fern-ignore": "#); +} + +#[test] +fn a_native_key_wins_and_the_other_is_dropped() { + let dir = dir("conflict"); + let file = dir.join("spec.json"); + fs::write( + &file, + r#"{"openapi":"3.0.3","info":{"title":"t","version":"1"},"paths":{"/a":{"get":{"x-fern-sdk-method-name":"old","x-forge-method-name":"new","x-forge-group-name":["a"],"responses":{"204":{"description":"ok"}}}}}}"#, + ) + .unwrap(); + + let out = rewrite(&[file.to_str().unwrap()], ""); + + succeeded(&out); + let written = fs::read_to_string(&file).unwrap(); + assert!(written.contains(r#""x-forge-method-name":"new""#), "{written}"); + assert!(!written.contains("x-fern"), "{written}"); +} + +#[test] +fn a_group_that_cannot_be_a_dotted_string_is_kept_and_nothing_is_said_about_it() { + let dir = dir("kept"); + let file = dir.join("spec.json"); + fs::write(&file, FERN.replace(r#"["a"]"#, r#"["a.b"]"#)).unwrap(); + + let out = rewrite(&[file.to_str().unwrap()], ""); + + succeeded(&out); + let err = text(&out.stderr); + assert!( + !err.contains("note") && !err.contains("warning"), + "an array that is kept is expected: {err}" + ); + assert_eq!( + fs::read_to_string(&file).unwrap(), + NATIVE.replace(r#""x-forge-group-name":"a""#, r#""x-forge-group-name":["a.b"]"#), + "the array is kept, and only the keys are renamed" + ); +} + +#[test] +fn what_it_did_is_a_line_for_each_file_and_the_keys_are_listed_with_v() { + let dir = dir("verbose"); + for name in ["a.json", "b.json"] { + fs::write(dir.join(name), FERN).unwrap(); + } + let (a, b) = (dir.join("a.json"), dir.join("b.json")); + + let quiet = rewrite(&[a.to_str().unwrap()], ""); + let verbose = rewrite(&["-v", b.to_str().unwrap()], ""); + + let (quiet, verbose) = (text(&quiet.stderr), text(&verbose.stderr)); + assert_eq!(quiet.lines().count(), 2, "{quiet}"); + assert!(verbose.contains(" x-fern-sdk-group-name: 1"), "{verbose}"); + assert!(verbose.contains(" x-fern-sdk-method-name: 1"), "{verbose}"); +} + +#[test] +fn check_writes_nothing_and_fails_when_there_is_something_to_rewrite() { + let dir = dir("check"); + let (old, native) = (dir.join("old.json"), dir.join("native.json")); + fs::write(&old, FERN).unwrap(); + fs::write(&native, NATIVE).unwrap(); + + let out = normalize(&["--check", old.to_str().unwrap(), native.to_str().unwrap()], ""); + + let err = failed(&out); + assert!(err.contains("old.json: would rewrite 2 keys"), "{err}"); + assert!(err.contains("forge: 1 of 2 files would be rewritten"), "{err}"); + assert_eq!(fs::read_to_string(&old).unwrap(), FERN, "nothing is written"); +} + +#[test] +fn check_succeeds_when_there_is_nothing_to_rewrite() { + let dir = dir("check-clean"); + let file = dir.join("spec.json"); + fs::write(&file, NATIVE).unwrap(); + + let out = normalize(&["--check", file.to_str().unwrap()], ""); + + succeeded(&out); + assert!( + text(&out.stderr).contains("1 file checked, none would be rewritten"), + "{}", + text(&out.stderr) + ); +} + +#[test] +fn check_on_standard_input_writes_nothing_and_fails_when_there_is_something_to_rewrite() { + let out = normalize(&["--check", "-"], FERN); + + let err = failed(&out); + assert!(err.contains("the document would be rewritten"), "{err}"); + assert!(out.stdout.is_empty(), "nothing is written"); + assert!(normalize(&["--check", "-"], NATIVE).status.success()); +} + +#[test] +fn a_yaml_file_is_an_error_and_is_not_reported_as_native() { + let dir = dir("yaml"); + for name in ["spec.yml", "spec.YAML"] { + fs::write(dir.join(name), "openapi: 3.0.3\nx-fern-ignore: true\n").unwrap(); + let file = dir.join(name); + + let out = rewrite(&[file.to_str().unwrap()], ""); + + let err = failed(&out); + assert!(err.contains("YAML is not supported yet"), "{name}: {err}"); + assert!(err.contains(name), "{name}: {err}"); + assert!(!err.contains("unchanged"), "{name}: {err}"); + } +} + +#[test] +fn a_file_that_is_not_json_is_an_error_whatever_it_is_named() { + let dir = dir("not-an-object"); + for (name, content) in [ + ("yaml.json", "openapi: 3.0.3\nx-fern-ignore: true\n"), + ("empty.json", ""), + ("blank.json", " \n"), + ] { + fs::write(dir.join(name), content).unwrap(); + let file = dir.join(name); + + let out = rewrite(&[file.to_str().unwrap()], ""); + + let err = failed(&out); + assert!(err.contains(name), "{name}: {err}"); + assert!(!err.contains("unchanged"), "{name}: {err}"); + assert_eq!(fs::read_to_string(&file).unwrap(), content, "{name} is left as it is"); + } +} + +#[test] +fn the_usage_says_forge_normalize_and_the_help_of_forge_lists_the_command() { + let help = normalize(&["--help"], ""); + let general = forge(&["--help"], ""); + + assert!( + text(&help.stdout).contains("Usage: forge normalize"), + "{}", + text(&help.stdout) + ); + assert!(text(&help.stdout).contains("--check"), "{}", text(&help.stdout)); + assert!( + text(&help.stdout).contains("--from ") + && text(&help.stdout).contains("cloudflare") + && text(&help.stdout).contains("deprecated") + && text(&help.stdout).contains("fern") + && text(&help.stdout).contains("stainless"), + "{}", + text(&help.stdout) + ); + assert!( + text(&general.stdout).contains("Commands:") && text(&general.stdout).contains(" normalize "), + "{}", + text(&general.stdout) + ); + assert!( + text(&general.stdout).contains("forge "), + "{}", + text(&general.stdout) + ); + let missing = normalize(&[], ""); + assert!(failed(&missing).contains("Usage: forge normalize")); +} + +#[cfg(unix)] +#[test] +fn a_link_stays_a_link_and_the_file_it_points_to_is_rewritten() { + let dir = dir("symlink"); + fs::create_dir_all(dir.join("real")).unwrap(); + fs::create_dir_all(dir.join("links")).unwrap(); + fs::write(dir.join("real/spec.json"), FERN).unwrap(); + std::os::unix::fs::symlink("../real/spec.json", dir.join("links/spec.json")).unwrap(); + + let out = rewrite(&[dir.join("links/spec.json").to_str().unwrap()], ""); + + succeeded(&out); + assert!( + fs::symlink_metadata(dir.join("links/spec.json")) + .unwrap() + .file_type() + .is_symlink(), + "the link was replaced by a file" + ); + assert_eq!(fs::read_to_string(dir.join("real/spec.json")).unwrap(), NATIVE); + assert!( + text(&out.stderr).contains("links/spec.json: rewrote 2 keys"), + "{}", + text(&out.stderr) + ); +} + +#[test] +fn a_file_given_more_than_once_is_rewritten_once() { + let dir = dir("twice"); + let file = dir.join("spec.json"); + fs::write(&file, FERN).unwrap(); + let again = format!("{}/./spec.json", dir.display()); + + for _ in 0..5 { + fs::write(&file, FERN).unwrap(); + let out = rewrite(&[file.to_str().unwrap(), &again], ""); + + succeeded(&out); + assert!( + text(&out.stderr).contains("1 file: 1 rewritten"), + "{}", + text(&out.stderr) + ); + assert_eq!(fs::read_to_string(&file).unwrap(), NATIVE); + } +} + +/// Nothing is changed unless `--write` says so. +#[test] +fn one_file_is_printed_rewritten_and_is_not_changed_without_write() { + let dir = dir("print"); + let file = dir.join("spec.json"); + fs::write(&file, FERN).unwrap(); + + let out = normalize(&[file.to_str().unwrap()], ""); + + succeeded(&out); + assert_eq!(text(&out.stdout), NATIVE); + assert_eq!(fs::read_to_string(&file).unwrap(), FERN, "the file is as it was"); + assert!( + text(&out.stderr).contains("spec.json: 2 keys rewritten in the output"), + "{}", + text(&out.stderr) + ); + assert!(!text(&out.stderr).contains("rewrote"), "nothing was written"); +} + +#[test] +fn a_file_with_nothing_to_rewrite_is_printed_as_it_is() { + let dir = dir("print-nothing"); + let file = dir.join("spec.json"); + fs::write(&file, NATIVE).unwrap(); + + let out = normalize(&[file.to_str().unwrap()], ""); + + succeeded(&out); + assert_eq!(text(&out.stdout), NATIVE); + assert_eq!(text(&out.stderr), ""); +} + +#[test] +fn several_files_are_only_rewritten_with_write_or_checked_with_check() { + let dir = dir("several"); + let (a, b) = (dir.join("a.json"), dir.join("b.json")); + fs::write(&a, FERN).unwrap(); + fs::write(&b, FERN).unwrap(); + let both = [a.to_str().unwrap(), b.to_str().unwrap()]; + + let out = normalize(&both, ""); + + let err = failed(&out); + assert!(err.contains("--write") && err.contains("--check"), "{err}"); + assert_eq!(text(&out.stdout), "", "nothing is printed"); + assert_eq!(fs::read_to_string(&a).unwrap(), FERN); + assert_eq!(fs::read_to_string(&b).unwrap(), FERN); + + // The same file twice is one file, which is printed. + succeeded(&normalize(&[both[0], both[0]], "")); +} + +#[test] +fn write_and_check_cannot_be_given_together() { + let dir = dir("write-check"); + let file = dir.join("spec.json"); + fs::write(&file, FERN).unwrap(); + + let out = normalize(&["--write", "--check", file.to_str().unwrap()], ""); + + let err = failed(&out); + assert!(err.contains("--write") && err.contains("--check"), "{err}"); + assert_eq!(fs::read_to_string(&file).unwrap(), FERN); +} + +#[test] +fn standard_input_cannot_be_rewritten_in_place() { + let out = normalize(&["--write", "-"], FERN); + + let err = failed(&out); + assert!(err.contains("- cannot be given with --write"), "{err}"); + assert_eq!(text(&out.stdout), ""); +} + +#[test] +fn write_rewrites_each_file_in_place_and_says_so() { + let dir = dir("write"); + let (a, b) = (dir.join("a.json"), dir.join("b.json")); + fs::write(&a, FERN).unwrap(); + fs::write(&b, NATIVE).unwrap(); + + let out = rewrite(&[a.to_str().unwrap(), b.to_str().unwrap()], ""); + + succeeded(&out); + assert_eq!(text(&out.stdout), "", "what is written is the files"); + assert_eq!(fs::read_to_string(&a).unwrap(), NATIVE); + assert!( + text(&out.stderr).contains("2 files: 1 rewritten, 1 unchanged"), + "{}", + text(&out.stderr) + ); +} + +#[test] +fn a_folder_is_an_error_that_says_so() { + let dir = dir("a-folder"); + fs::write(dir.join("a.json"), FERN).unwrap(); + + let out = rewrite(&[dir.to_str().unwrap()], ""); + + let err = failed(&out); + assert!(err.contains("is a folder, not a file"), "{err}"); + assert_eq!( + fs::read_to_string(dir.join("a.json")).unwrap(), + FERN, + "nothing in it is rewritten" + ); +} + +#[test] +fn a_pattern_is_for_the_shell_and_is_not_expanded() { + let dir = dir("pattern"); + fs::write(dir.join("a.json"), FERN).unwrap(); + + let out = rewrite(&[&format!("{}/*.json", dir.display())], ""); + + let err = failed(&out); + assert!(err.contains("*.json: "), "{err}"); + assert_eq!(fs::read_to_string(dir.join("a.json")).unwrap(), FERN); +} + +#[test] +fn a_file_whose_name_has_glob_characters_is_that_file() { + let dir = dir("brackets"); + let file = dir.join("spec[1].json"); + fs::write(&file, FERN).unwrap(); + + let out = rewrite(&[file.to_str().unwrap()], ""); + + succeeded(&out); + assert_eq!(fs::read_to_string(&file).unwrap(), NATIVE); +} + +#[cfg(unix)] +#[test] +fn a_read_only_file_is_not_rewritten() { + use std::os::unix::fs::PermissionsExt; + + let dir = dir("read-only"); + let file = dir.join("spec.json"); + fs::write(&file, FERN).unwrap(); + fs::set_permissions(&file, fs::Permissions::from_mode(0o444)).unwrap(); + + let out = rewrite(&[file.to_str().unwrap()], ""); + let check = normalize(&["--check", file.to_str().unwrap()], ""); + + let err = failed(&out); + assert!( + err.contains("spec.json: the file is read-only, so it is not rewritten"), + "{err}" + ); + assert_eq!(fs::read_to_string(&file).unwrap(), FERN); + assert!( + text(&check.stderr).contains("would rewrite 2 keys"), + "a check only reads: {}", + text(&check.stderr) + ); +} + +#[test] +fn a_byte_order_mark_is_an_error_and_the_file_is_left_as_it_is() { + let dir = dir("bom"); + let file = dir.join("spec.json"); + let content = format!("\u{feff}{FERN}"); + fs::write(&file, &content).unwrap(); + + let out = rewrite(&[file.to_str().unwrap()], ""); + + let err = failed(&out); + assert!(err.contains("spec.json: expected value at line 1 column 1"), "{err}"); + assert_eq!(fs::read_to_string(&file).unwrap(), content); +} + +#[test] +fn standard_input_is_called_stdin_and_a_failure_on_it_says_so() { + let out = normalize(&["-"], FERN); + let broken = normalize(&["-"], "{\"x-fern-ignore\": "); + + assert!( + text(&out.stderr).starts_with("stdin: 2 keys rewritten in the output"), + "{}", + text(&out.stderr) + ); + assert!(failed(&broken).contains("forge: stdin: "), "{}", text(&broken.stderr)); +} + +#[test] +fn v_says_how_many_group_paths_were_joined() { + let dir = dir("joined"); + let file = dir.join("spec.json"); + fs::write(&file, FERN).unwrap(); + + let out = rewrite(&["-v", file.to_str().unwrap()], ""); + + assert!( + text(&out.stderr).contains(" 1 group path written as a dotted string"), + "{}", + text(&out.stderr) + ); +} + +#[cfg(unix)] +#[test] +fn a_hard_link_stays_a_link_to_the_file_that_is_rewritten() { + let dir = dir("hard-link"); + let (a, b) = (dir.join("a.json"), dir.join("b.json")); + fs::write(&a, FERN).unwrap(); + fs::hard_link(&a, &b).unwrap(); + + succeeded(&rewrite(&[a.to_str().unwrap()], "")); + + assert_eq!(fs::read_to_string(&a).unwrap(), NATIVE); + assert_eq!( + fs::read_to_string(&b).unwrap(), + NATIVE, + "the other name is the same file, which is rewritten" + ); +} + +#[cfg(unix)] +#[test] +fn two_names_of_one_file_are_one_file_and_are_rewritten_once() { + let dir = dir("hard-links-given"); + let (a, b) = (dir.join("a.json"), dir.join("b.json")); + fs::write(&a, FERN).unwrap(); + fs::hard_link(&a, &b).unwrap(); + + let out = rewrite(&[a.to_str().unwrap(), b.to_str().unwrap()], ""); + + succeeded(&out); + assert!( + text(&out.stderr).contains("1 file: 1 rewritten"), + "{}", + text(&out.stderr) + ); + assert_eq!(fs::read_to_string(&b).unwrap(), NATIVE); +} + +#[cfg(unix)] +#[test] +fn a_file_in_a_folder_that_cannot_be_written_to_is_rewritten_in_place() { + use std::os::unix::fs::PermissionsExt; + + let dir = dir("read-only-folder"); + let file = dir.join("spec.json"); + fs::write(&file, FERN).unwrap(); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o555)).unwrap(); + // The folder cannot be written to, unless this is run as the owner of + // everything, which it is not a test of. + if fs::write(dir.join("probe"), "").is_ok() { + fs::set_permissions(&dir, fs::Permissions::from_mode(0o755)).unwrap(); + return; + } + + let out = rewrite(&[file.to_str().unwrap()], ""); + + fs::set_permissions(&dir, fs::Permissions::from_mode(0o755)).unwrap(); + succeeded(&out); + assert_eq!(fs::read_to_string(&file).unwrap(), NATIVE); +} From 3010cecd4c81007bd5f3a5ee18f3dcc0e7f810eb Mon Sep 17 00:00:00 2001 From: Niels Swimberghe Date: Fri, 30 Oct 2026 14:29:24 -0400 Subject: [PATCH 2/2] docs: describe staged normalization state --- src/normalize/vendor/README.md | 56 ++++++++++++++++------------------ src/normalize/vendor/mod.rs | 6 ++-- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/src/normalize/vendor/README.md b/src/normalize/vendor/README.md index 85e841baa..f79023723 100644 --- a/src/normalize/vendor/README.md +++ b/src/normalize/vendor/README.md @@ -1,12 +1,13 @@ # The vendor tables -`forge` reads one vocabulary: the native `x-forge-*` extensions, in the readers -of `src/link/`. The extensions other tools write -(Cloudflare's `x-sensitive`, Fern's `x-fern-*`, and Stainless's -`x-stainless-*`) are ordinary extensions to it, kept for plugins like any -other it does not read. This folder is the only place that knows them, their -source, and what they say: `forge normalize` uses it to rewrite them into -native keys, before `forge` reads the document. +`forge normalize` rewrites the extensions other tools write (Cloudflare's +`x-sensitive`, Fern's `x-fern-*`, and Stainless's `x-stainless-*`) into the +native `x-forge-*` vocabulary. This folder owns those normalization mappings: +their source, native replacement, and value form. + +The parser still reads the compatibility extensions directly. Moving it to the +native vocabulary is a separate change, so vendor names remain in `src/link/` +for now. Normalization is explicit; `forge` does not run it before parsing. `deprecated` is also treated as a vendor here, though it is not literally one: its adapter holds old Forge keys that have a current replacement. This keeps @@ -14,23 +15,19 @@ renames out of the parser by giving them the same explicit migration path. ## Rules -- **Only this folder names a vendor extension.** That includes the readers, - the linker, the plugin host, the WIT, the `packages/forge` npm package, and - their comments and docs. `tests/boundary.rs` fails, with file and line, if - one appears elsewhere (`just boundary`; it also runs with `cargo test`). - The tests of vendor-keyed documents live in `tests/normalize/`. +- **Normalization mappings live only here.** The current parser still names + compatibility extensions in `src/link/`; do not add another normalization + table outside this folder. Normalization tests live in `tests/normalize/`. - **Mapping, not semantics.** A table says which native key a key becomes, and whether its value is written as the native key's is, as data. Anything structural, such as how aliases become methods, belongs to the builder, not here. The one value that is written again is a group path, which becomes the names joined by dots (`walk.rs`). - **One adapter per source.** `--from` selects the `cloudflare`, `deprecated`, - `fern`, and `stainless` adapters, and no `--from` selects them all. The - temporary in-memory compatibility bridge always uses all adapters, so it - preserves what the old parser read. -- **A key is read only at the level `forge` reads its native key at**: an - operation or an alias, a parameter, or a schema. A key of an object `forge` - does not read it at is left alone, since rewriting it would give it a + `fern`, and `stainless` adapters, and no `--from` selects them all. +- **A key is rewritten only at the level where Forge reads the corresponding + metadata**: an operation or an alias, a parameter, or a schema. The same key + on another kind of object is left alone, since rewriting it would give it a meaning it did not have. - **Native wins.** When a document has the native key, the vendor one is dropped, whatever order they appear in. When a vendor key has no native form, it is left as it is written: on a @@ -41,15 +38,16 @@ renames out of the parser by giving them the same explicit migration path. ## Adding a vendor key -1. Find the native key it stands for in the readers of `src/link/`. If there - is none, add a native `x-forge-*` key to the reader first: a vendor key - never introduces a primitive. +1. Identify the Forge primitive the vendor key represents and the native + `x-forge-*` key for that primitive. Normalization does not introduce new + semantics; native parser support belongs in a separate change. 2. Map it in the table of the adapter for its source, or add an adapter next to - `cloudflare.rs`, `deprecated.rs`, `fern.rs` and `stainless.rs` and ask it - from `mod.rs`. A test fails if the table names a key that `forge` does not - read at that kind of object. A renamed Forge key goes in `deprecated.rs` - and its exact old name goes in the table for that object kind. -3. Add a test in `tests/normalize/` showing that the vendor-keyed document, - after `forge normalize`, builds the same `Forge` as the native-keyed one. - -`forge` itself needs no change: it never learns the key. + `cloudflare.rs`, `deprecated.rs`, `fern.rs` and `stainless.rs` and include it + from `mod.rs`. A renamed Forge key goes in `deprecated.rs`, with its exact + old name in the table for that object kind. +3. Add a test in `tests/normalize/` showing the exact native document produced + from the vendor-keyed input. + +Until the parser moves to the native vocabulary, parser support for a new +compatibility key remains a separate concern from adding its normalization +mapping. diff --git a/src/normalize/vendor/mod.rs b/src/normalize/vendor/mod.rs index b6fa80984..de34e542c 100644 --- a/src/normalize/vendor/mod.rs +++ b/src/normalize/vendor/mod.rs @@ -1,6 +1,6 @@ -//! The keys other tools' metadata is written with, and the native extension -//! that each says the same as. This is the only code that holds them: `forge` -//! normalization keeps isolated from the rest of Forge. +//! The normalization mappings from keys other tools write to the native +//! extensions that say the same thing. The current parser still reads the +//! compatibility keys directly; moving it to native keys is a separate change. //! //! Each adapter, [`cloudflare`], [`deprecated`], [`fern`] and [`stainless`], is //! a table per kind of object that maps its compatibility keys onto native