diff --git a/cli/src/main.rs b/cli/src/main.rs index 7040519f..b1188186 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -7,7 +7,7 @@ use std::{ thread::JoinHandle, }; -use clap::{Args, Parser, Subcommand}; +use clap::{Args, Parser, Subcommand, ValueEnum}; use clap_verbosity_flag::{InfoLevel, Verbosity, log::LevelFilter}; use display::{ message::{DisplayMessage, send_message}, @@ -46,8 +46,8 @@ impl Cli { pub fn debug_props(&self) -> String { let token = match &self.command { Commands::Test(args) => args.token(), - Commands::Upload(args) => args.token.clone(), - Commands::Validate(..) + Commands::Upload { args, .. } => args.token.clone(), + Commands::Validate { .. } | Commands::SummaryOutputSchema | Commands::GithubAnnotations(..) => None, }; @@ -61,8 +61,8 @@ impl Cli { pub fn command_name(&self) -> &str { match &self.command { Commands::Test(..) => "test", - Commands::Upload(..) => "upload", - Commands::Validate(..) => "validate", + Commands::Upload { .. } => "upload", + Commands::Validate { .. } => "validate", Commands::SummaryOutputSchema => "summary-output-schema", Commands::GithubAnnotations(..) => "github-annotations", } @@ -76,8 +76,8 @@ impl Cli { pub fn org_url_slug(&self) -> String { match &self.command { Commands::Test(args) => args.org_url_slug(), - Commands::Upload(args) => args.org_url_slug.clone(), - Commands::Validate(..) + Commands::Upload { args, .. } => args.org_url_slug.clone(), + Commands::Validate { .. } | Commands::SummaryOutputSchema | Commands::GithubAnnotations(..) => String::from("not used"), } @@ -86,18 +86,29 @@ impl Cli { pub fn summary_output_file(&self) -> Option { match &self.command { Commands::Test(args) => args.summary_output_file(), - Commands::Upload(args) => args.summary_output_file.clone(), - Commands::Validate(..) + Commands::Upload { args, .. } => args.summary_output_file.clone(), + Commands::Validate { .. } | Commands::SummaryOutputSchema | Commands::GithubAnnotations(..) => None, } } + pub fn output_format(&self) -> OutputFormat { + match &self.command { + Commands::Upload { output, .. } | Commands::Validate { output, .. } => { + output.output_format + } + Commands::Test(..) + | Commands::SummaryOutputSchema + | Commands::GithubAnnotations(..) => OutputFormat::Text, + } + } + pub fn repo_root(&self) -> String { let explicit_root = match &self.command { Commands::Test(args) => args.repo_root(), - Commands::Upload(args) => args.repo_root.clone(), - Commands::Validate(..) + Commands::Upload { args, .. } => args.repo_root.clone(), + Commands::Validate { .. } | Commands::SummaryOutputSchema | Commands::GithubAnnotations(..) => None, }; @@ -117,11 +128,21 @@ enum Commands { Test(TestArgs), /// Upload test results to Trunk Flaky Tests. Use this when you've already run tests and have result files. #[command(after_long_help = ExitCode::help())] - Upload(UploadArgs), + Upload { + #[command(flatten)] + args: UploadArgs, + #[command(flatten)] + output: OutputFormatArgs, + }, /// Validate test report files for compatibility with Trunk Flaky Tests. Runs locally without uploading data. #[command(after_long_help = ExitCode::help())] - Validate(ValidateArgs), - /// Print the JSON Schema of the summary that `--summary-output-file` writes. + Validate { + #[command(flatten)] + args: ValidateArgs, + #[command(flatten)] + output: OutputFormatArgs, + }, + /// Print the JSON Schema of the summary that `--summary-output-file` writes and `--output-format json` prints. SummaryOutputSchema, /// Print GitHub Actions annotations for why a run failed, from its `--summary-output-file`. /// Exits 0 only when it printed one. Used by the analytics-uploader action. @@ -129,6 +150,25 @@ enum Commands { GithubAnnotations(GithubAnnotationsArgs), } +/// Not in `UploadArgs`, because `test` includes those and its test command writes to stdout. +#[derive(Debug, Args)] +struct OutputFormatArgs { + #[arg( + long, + env = constants::TRUNK_OUTPUT_FORMAT_ENV, + value_enum, + default_value_t = OutputFormat::Text, + help = "Print the summary on stdout as text, or as the JSON that --summary-output-file writes. Logs and progress stay on stderr." + )] + output_format: OutputFormat, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +enum OutputFormat { + Text, + Json, +} + #[derive(Debug, Args)] struct GithubAnnotationsArgs { /// The summary that `upload` or `test` wrote to `--summary-output-file`. @@ -136,8 +176,11 @@ struct GithubAnnotationsArgs { summary_file: PathBuf, } +/// `json` is the summary for `--output-format json`, printed once the renderer has finished so +/// nothing else is written to stdout around it. fn close_out_and_exit( exit_code: ExitCode, + json: Option, guard: ClientInitGuard, render_sender: Sender, render_handle: JoinHandle<()>, @@ -145,6 +188,12 @@ fn close_out_and_exit( guard.flush(None); drop(render_sender); let _ = render_handle.join(); + if let Some(report) = json { + match report.to_json() { + Ok(json) => println!("{json}"), + Err(error) => tracing::error!("Failed to print the summary as JSON: {error:?}"), + } + } std::process::exit(exit_code.code()) } @@ -166,6 +215,7 @@ fn main() -> anyhow::Result<()> { .block_on(async { let org_url_slug = cli.org_url_slug(); let summary_output_file = cli.summary_output_file(); + let output_format = cli.output_format(); let log_level_filter = cli.verbose.log_level_filter(); setup_logger( log_level_filter, @@ -193,7 +243,7 @@ fn main() -> anyhow::Result<()> { tracing::warn!( "Received termination signal; aborting in-flight work and shutting down." ); - close_out_and_exit(exit_code, guard, render_sender, render_handle) + close_out_and_exit(exit_code, None, guard, render_sender, render_handle) } MainOutcome::Finished(Ok(Report::Upload(report, options))) => { show( @@ -202,18 +252,30 @@ fn main() -> anyhow::Result<()> { "notices", &render_sender, ); - show( - DisplayMessage::Report, - text::render_upload(&report, &options), - "upload summary", - &render_sender, - ); + let json = match output_format { + OutputFormat::Text => { + show( + DisplayMessage::Report, + text::render_upload(&report, &options), + "upload summary", + &render_sender, + ); + None + } + OutputFormat::Json => Some(json::Report::Upload(&report)), + }; write_summary_file( summary_output_file.as_deref(), json::Report::Upload(&report), &render_sender, ); - close_out_and_exit(report.cli_exit_code(), guard, render_sender, render_handle) + close_out_and_exit( + report.cli_exit_code(), + json, + guard, + render_sender, + render_handle, + ) } MainOutcome::Finished(Ok(Report::Validate(report))) => { show( @@ -222,17 +284,23 @@ fn main() -> anyhow::Result<()> { "notices", &render_sender, ); - show( - DisplayMessage::Report, - text::render_validate(&report), - "validate summary", - &render_sender, - ); - close_out_and_exit(report.exit_code, guard, render_sender, render_handle) + let json = match output_format { + OutputFormat::Text => { + show( + DisplayMessage::Report, + text::render_validate(&report), + "validate summary", + &render_sender, + ); + None + } + OutputFormat::Json => Some(json::Report::Validate(&report)), + }; + close_out_and_exit(report.exit_code, json, guard, render_sender, render_handle) } MainOutcome::Finished(Ok(Report::SummaryOutputSchema)) => { println!("{}", serde_json::to_string_pretty(&json::Report::schema())?); - close_out_and_exit(ExitCode::Success, guard, render_sender, render_handle) + close_out_and_exit(ExitCode::Success, None, guard, render_sender, render_handle) } MainOutcome::Finished(Err(error)) => { let error = ErrorSummary::new(&ErrorReport::new(error, org_url_slug, None)); @@ -248,7 +316,9 @@ fn main() -> anyhow::Result<()> { json::Report::Error(&error), &render_sender, ); - close_out_and_exit(exit_code, guard, render_sender, render_handle) + let json = (output_format == OutputFormat::Json) + .then_some(json::Report::Error(&error)); + close_out_and_exit(exit_code, json, guard, render_sender, render_handle) } } }) @@ -304,7 +374,9 @@ async fn run(cli: Cli, render_sender: Sender) -> anyhow::Result< env!("VERGEN_RUSTC_SEMVER") ); match cli.command { - Commands::Upload(upload_args) => { + Commands::Upload { + args: upload_args, .. + } => { let result = run_upload( upload_args, RunUploadOptions { @@ -325,7 +397,10 @@ async fn run(cli: Cli, render_sender: Sender) -> anyhow::Result< TextOptions::from(&result), )) } - Commands::Validate(validate_args) => { + Commands::Validate { + args: validate_args, + .. + } => { let result = run_validate(validate_args).await?; Ok(Report::Validate(ValidateReport::new(&result))) } diff --git a/cli/src/summary_output/error.rs b/cli/src/summary_output/error.rs index 2b208e09..09338204 100644 --- a/cli/src/summary_output/error.rs +++ b/cli/src/summary_output/error.rs @@ -21,7 +21,7 @@ pub struct ErrorSummary { pub explanation: Option, /// Where to check the org slug and token. Only set for `UNAUTHORIZED` errors. pub settings_url: Option, - /// A run that fails outright exits with this code, or 0 when it's null. Upload reports exit with their own `exitCode`. + /// A run that fails outright exits with this code, or 0 when it's null. Upload and validate reports exit with their own `exitCode`. pub exit_code: Option, } diff --git a/cli/src/summary_output/json.rs b/cli/src/summary_output/json.rs index c7915bed..b9b4b1ce 100644 --- a/cli/src/summary_output/json.rs +++ b/cli/src/summary_output/json.rs @@ -4,7 +4,7 @@ use anyhow::Context; use schemars::{JsonSchema, Schema, generate::SchemaSettings}; use serde::{Deserialize, Serialize}; -use super::{ErrorSummary, UploadReport}; +use super::{ErrorSummary, UploadReport, ValidateReport}; pub const SCHEMA_VERSION: u32 = 1; @@ -13,6 +13,8 @@ pub const SCHEMA_VERSION: u32 = 1; pub enum Report<'a> { /// The run finished and summarized its tests. Upload(&'a UploadReport), + /// `validate` finished and summarized the files it checked. + Validate(&'a ValidateReport), /// The run failed before it could summarize any tests. Error(&'a ErrorSummary), } @@ -21,7 +23,7 @@ pub enum Report<'a> { #[serde(rename_all = "camelCase")] #[schemars( title = "Trunk Analytics CLI summary", - description = "What `upload` and `test` write to `--summary-output-file`, and the RSpec gem to `TRUNK_SUMMARY_OUTPUT_FILE`." + description = "What `upload` and `test` write to `--summary-output-file`, the RSpec gem to `TRUNK_SUMMARY_OUTPUT_FILE`, and `upload` and `validate` print with `--output-format json`." )] struct Versioned<'a> { /// Goes up when a change could break a reader, such as renaming or removing a field. diff --git a/cli/src/summary_output/validate.rs b/cli/src/summary_output/validate.rs index e2d732c7..35bbc1d2 100644 --- a/cli/src/summary_output/validate.rs +++ b/cli/src/summary_output/validate.rs @@ -1,6 +1,7 @@ use std::collections::HashSet; use context::junit::validator::JunitReportValidationFlatIssue; +use schemars::JsonSchema; use serde::Serialize; use super::{FileValidation, Notice, ParseError}; @@ -9,32 +10,37 @@ use crate::{ validate_command::{ParseIssues, TestIssues, ValidateRunResult}, }; -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ValidateReport { + /// The files each glob matched. pub matched_files: Vec, + /// One entry per test report file. pub validations: Vec, + /// The CODEOWNERS file, when one was found. pub codeowners: Option, pub counts: ValidationCounts, + /// The code the CLI exits with. pub exit_code: ExitCode, + /// Warnings that didn't stop the run, such as a Bazel BEP parse error. pub notices: Vec, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct MatchedFiles { pub glob: String, pub files: Vec, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct Codeowners { pub path: String, pub warnings: Vec, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ValidationCounts { pub files: usize, diff --git a/cli/tests/upload.rs b/cli/tests/upload.rs index 4523858a..67693004 100644 --- a/cli/tests/upload.rs +++ b/cli/tests/upload.rs @@ -1717,6 +1717,64 @@ async fn upload_keeps_its_exit_code_when_the_summary_output_file_cannot_be_writt assert!(leftovers.is_empty(), "left behind {leftovers:?}"); } +#[tokio::test(flavor = "multi_thread")] +async fn upload_prints_only_the_json_summary_on_stdout() { + let temp_dir = tempdir().unwrap(); + generate_mock_git_repo(&temp_dir); + generate_mock_valid_junit_xmls_with_failures(&temp_dir); + let summary_path = temp_dir.path().join("summary.json"); + + let mut mock_server_builder = MockServerBuilder::new(); + // Slow enough to show a progress message. + mock_server_builder.set_get_quarantining_config_handler( + |Json(_): Json| async move { + tokio::time::sleep(Duration::from_secs(3)).await; + Json(GetQuarantineConfigResponse::default()) + }, + ); + let state = mock_server_builder.spawn_mock_server().await; + let assert = CommandBuilder::upload(temp_dir.path(), state.host.clone()) + .verbose(true) + .command() + .args(["--output-format", "json", "--hide-banner"]) + .arg("--summary-output-file") + .arg(&summary_path) + .assert() + .code(ExitCode::Failure.code()) + .stderr(predicate::str::contains("Starting trunk-analytics-cli")) + .stderr(predicate::str::contains( + "⚠️ The --hide-banner flag is deprecated", + )) + .stderr(predicate::str::contains("taking longer than expected")) + .stderr(predicate::str::contains("Summary written to")); + + let summary: serde_json::Value = serde_json::from_slice(&assert.get_output().stdout).unwrap(); + assert_eq!(summary, read_summary_output_file(&summary_path)); + assert_eq!(summary["type"], "UPLOAD"); + assert_eq!(summary["exitCode"], ExitCode::Failure.code()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn upload_prints_a_json_error_on_stdout() { + let temp_dir = tempdir().unwrap(); + generate_mock_git_repo(&temp_dir); + + let state = MockServerBuilder::new().spawn_mock_server().await; + let assert = CommandBuilder::upload(temp_dir.path(), state.host.clone()) + .repo_root("../") + .command() + .env(constants::TRUNK_OUTPUT_FORMAT_ENV, "json") + .assert() + .code(ExitCode::Error.code()) + .stderr(predicate::str::contains("Error Encountered")); + + let summary: serde_json::Value = serde_json::from_slice(&assert.get_output().stdout).unwrap(); + assert_eq!(summary["schemaVersion"], 1); + assert_eq!(summary["type"], "ERROR"); + assert_eq!(summary["kind"], "GIT_REPOSITORY"); + assert_eq!(summary["exitCode"], ExitCode::Error.code()); +} + #[tokio::test(flavor = "multi_thread")] async fn upload_bundle_valid_repo_root_implicit() { let temp_dir = tempdir().unwrap(); diff --git a/cli/tests/validate.rs b/cli/tests/validate.rs index 8527c991..627b7c2c 100644 --- a/cli/tests/validate.rs +++ b/cli/tests/validate.rs @@ -241,3 +241,43 @@ fn validate_writes_summary_to_stdout_and_notices_to_stderr() { println!("{assert}"); } + +#[test] +fn validate_prints_only_the_json_summary_on_stdout() { + let temp_dir = tempdir().unwrap(); + generate_mock_valid_junit_xmls(&temp_dir); + generate_mock_codeowners(&temp_dir); + + let assert = CommandBuilder::validate(temp_dir.path()) + .command() + .args(["--output-format", "json", "--hide-banner", "--verbose"]) + .assert() + .success() + .stderr(predicate::str::contains("Starting trunk-analytics-cli")) + .stderr(predicate::str::contains( + "⚠️ The --hide-banner flag is deprecated", + )); + + let summary: serde_json::Value = serde_json::from_slice(&assert.get_output().stdout).unwrap(); + assert_eq!(summary["schemaVersion"], 1); + assert_eq!(summary["type"], "VALIDATE"); + assert_eq!(summary["counts"]["files"], 1); + assert_eq!(summary["exitCode"], 0); +} + +#[test] +fn validate_prints_json_with_the_same_exit_code() { + let temp_dir = tempdir().unwrap(); + generate_mock_invalid_junit_xmls(&temp_dir); + + let assert = CommandBuilder::validate(temp_dir.path()) + .command() + .args(["--output-format", "json"]) + .assert() + .code(1); + + let summary: serde_json::Value = serde_json::from_slice(&assert.get_output().stdout).unwrap(); + assert_eq!(summary["type"], "VALIDATE"); + assert_eq!(summary["counts"]["errors"], 1); + assert_eq!(summary["exitCode"], 1); +} diff --git a/constants/src/lib.rs b/constants/src/lib.rs index 9ada3b06..12816eb6 100644 --- a/constants/src/lib.rs +++ b/constants/src/lib.rs @@ -56,6 +56,7 @@ pub const TRUNK_VALIDATION_REPORT_ENV: &str = "TRUNK_VALIDATION_REPORT"; pub const TRUNK_SHOW_FAILURE_MESSAGES_ENV: &str = "TRUNK_SHOW_FAILURE_MESSAGES"; pub const TRUNK_HIDE_TEST_COLLECTION_LINKS_ENV: &str = "TRUNK_HIDE_TEST_COLLECTION_LINKS"; pub const TRUNK_SUMMARY_OUTPUT_FILE_ENV: &str = "TRUNK_SUMMARY_OUTPUT_FILE"; +pub const TRUNK_OUTPUT_FORMAT_ENV: &str = "TRUNK_OUTPUT_FORMAT"; pub const TRUNK_DEBUG_ENV: &str = "TRUNK_DEBUG"; pub const TRUNK_USE_EXPERIMENTAL_XCRESULT_TEST_LOCATIONS_ENV: &str = "TRUNK_USE_EXPERIMENTAL_XCRESULT_TEST_LOCATIONS"; @@ -110,6 +111,7 @@ pub const TRUNK_ENVS_TO_CAPTURE: &[&str] = &[ TRUNK_SHOW_FAILURE_MESSAGES_ENV, TRUNK_HIDE_TEST_COLLECTION_LINKS_ENV, TRUNK_SUMMARY_OUTPUT_FILE_ENV, + TRUNK_OUTPUT_FORMAT_ENV, TRUNK_DEBUG_ENV, TRUNK_USE_EXPERIMENTAL_XCRESULT_TEST_LOCATIONS_ENV, ];