From 03bb60ff65685fda7c41903b6395775a836663c2 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Tue, 11 Aug 2026 23:08:26 +0200 Subject: [PATCH] drop `releases.files` and `releases.archive_storage`, don't generate file list --- ...5438de9858c5a88ce1c7e0539a4fbe96da82.json} | 4 +- ...d6054a31d0a2df37a8a1d7659208092929ce8.json | 14 ------ ...f8d6755d85d4bde1de74f4335ff8b31109ed.json} | 5 +- Cargo.lock | 1 - .../src/docbuilder/rustwide_builder.rs | 25 ++++------ .../bin/docs_rs_import_release/src/import.rs | 21 ++++---- crates/bin/docs_rs_web/src/handlers/source.rs | 32 ------------ ...0608_remove-release-files-archive.down.sql | 3 ++ ...210608_remove-release-files-archive.up.sql | 3 ++ crates/lib/docs_rs_database/src/releases.rs | 29 ++++------- crates/lib/docs_rs_storage/Cargo.toml | 1 - crates/lib/docs_rs_storage/src/file.rs | 16 ------ crates/lib/docs_rs_storage/src/lib.rs | 4 +- crates/lib/docs_rs_storage/src/result.rs | 22 ++++++++ .../docs_rs_storage/src/storage/blocking.rs | 6 ++- .../src/storage/non_blocking.rs | 50 ++++++++----------- crates/lib/docs_rs_test_fakes/src/legacy.rs | 27 +++++----- 17 files changed, 103 insertions(+), 160 deletions(-) rename .sqlx/{query-4b4bfd97b03f632357a68c84f281c286fa0ec399ed98db0dda1ee81e6a6e7f3a.json => query-080c12e692d91033f67c040349125438de9858c5a88ce1c7e0539a4fbe96da82.json} (51%) delete mode 100644 .sqlx/query-8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8.json rename .sqlx/{query-d5d73e5899c9f0318d0357c7c66a980796789de5cb46b0cd169fcc86f13268c9.json => query-9d708bc42945bb13690e8f87ca2af8d6755d85d4bde1de74f4335ff8b31109ed.json} (79%) create mode 100644 crates/lib/docs_rs_database/migrations/20260811210608_remove-release-files-archive.down.sql create mode 100644 crates/lib/docs_rs_database/migrations/20260811210608_remove-release-files-archive.up.sql create mode 100644 crates/lib/docs_rs_storage/src/result.rs diff --git a/.sqlx/query-4b4bfd97b03f632357a68c84f281c286fa0ec399ed98db0dda1ee81e6a6e7f3a.json b/.sqlx/query-080c12e692d91033f67c040349125438de9858c5a88ce1c7e0539a4fbe96da82.json similarity index 51% rename from .sqlx/query-4b4bfd97b03f632357a68c84f281c286fa0ec399ed98db0dda1ee81e6a6e7f3a.json rename to .sqlx/query-080c12e692d91033f67c040349125438de9858c5a88ce1c7e0539a4fbe96da82.json index fa31fef7dd..d9102fe342 100644 --- a/.sqlx/query-4b4bfd97b03f632357a68c84f281c286fa0ec399ed98db0dda1ee81e6a6e7f3a.json +++ b/.sqlx/query-080c12e692d91033f67c040349125438de9858c5a88ce1c7e0539a4fbe96da82.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO releases (crate_id, version, archive_storage)\n VALUES ($1, $2, TRUE)\n ON CONFLICT (crate_id, version) DO UPDATE\n SET -- this `SET` is needed so the id is always returned.\n version = EXCLUDED.version\n RETURNING id as \"id: ReleaseId\" ", + "query": "INSERT INTO releases (crate_id, version)\n VALUES ($1, $2)\n ON CONFLICT (crate_id, version) DO UPDATE\n SET -- this `SET` is needed so the id is always returned.\n version = EXCLUDED.version\n RETURNING id as \"id: ReleaseId\" ", "describe": { "columns": [ { @@ -25,5 +25,5 @@ false ] }, - "hash": "4b4bfd97b03f632357a68c84f281c286fa0ec399ed98db0dda1ee81e6a6e7f3a" + "hash": "080c12e692d91033f67c040349125438de9858c5a88ce1c7e0539a4fbe96da82" } diff --git a/.sqlx/query-8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8.json b/.sqlx/query-8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8.json deleted file mode 100644 index 4f5d74702c..0000000000 --- a/.sqlx/query-8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE releases\n SET files = NULL\n WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4" - ] - }, - "nullable": [] - }, - "hash": "8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8" -} diff --git a/.sqlx/query-d5d73e5899c9f0318d0357c7c66a980796789de5cb46b0cd169fcc86f13268c9.json b/.sqlx/query-9d708bc42945bb13690e8f87ca2af8d6755d85d4bde1de74f4335ff8b31109ed.json similarity index 79% rename from .sqlx/query-d5d73e5899c9f0318d0357c7c66a980796789de5cb46b0cd169fcc86f13268c9.json rename to .sqlx/query-9d708bc42945bb13690e8f87ca2af8d6755d85d4bde1de74f4335ff8b31109ed.json index 3706dcb91d..10c53898b3 100644 --- a/.sqlx/query-d5d73e5899c9f0318d0357c7c66a980796789de5cb46b0cd169fcc86f13268c9.json +++ b/.sqlx/query-9d708bc42945bb13690e8f87ca2af8d6755d85d4bde1de74f4335ff8b31109ed.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE releases\n SET release_time = $2,\n dependencies = $3,\n target_name = $4,\n yanked = $5,\n rustdoc_status = $6,\n test_status = $7,\n license = $8,\n repository_url = $9,\n homepage_url = $10,\n description = $11,\n description_long = $12,\n readme = $13,\n keywords = $14,\n have_examples = $15,\n downloads = $16,\n files = $17,\n doc_targets = $18,\n is_library = $19,\n documentation_url = $20,\n default_target = $21,\n features = $22,\n repository_id = $23,\n source_size = $24\n WHERE id = $1", + "query": "UPDATE releases\n SET release_time = $2,\n dependencies = $3,\n target_name = $4,\n yanked = $5,\n rustdoc_status = $6,\n test_status = $7,\n license = $8,\n repository_url = $9,\n homepage_url = $10,\n description = $11,\n description_long = $12,\n readme = $13,\n keywords = $14,\n have_examples = $15,\n downloads = $16,\n doc_targets = $17,\n is_library = $18,\n documentation_url = $19,\n default_target = $20,\n features = $21,\n repository_id = $22,\n source_size = $23\n WHERE id = $1", "describe": { "columns": [], "parameters": { @@ -22,7 +22,6 @@ "Bool", "Int4", "Json", - "Json", "Bool", "Varchar", "Varchar", @@ -56,5 +55,5 @@ }, "nullable": [] }, - "hash": "d5d73e5899c9f0318d0357c7c66a980796789de5cb46b0cd169fcc86f13268c9" + "hash": "9d708bc42945bb13690e8f87ca2af8d6755d85d4bde1de74f4335ff8b31109ed" } diff --git a/Cargo.lock b/Cargo.lock index 653429f58f..49071c3b4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2396,7 +2396,6 @@ dependencies = [ "moka", "opentelemetry", "rand 0.10.2", - "serde_json", "sqlx", "strum", "tempfile", diff --git a/crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs b/crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs index bc97736622..a486cfa9f3 100644 --- a/crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs +++ b/crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs @@ -29,8 +29,7 @@ use docs_rs_rustdoc_json::{ read_format_version_from_rustdoc_json, }; use docs_rs_storage::{ - AsyncStorage, Storage, compress, file_list_to_json, rustdoc_archive_path, rustdoc_json_path, - source_archive_path, + AsyncStorage, Storage, compress, rustdoc_archive_path, rustdoc_json_path, source_archive_path, }; use docs_rs_types::{ BuildId, BuildStatus, CompressionAlgorithm, CrateId, KrateName, ReleaseId, Version, @@ -640,23 +639,22 @@ impl RustwideBuilder { let local_storage = tempfile::tempdir_in(&self.config.temp_dir)?; let mut algs = HashSet::new(); - let (source_files_list, source_size) = { + let source_stats = { let _span = info_span!("adding sources into database").entered(); debug!("adding sources into database"); let temp_dir = tempfile::tempdir_in(&self.config.temp_dir)?; krate.copy_source_to(&self.workspace, temp_dir.path())?; - let (files_list, new_alg) = self.runtime.block_on( + let stats = self.runtime.block_on( self.storage .store_all_in_archive(&source_archive_path(name, version), &temp_dir), )?; fs::remove_dir_all(temp_dir.path())?; - algs.insert(new_alg); - let source_size: u64 = files_list.iter().map(|info| info.size).sum(); - (files_list, source_size) + algs.insert(stats.alg); + stats }; let successful = build_dir @@ -740,16 +738,15 @@ impl RustwideBuilder { target_build_logs.insert(target, (target_res.build_log, successful)); } - let (file_list, new_alg) = + let doc_stats = self.runtime.block_on( self.storage.store_all_in_archive( &rustdoc_archive_path(name, version), local_storage.path(), ))?; - let documentation_size = file_list.iter().map(|info| info.size).sum::(); - self.builder_metrics.documentation_size.record(documentation_size, &[]); - algs.insert(new_alg); - Some(documentation_size) + self.builder_metrics.documentation_size.record(doc_stats.original_size, &[]); + algs.insert(doc_stats.alg); + Some(doc_stats.original_size) } else { None }; @@ -845,14 +842,13 @@ impl RustwideBuilder { cargo_metadata, &build.host_source_dir(), &res.target, - file_list_to_json(source_files_list), successful_targets, &release_data, has_docs, has_examples, algs, repository, - source_size, + source_stats.original_size, ))?; if let Some(repository_id) = repository { @@ -1836,7 +1832,6 @@ mod tests { }, Path::new("/unknown/"), "x86_64-unknown-linux-gnu", - serde_json::Value::Array(vec![]), vec![ "i686-pc-windows-msvc".into(), "aarch64-unknown-linux-gnu".into(), diff --git a/crates/bin/docs_rs_import_release/src/import.rs b/crates/bin/docs_rs_import_release/src/import.rs index df9c6dd05f..7308921799 100644 --- a/crates/bin/docs_rs_import_release/src/import.rs +++ b/crates/bin/docs_rs_import_release/src/import.rs @@ -16,7 +16,7 @@ use docs_rs_rustdoc_json::{ RUSTDOC_JSON_COMPRESSION_ALGORITHMS, RustdocJsonFormatVersion, read_format_version_from_rustdoc_json, }; -use docs_rs_storage::{AsyncStorage, file_list_to_json, rustdoc_archive_path, source_archive_path}; +use docs_rs_storage::{AsyncStorage, rustdoc_archive_path, source_archive_path}; use docs_rs_storage::{compress, decompress, rustdoc_json_path}; use docs_rs_types::{ BuildId, BuildStatus, CrateId, KrateName, ReleaseId, ReqVersion, SimpleBuildError, Version, @@ -113,15 +113,14 @@ async fn import_test_release_inner( .await?; let mut algs = HashSet::new(); - let (source_files_list, source_size) = { + let source_stats = { info!("writing source files to storage..."); - let (files_list, new_alg) = storage + let stats = storage .store_all_in_archive(&source_archive_path(name, version), &source_dir) .await?; - algs.insert(new_alg); - let source_size: u64 = files_list.iter().map(|info| info.size).sum(); - (files_list, source_size) + algs.insert(stats.alg); + stats }; let registry_data = registry_api @@ -178,11 +177,10 @@ async fn import_test_release_inner( } info!("writing rustdoc files to storage..."); - let (rustdoc_file_list, new_alg) = storage + let doc_stats = storage .store_all_in_archive(&rustdoc_archive_path(name, version), &rustdoc_dir) .await?; - let documentation_size: u64 = rustdoc_file_list.iter().map(|info| info.size).sum(); - algs.insert(new_alg); + algs.insert(doc_stats.alg); info!("loading repository stats..."); let repository_id = repository_stats @@ -232,14 +230,13 @@ async fn import_test_release_inner( cargo_metadata.root(), &source_dir, default_target, - file_list_to_json(source_files_list), all_targets, ®istry_data, true, false, // FIXME: real has_examples? algs, repository_id, - source_size, + source_stats.original_size, ) .await?; @@ -249,7 +246,7 @@ async fn import_test_release_inner( "rustc 1.95.0-nightly (873d4682c 2026-01-25)", BUILD_VERSION, BuildStatus::Success, - Some(documentation_size), + Some(doc_stats.original_size), None, None::<&SimpleBuildError>, ) diff --git a/crates/bin/docs_rs_web/src/handlers/source.rs b/crates/bin/docs_rs_web/src/handlers/source.rs index f0cf9ee0a4..f9a148b5e6 100644 --- a/crates/bin/docs_rs_web/src/handlers/source.rs +++ b/crates/bin/docs_rs_web/src/handlers/source.rs @@ -425,38 +425,6 @@ mod tests { }); } - #[test] - fn empty_file_list_dont_break_the_view() { - async_wrapper(|env| async move { - let release_id = env - .fake_release() - .await - .name("fake") - .version("0.1.0") - .source_file("README.md", b"hello") - .create() - .await?; - - let path = "/crate/fake/0.1.0/source/README.md"; - let web = env.web_app().await; - web.assert_success(path).await?; - - let mut conn = env.async_conn().await?; - sqlx::query!( - "UPDATE releases - SET files = NULL - WHERE id = $1", - release_id.0, - ) - .execute(&mut *conn) - .await?; - - assert!(web.get(path).await?.status().is_success()); - - Ok(()) - }); - } - #[test] fn latest_contains_links_to_latest() { async_wrapper(|env| async move { diff --git a/crates/lib/docs_rs_database/migrations/20260811210608_remove-release-files-archive.down.sql b/crates/lib/docs_rs_database/migrations/20260811210608_remove-release-files-archive.down.sql new file mode 100644 index 0000000000..113a16d286 --- /dev/null +++ b/crates/lib/docs_rs_database/migrations/20260811210608_remove-release-files-archive.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE releases +ADD COLUMN archive_storage BOOL NOT NULL DEFAULT TRUE, +ADD COLUMN files JSON ; diff --git a/crates/lib/docs_rs_database/migrations/20260811210608_remove-release-files-archive.up.sql b/crates/lib/docs_rs_database/migrations/20260811210608_remove-release-files-archive.up.sql new file mode 100644 index 0000000000..8349967f20 --- /dev/null +++ b/crates/lib/docs_rs_database/migrations/20260811210608_remove-release-files-archive.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE releases +DROP COLUMN files, +DROP COLUMN archive_storage; diff --git a/crates/lib/docs_rs_database/src/releases.rs b/crates/lib/docs_rs_database/src/releases.rs index 38eee54601..a2a1390165 100644 --- a/crates/lib/docs_rs_database/src/releases.rs +++ b/crates/lib/docs_rs_database/src/releases.rs @@ -8,7 +8,6 @@ use docs_rs_types::{ }; use docs_rs_utils::rustc_version::parse_rustc_date; use futures_util::stream::TryStreamExt; -use serde_json::Value; use slug::slugify; use std::{ collections::{HashMap, HashSet}, @@ -21,11 +20,8 @@ use tracing::{debug, error, info, instrument}; /// Adds a package into database. /// /// Package must be built first. -/// -/// NOTE: `source_files` refers to the files originally in the crate, -/// not the files generated by rustdoc. #[allow(clippy::too_many_arguments)] -#[instrument(skip(conn, compression_algorithms, source_files))] +#[instrument(skip(conn, compression_algorithms))] pub async fn finish_release( conn: &mut sqlx::PgConnection, crate_id: CrateId, @@ -33,7 +29,6 @@ pub async fn finish_release( metadata_pkg: &MetadataPackage, source_dir: impl AsRef + fmt::Debug, default_target: &str, - source_files: Value, doc_targets: Vec, registry_data: &ReleaseData, has_docs: bool, @@ -72,14 +67,13 @@ pub async fn finish_release( keywords = $14, have_examples = $15, downloads = $16, - files = $17, - doc_targets = $18, - is_library = $19, - documentation_url = $20, - default_target = $21, - features = $22, - repository_id = $23, - source_size = $24 + doc_targets = $17, + is_library = $18, + documentation_url = $19, + default_target = $20, + features = $21, + repository_id = $22, + source_size = $23 WHERE id = $1"#, release_id.0, registry_data.release_time, @@ -97,7 +91,6 @@ pub async fn finish_release( serde_json::to_value(&metadata_pkg.keywords)?, has_examples, registry_data.downloads, - source_files, serde_json::to_value(doc_targets)?, is_library, metadata_pkg.documentation, @@ -337,8 +330,8 @@ pub async fn initialize_release( version: &Version, ) -> Result { let release_id = sqlx::query_scalar!( - r#"INSERT INTO releases (crate_id, version, archive_storage) - VALUES ($1, $2, TRUE) + r#"INSERT INTO releases (crate_id, version) + VALUES ($1, $2) ON CONFLICT (crate_id, version) DO UPDATE SET -- this `SET` is needed so the id is always returned. version = EXCLUDED.version @@ -697,7 +690,6 @@ mod test { }, tempdir.path(), DEFAULT_TARGET, - Value::Array(vec![]), vec![DEFAULT_TARGET.to_string()], &ReleaseData::default(), true, @@ -1471,7 +1463,6 @@ mod test { }, tempdir.path(), DEFAULT_TARGET, - Value::Array(vec![]), vec![DEFAULT_TARGET.to_string()], &ReleaseData::default(), true, diff --git a/crates/lib/docs_rs_storage/Cargo.toml b/crates/lib/docs_rs_storage/Cargo.toml index 3331ec9012..1980060474 100644 --- a/crates/lib/docs_rs_storage/Cargo.toml +++ b/crates/lib/docs_rs_storage/Cargo.toml @@ -45,7 +45,6 @@ mime = { workspace = true } moka = { version = "0.12.14", features = ["future"] } opentelemetry = { workspace = true } rand = { workspace = true, optional = true } -serde_json = { workspace = true } sqlx = { workspace = true } # for sqlite strum = { workspace = true } tempfile = { workspace = true } diff --git a/crates/lib/docs_rs_storage/src/file.rs b/crates/lib/docs_rs_storage/src/file.rs index b26485d0e3..2ec0e91850 100644 --- a/crates/lib/docs_rs_storage/src/file.rs +++ b/crates/lib/docs_rs_storage/src/file.rs @@ -1,6 +1,5 @@ use docs_rs_mimes::detect_mime; use mime::Mime; -use serde_json::Value; use std::path::PathBuf; /// represents a file path from our source or documentation builds. @@ -17,21 +16,6 @@ impl FileEntry { } } -pub fn file_list_to_json(files: impl IntoIterator) -> Value { - Value::Array( - files - .into_iter() - .map(|info| { - Value::Array(vec![ - Value::String(info.mime().as_ref().to_string()), - Value::String(info.path.into_os_string().into_string().unwrap()), - Value::Number(info.size.into()), - ]) - }) - .collect(), - ) -} - #[derive(Debug, Clone, Eq)] pub enum FolderEntry { File(String, Mime), diff --git a/crates/lib/docs_rs_storage/src/lib.rs b/crates/lib/docs_rs_storage/src/lib.rs index 26f97733c2..f9f23278a0 100644 --- a/crates/lib/docs_rs_storage/src/lib.rs +++ b/crates/lib/docs_rs_storage/src/lib.rs @@ -6,6 +6,7 @@ mod config; pub(crate) mod errors; mod file; mod metrics; +mod result; pub(crate) mod storage; #[cfg(any(test, feature = "testing"))] pub mod testing; @@ -16,7 +17,8 @@ pub use blob::{Blob, BlobUpload, StreamingBlob}; pub use compression::{compress, compress_async, decompress}; pub use config::Config; pub use errors::{PathNotFoundError, SizeLimitReached}; -pub use file::{FileEntry, FolderEntry, file_list_to_json}; +pub use file::{FileEntry, FolderEntry}; +pub use result::ArchiveStatistics; pub use storage::blocking::Storage; pub use storage::non_blocking::AsyncStorage; pub use types::StorageKind; diff --git a/crates/lib/docs_rs_storage/src/result.rs b/crates/lib/docs_rs_storage/src/result.rs new file mode 100644 index 0000000000..d9c50a4a19 --- /dev/null +++ b/crates/lib/docs_rs_storage/src/result.rs @@ -0,0 +1,22 @@ +use docs_rs_types::CompressionAlgorithm; + +/// When we create an zip archive for source or rustdoc files, +/// we collect some statistics we need. +pub struct ArchiveStatistics { + /// used compression algorithm + pub alg: CompressionAlgorithm, + /// original size of all files + pub original_size: u64, + /// file count in the archive. + pub file_count: u64, +} + +impl ArchiveStatistics { + pub fn new(alg: CompressionAlgorithm) -> Self { + Self { + alg, + original_size: 0, + file_count: 0, + } + } +} diff --git a/crates/lib/docs_rs_storage/src/storage/blocking.rs b/crates/lib/docs_rs_storage/src/storage/blocking.rs index e8ffb056c4..ea7420895c 100644 --- a/crates/lib/docs_rs_storage/src/storage/blocking.rs +++ b/crates/lib/docs_rs_storage/src/storage/blocking.rs @@ -1,4 +1,6 @@ -use crate::{blob::Blob, file::FileEntry, storage::non_blocking::AsyncStorage}; +use crate::{ + blob::Blob, file::FileEntry, result::ArchiveStatistics, storage::non_blocking::AsyncStorage, +}; use anyhow::Result; use docs_rs_types::{BuildId, CompressionAlgorithm, KrateName, Version}; use std::{fmt, path::Path, sync::Arc}; @@ -62,7 +64,7 @@ impl Storage { &self, archive_path: &str, root_dir: &Path, - ) -> Result<(Vec, CompressionAlgorithm)> { + ) -> Result { self.runtime .block_on(self.inner.store_all_in_archive(archive_path, root_dir)) } diff --git a/crates/lib/docs_rs_storage/src/storage/non_blocking.rs b/crates/lib/docs_rs_storage/src/storage/non_blocking.rs index 2109d4301b..cbd8fe172a 100644 --- a/crates/lib/docs_rs_storage/src/storage/non_blocking.rs +++ b/crates/lib/docs_rs_storage/src/storage/non_blocking.rs @@ -9,6 +9,7 @@ use crate::{ errors::PathNotFoundError, file::FileEntry, metrics::StorageMetrics, + result::ArchiveStatistics, types::{FileRange, StorageKind}, utils::{ file_list::{get_file_list, walk_dir_recursive}, @@ -275,7 +276,7 @@ impl AsyncStorage { &self, archive_path: &str, root_dir: impl AsRef + fmt::Debug, - ) -> Result<(Vec, CompressionAlgorithm)> { + ) -> Result { let root_dir = root_dir.as_ref(); // Keep the TempPath guards alive until after both uploads complete; dropping them earlier @@ -283,16 +284,15 @@ impl AsyncStorage { let zip_temp_path = tempfile::NamedTempFile::new()?.into_temp_path(); let zip_path = zip_temp_path.to_path_buf(); - let file_paths = - spawn_blocking({ + let stats = spawn_blocking({ use std::{io, fs}; let archive_path = archive_path.to_owned(); let root_dir = root_dir.to_owned(); let zip_path = zip_path.clone(); - move || { - let mut file_paths = Vec::new(); + let mut stats = ArchiveStatistics::new(CompressionAlgorithm::Deflate); + move || { // We are only using the `zip` library to create the archives and the matching // index-file. The ZIP format allows more compression formats, and these can even be mixed // in a single archive. @@ -322,20 +322,23 @@ impl AsyncStorage { let mut file = fs::File::open(root_dir.join(&file_path))?; zip.start_file(file_path.to_str().unwrap(), options)?; io::copy(&mut file, &mut zip)?; - file_paths.push(FileEntry{path: file_path, size: file.metadata()?.len()}); + + stats.file_count +=1 ; + stats.original_size += file.metadata()?.len(); } let mut zip_file = zip.finish()?.into_inner()?; zip_file.flush()?; } - Ok(file_paths) + Ok(stats) } }) .await?; - let alg = CompressionAlgorithm::default(); let remote_index_path = format!("{}.{ARCHIVE_INDEX_FILE_EXTENSION}", archive_path); + let index_compression_alg = CompressionAlgorithm::default(); + let compressed_index_temp_path = tempfile::NamedTempFile::new()?.into_temp_path(); let compressed_index_path = compressed_index_temp_path.to_path_buf(); { @@ -356,7 +359,7 @@ impl AsyncStorage { compress_async( &mut io::BufReader::new(fs::File::open(&local_index_path).await?), &mut compressed_index_writer, - alg, + index_compression_alg, ) .await?; compressed_index_writer.flush().await?; @@ -374,11 +377,11 @@ impl AsyncStorage { path: remote_index_path, mime: mime::APPLICATION_OCTET_STREAM, source: StreamUploadSource::File(compressed_index_path), - compression: Some(alg), + compression: Some(index_compression_alg), }) )?; - Ok((file_paths, CompressionAlgorithm::Deflate)) + Ok(stats) } /// Store all files in `root_dir` into the backend under `prefix`. @@ -809,35 +812,26 @@ mod backend_tests { fs::write(path, "data").await?; } + const ARCHIVE_PATH: &str = "folder/test.zip"; + let local_index_location = storage .config .archive_index_cache .path - .join(format!("folder/test.zip.0.{ARCHIVE_INDEX_FILE_EXTENSION}")); + .join(format!("{ARCHIVE_PATH}.0.{ARCHIVE_INDEX_FILE_EXTENSION}")); - let (stored_files, compression_alg) = storage - .store_all_in_archive("folder/test.zip", dir.path()) + let stats = storage + .store_all_in_archive(ARCHIVE_PATH, dir.path()) .await?; assert!( storage - .exists(&format!("folder/test.zip.{ARCHIVE_INDEX_FILE_EXTENSION}")) + .exists(&format!("{ARCHIVE_PATH}.{ARCHIVE_INDEX_FILE_EXTENSION}")) .await? ); - assert_eq!(compression_alg, CompressionAlgorithm::Deflate); - assert_eq!(stored_files.len(), files.len()); - for name in &files { - assert!(get_file_info(&stored_files, name).is_some()); - } - assert_eq!( - get_file_info(&stored_files, "Cargo.toml").unwrap().mime(), - "text/toml" - ); - assert_eq!( - get_file_info(&stored_files, "src/main.rs").unwrap().mime(), - "text/rust" - ); + assert_eq!(stats.alg, CompressionAlgorithm::Deflate); + assert_eq!(stats.file_count, files.len() as u64); // delete the existing index to test the download of it if local_index_location.exists() { diff --git a/crates/lib/docs_rs_test_fakes/src/legacy.rs b/crates/lib/docs_rs_test_fakes/src/legacy.rs index f29887ec68..b5246892cb 100644 --- a/crates/lib/docs_rs_test_fakes/src/legacy.rs +++ b/crates/lib/docs_rs_test_fakes/src/legacy.rs @@ -11,12 +11,12 @@ use docs_rs_database::{ use docs_rs_registry_api::{CrateData, CrateOwner, ReleaseData}; use docs_rs_rustdoc_json::{RUSTDOC_JSON_COMPRESSION_ALGORITHMS, RustdocJsonFormatVersion}; use docs_rs_storage::{ - AsyncStorage, FileEntry, compress, file_list_to_json, rustdoc_archive_path, rustdoc_json_path, + ArchiveStatistics, AsyncStorage, compress, rustdoc_archive_path, rustdoc_json_path, source_archive_path, }; use docs_rs_types::{ - BuildError, BuildId, BuildStatus, CompressionAlgorithm, DocCoverage, KrateName, ReleaseId, - SimpleBuildError, Version, VersionReq, + BuildError, BuildId, BuildStatus, DocCoverage, KrateName, ReleaseId, SimpleBuildError, Version, + VersionReq, }; use std::{ collections::{BTreeMap, HashMap}, @@ -418,7 +418,7 @@ impl<'a> FakeRelease<'a> { source_directory: &Path, package: &MetadataPackage, storage: &AsyncStorage, - ) -> Result<(Vec, CompressionAlgorithm)> { + ) -> Result { debug!( "adding directory {:?} from {}", kind, @@ -430,9 +430,11 @@ impl<'a> FakeRelease<'a> { FileKind::Sources => source_archive_path(&krate_name, &package.version), }; debug!("store in archive: {:?}", archive); - storage + let stats = storage .store_all_in_archive(&archive, source_directory) - .await + .await?; + + Ok(stats) } debug!("before upload source"); @@ -456,9 +458,8 @@ impl<'a> FakeRelease<'a> { store_files_into(&[("Cargo.toml", content.as_bytes())], source_tmp.path())?; } - let (source_meta, algs) = - upload_files(FileKind::Sources, source_tmp.path(), &package, &storage).await?; - debug!(?source_meta, "added source files"); + let stats = upload_files(FileKind::Sources, source_tmp.path(), &package, &storage).await?; + debug!("added source files"); // If the test didn't add custom builds, inject a default one let builds = self.builds.unwrap_or_else(|| vec![FakeBuild::default()]); @@ -485,9 +486,8 @@ impl<'a> FakeRelease<'a> { debug!("added platform files for {}", platform); } - let (files, _) = - upload_files(FileKind::Rustdoc, rustdoc_path, &package, &storage).await?; - debug!(?files, "uploaded rustdoc files"); + upload_files(FileKind::Rustdoc, rustdoc_path, &package, &storage).await?; + debug!("uploaded rustdoc files"); } let mut async_conn = pool.get_async().await?; @@ -557,12 +557,11 @@ impl<'a> FakeRelease<'a> { &package, crate_dir, default_target, - file_list_to_json(source_meta), self.doc_targets, &self.registry_release_data, self.has_docs, self.has_examples, - iter::once(algs), + iter::once(stats.alg), repository, 24, )