don't compress rustdoc json twice

Author: syphar

src/docbuilder/rustwide_builder.rs

@@ -909,21 +909,11 @@ impl RustwideBuilder {
                 .context("couldn't parse rustdoc json to find format version")?
         };
 
-        let compressed_json: Vec<u8> = {
-            let _span =
-                info_span!("compress_json", file_size = json_filename.metadata()?.len()).entered();
-
-            compress(
-                BufReader::new(File::open(&json_filename)?),
-                CompressionAlgorithm::Zstd,
-            )?
-        };
-
         for format_version in [format_version, RustdocJsonFormatVersion::Latest] {
             let _span = info_span!("store_json", %format_version).entered();
             let path = rustdoc_json_path(name, version, target, format_version);
 
-            self.storage.store_one(&path, compressed_json.clone())?;
+            self.storage.store_path(&path, &json_filename)?;
             self.storage.set_public_access(&path, true)?;
         }
 

src/storage/mod.rs

@@ -24,7 +24,8 @@ use mime::Mime;
 use path_slash::PathExt;
 use serde_with::{DeserializeFromStr, SerializeDisplay};
 use std::{
-    fmt, fs,
+    fmt,
+    fs::{self, File},
     io::{self, BufReader},
     num::ParseIntError,
     ops::RangeInclusive,
@@ -569,6 +570,33 @@ impl AsyncStorage {
         Ok(alg)
     }
 
+    #[instrument(skip(self))]
+    pub(crate) async fn store_path(
+        &self,
+        target_path: impl Into<String> + std::fmt::Debug,
+        source_path: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<CompressionAlgorithm> {
+        let target_path = target_path.into();
+        let source_path = source_path.as_ref();
+
+        let alg = CompressionAlgorithm::default();
+        let content = compress(BufReader::new(File::open(&source_path)?), alg)?;
+
+        let mime = detect_mime(&target_path).to_owned();
+
+        self.store_inner(vec![Blob {
+            path: target_path,
+            mime,
+            content,
+            compression: Some(alg),
+            // this field is ignored by the backend
+            date_updated: Utc::now(),
+        }])
+        .await?;
+
+        Ok(alg)
+    }
+
     async fn store_inner(&self, batch: Vec<Blob>) -> Result<()> {
         match &self.backend {
             StorageBackend::Database(db) => db.store_batch(batch).await,
@@ -777,6 +805,18 @@ impl Storage {
         self.runtime.block_on(self.inner.store_one(path, content))
     }
 
+    // Store file into the backend at the given path (also used to detect mime type), returns the
+    // chosen compression algorithm
+    #[instrument(skip(self))]
+    pub(crate) fn store_path(
+        &self,
+        target_path: impl Into<String> + std::fmt::Debug,
+        source_path: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<CompressionAlgorithm> {
+        self.runtime
+            .block_on(self.inner.store_path(target_path, source_path))
+    }
+
     /// sync wrapper for the list_prefix function
     /// purely for testing purposes since it collects all files into a Vec.
     #[cfg(test)]
@@ -843,7 +883,7 @@ pub(crate) fn rustdoc_json_path(
     format_version: RustdocJsonFormatVersion,
 ) -> String {
     format!(
-        "rustdoc-json/{name}/{version}/{target}/{name}_{version}_{target}_{format_version}.json.zst"
+        "rustdoc-json/{name}/{version}/{target}/{name}_{version}_{target}_{format_version}.json"
     )
 }