From 92fc1f8f9e0b38e6d292514a6f3a6e8627ae0cdc Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 28 Oct 2020 14:41:02 +0100 Subject: [PATCH] build-manifest: include artifacts in a new table This commit adds to the generated manifest all files we ship that are not rustup components, namely: * Source code tarballs (rustc-{channel}-src.tar.xz) * Windows installers (rust-{channel}-{target}.msi) * macOS installers (rust-{channel}-{target}.pkg) Those files are included in a new "artifacts" table of the manifest, to avoid interfering with existing rustup installations. --- src/tools/build-manifest/src/main.rs | 33 ++++++++ src/tools/build-manifest/src/manifest.rs | 95 +++++++++++++++++++----- src/tools/build-manifest/src/versions.rs | 19 ++++- 3 files changed, 125 insertions(+), 22 deletions(-) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index ffcf10571ca..e36a43ff37f 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -171,6 +171,16 @@ static DOCS_TARGETS: &[&str] = &[ "x86_64-unknown-linux-musl", ]; +static MSI_INSTALLERS: &[&str] = &[ + "aarch64-pc-windows-msvc", + "i686-pc-windows-gnu", + "i686-pc-windows-msvc", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", +]; + +static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"]; + static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"]; static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-analyzer-preview"]; @@ -313,10 +323,12 @@ impl Builder { manifest_version: "2".to_string(), date: self.date.to_string(), pkg: BTreeMap::new(), + artifacts: BTreeMap::new(), renames: BTreeMap::new(), profiles: BTreeMap::new(), }; self.add_packages_to(&mut manifest); + self.add_artifacts_to(&mut manifest); self.add_profiles_to(&mut manifest); self.add_renames_to(&mut manifest); manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest)); @@ -345,6 +357,27 @@ impl Builder { package("llvm-tools-preview", TARGETS); } + fn add_artifacts_to(&mut self, manifest: &mut Manifest) { + manifest.add_artifact("source-code", |artifact| { + let tarball = self.versions.tarball_name(&PkgType::Rustc, "src").unwrap(); + artifact.add_tarball(self, "*", &tarball); + }); + + manifest.add_artifact("installer-msi", |artifact| { + for target in MSI_INSTALLERS { + let msi = self.versions.archive_name(&PkgType::Rust, target, "msi").unwrap(); + artifact.add_file(self, target, &msi); + } + }); + + manifest.add_artifact("installer-pkg", |artifact| { + for target in PKG_INSTALLERS { + let pkg = self.versions.archive_name(&PkgType::Rust, target, "pkg").unwrap(); + artifact.add_file(self, target, &pkg); + } + }); + } + fn add_profiles_to(&mut self, manifest: &mut Manifest) { let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs); profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]); diff --git a/src/tools/build-manifest/src/manifest.rs b/src/tools/build-manifest/src/manifest.rs index 873f709a532..547c270d89a 100644 --- a/src/tools/build-manifest/src/manifest.rs +++ b/src/tools/build-manifest/src/manifest.rs @@ -9,10 +9,19 @@ pub(crate) struct Manifest { pub(crate) manifest_version: String, pub(crate) date: String, pub(crate) pkg: BTreeMap, + pub(crate) artifacts: BTreeMap, pub(crate) renames: BTreeMap, pub(crate) profiles: BTreeMap>, } +impl Manifest { + pub(crate) fn add_artifact(&mut self, name: &str, f: impl FnOnce(&mut Artifact)) { + let mut artifact = Artifact { target: BTreeMap::new() }; + f(&mut artifact); + self.artifacts.insert(name.to_string(), artifact); + } +} + #[derive(Serialize)] pub(crate) struct Package { pub(crate) version: String, @@ -25,6 +34,42 @@ pub(crate) struct Rename { pub(crate) to: String, } +#[derive(Serialize)] +pub(crate) struct Artifact { + pub(crate) target: BTreeMap>, +} + +impl Artifact { + pub(crate) fn add_file(&mut self, builder: &mut Builder, target: &str, path: &str) { + if let Some(path) = record_shipped_file(builder, builder.input.join(path)) { + self.target.entry(target.into()).or_insert_with(Vec::new).push(ArtifactFile { + url: builder.url(&path), + hash_sha256: FileHash::Missing(path), + }); + } + } + + pub(crate) fn add_tarball(&mut self, builder: &mut Builder, target: &str, base_path: &str) { + let files = self.target.entry(target.into()).or_insert_with(Vec::new); + let base_path = builder.input.join(base_path); + for compression in &["gz", "xz"] { + if let Some(tarball) = tarball_variant(builder, &base_path, compression) { + files.push(ArtifactFile { + url: builder.url(&tarball), + hash_sha256: FileHash::Missing(tarball), + }); + } + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct ArtifactFile { + pub(crate) url: String, + pub(crate) hash_sha256: FileHash, +} + #[derive(Serialize, Default)] pub(crate) struct Target { pub(crate) available: bool, @@ -39,8 +84,8 @@ pub(crate) struct Target { impl Target { pub(crate) fn from_compressed_tar(builder: &mut Builder, base_path: &str) -> Self { let base_path = builder.input.join(base_path); - let gz = Self::tarball_variant(builder, &base_path, "gz"); - let xz = Self::tarball_variant(builder, &base_path, "xz"); + let gz = tarball_variant(builder, &base_path, "gz"); + let xz = tarball_variant(builder, &base_path, "xz"); if gz.is_none() { return Self::unavailable(); @@ -59,23 +104,6 @@ impl Target { } } - fn tarball_variant(builder: &mut Builder, base: &Path, ext: &str) -> Option { - let mut path = base.to_path_buf(); - path.set_extension(ext); - if path.is_file() { - builder.shipped_files.insert( - path.file_name() - .expect("missing filename") - .to_str() - .expect("non-utf-8 filename") - .to_string(), - ); - Some(path) - } else { - None - } - } - pub(crate) fn unavailable() -> Self { Self::default() } @@ -111,6 +139,27 @@ impl Serialize for FileHash { } } +fn tarball_variant(builder: &mut Builder, base: &Path, ext: &str) -> Option { + let mut path = base.to_path_buf(); + path.set_extension(ext); + record_shipped_file(builder, path) +} + +fn record_shipped_file(builder: &mut Builder, path: PathBuf) -> Option { + if path.is_file() { + builder.shipped_files.insert( + path.file_name() + .expect("missing filename") + .to_str() + .expect("non-utf-8 filename") + .to_string(), + ); + Some(path) + } else { + None + } +} + pub(crate) fn visit_file_hashes(manifest: &mut Manifest, mut f: impl FnMut(&mut FileHash)) { for pkg in manifest.pkg.values_mut() { for target in pkg.target.values_mut() { @@ -122,4 +171,12 @@ pub(crate) fn visit_file_hashes(manifest: &mut Manifest, mut f: impl FnMut(&mut } } } + + for artifact in manifest.artifacts.values_mut() { + for target in artifact.target.values_mut() { + for file in target { + f(&mut file.hash_sha256); + } + } + } } diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 79f2ef8dfc4..f1a42e7145f 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -13,6 +13,7 @@ const RUSTC_VERSION: &str = include_str!("../../../version"); pub(crate) enum PkgType { Rust, RustSrc, + Rustc, Cargo, Rls, RustAnalyzer, @@ -28,6 +29,7 @@ impl PkgType { match component { "rust" => PkgType::Rust, "rust-src" => PkgType::RustSrc, + "rustc" => PkgType::Rustc, "cargo" => PkgType::Cargo, "rls" | "rls-preview" => PkgType::Rls, "rust-analyzer" | "rust-analyzer-preview" => PkgType::RustAnalyzer, @@ -44,6 +46,7 @@ impl PkgType { match self { PkgType::Rust => "rust", PkgType::RustSrc => "rust-src", + PkgType::Rustc => "rustc", PkgType::Cargo => "cargo", PkgType::Rls => "rls", PkgType::RustAnalyzer => "rust-analyzer", @@ -69,6 +72,7 @@ impl PkgType { PkgType::Rust => true, PkgType::RustSrc => true, + PkgType::Rustc => true, PkgType::Other(_) => true, } } @@ -165,10 +169,11 @@ impl Versions { } } - pub(crate) fn tarball_name( + pub(crate) fn archive_name( &mut self, package: &PkgType, target: &str, + extension: &str, ) -> Result { let component_name = package.tarball_component_name(); let version = match self.channel.as_str() { @@ -179,12 +184,20 @@ impl Versions { }; if package.target_independent() { - Ok(format!("{}-{}.tar.gz", component_name, version)) + Ok(format!("{}-{}.{}", component_name, version, extension)) } else { - Ok(format!("{}-{}-{}.tar.gz", component_name, version, target)) + Ok(format!("{}-{}-{}.{}", component_name, version, target, extension)) } } + pub(crate) fn tarball_name( + &mut self, + package: &PkgType, + target: &str, + ) -> Result { + self.archive_name(package, target, "tar.gz") + } + pub(crate) fn rustc_version(&self) -> &str { RUSTC_VERSION }