squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

Auto merge of #78486 - pietroalbini:manifest-artifacts, r=Mark-Simulacrum

Browse source 
Include non-rustup artifacts in the manifest

This PR fixes https://github.com/rust-lang/promote-release/issues/22 by including all the files we ship in the generated manifests, even the ones that are not installable through rustup. In practice this adds the following "artifacts":

* `source-code`: the tarball containing the full source code used to build the release (`rustc-{channel}-src.tar.xz`)
* `installer-msi`: the MSI installer for Windows systems (`rust-{channel}-{target}.msi`)
* `installer-pkg`: the PKG installer for macOS systems (`rust-{channel}-{target}.pkg`)

These files are included in a new `artifacts` table of the manifest, like so:

```toml
[[artifacts.installer-msi.target.aarch64-pc-windows-msvc]]
url = "https://example.com/2020-10-28/rust-nightly-aarch64-pc-windows-msvc.msi"
hash-sha256 = "6b41d5b829d20834c5d93628d008ec618f8914ee79303363bd13a86fd5f305dd"

[[artifacts.installer-msi.target.i686-pc-windows-gnu]]
url = "https://example.com/2020-10-28/rust-nightly-i686-pc-windows-gnu.msi"
hash-sha256 = "83f020de6e180c155add9fce1cea2ac6e5f744edbd6dc1581e24de8f56b2ca7a"

[[artifacts.installer-msi.target.i686-pc-windows-msvc]]
url = "https://example.com/2020-10-28/rust-nightly-i686-pc-windows-msvc.msi"
hash-sha256 = "dbc80c24e9d5df01616c6f216114b4351f51a94218e2368b5cebe4165b270702"

[[artifacts.installer-msi.target.x86_64-pc-windows-gnu]]
url = "https://example.com/2020-10-28/rust-nightly-x86_64-pc-windows-gnu.msi"
hash-sha256 = "8196eca3f02d72d4c8776ad4fcc72897125e2cf6404ae933e31c07e197e3c9fa"

[[artifacts.installer-msi.target.x86_64-pc-windows-msvc]]
url = "https://example.com/2020-10-28/rust-nightly-x86_64-pc-windows-msvc.msi"
hash-sha256 = "b2e7fd6463790732fcf9c726b9448068712341943199cb40fc11d1138b8a207b"

[[artifacts.installer-pkg.target.aarch64-apple-darwin]]
url = "https://example.com/2020-10-28/rust-nightly-aarch64-apple-darwin.pkg"
hash-sha256 = "70421c191752fb33886f8033b029e634bcc993b72308cef52a38405840e91f5c"

[[artifacts.installer-pkg.target.x86_64-apple-darwin]]
url = "https://example.com/2020-10-28/rust-nightly-x86_64-apple-darwin.pkg"
hash-sha256 = "ebd7a5acb61e82d85e855146cc9bd856f32228ee7f40dd94c659b00614ed4f1f"

[[artifacts.source-code.target."*"]]
url = "https://example.com/2020-10-28/rustc-nightly-src.tar.gz"
hash-sha256 = "5fcc487ee4c15c689de8ddf7daac7ff6a65c80498197b9aea58622dc2b3bca10"

[[artifacts.source-code.target."*"]]
url = "https://example.com/2020-10-28/rustc-nightly-src.tar.xz"
hash-sha256 = "0c618ef0ec5f64da1801e9d0df6c755f6ed1a8780ec5c8ee75e55614be51d42c"

```

Each artifact can be available for multiple targets, and each target can have multiple versions of the same file (for example, a `gz`-compressed one and a `xz`-compressed one). In the future rustup might add functionality to let users retrieve the artifacts, but that's not needed to land this PR, and whether to do the implementation is up to the rustup maintainers.

r? `@kinnison`
cc `@Mark-Simulacrum`
This commit is contained in:
bors 2020-10-29 06:18:12 +00:00
parent 49720d2b9a 92fc1f8f9e
commit 1d6a0b0c72
3 changed files with 125 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
src/tools/build-manifest/src/main.rs
View file

@ -168,6 +168,16 @@ static DOCS_TARGETS: &[&str] = &[
"x86_64-unknown-linux-musl",
];
static MSI_INSTALLERS: &[&str] = &[
"aarch64-pc-windows-msvc",
"i686-pc-windows-gnu",
"i686-pc-windows-msvc",
"x86_64-pc-windows-gnu",
"x86_64-pc-windows-msvc",
];
static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"];
static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-analyzer-preview"];
@ -314,10 +324,12 @@ impl Builder {
manifest_version: "2".to_string(),
date: self.date.to_string(),
pkg: BTreeMap::new(),
artifacts: BTreeMap::new(),
renames: BTreeMap::new(),
profiles: BTreeMap::new(),
};
self.add_packages_to(&mut manifest);
self.add_artifacts_to(&mut manifest);
self.add_profiles_to(&mut manifest);
self.add_renames_to(&mut manifest);
manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
@ -346,6 +358,27 @@ impl Builder {
package("llvm-tools-preview", TARGETS);
}
fn add_artifacts_to(&mut self, manifest: &mut Manifest) {
manifest.add_artifact("source-code", |artifact| {
let tarball = self.versions.tarball_name(&PkgType::Rustc, "src").unwrap();
artifact.add_tarball(self, "*", &tarball);
});
manifest.add_artifact("installer-msi", |artifact| {
for target in MSI_INSTALLERS {
let msi = self.versions.archive_name(&PkgType::Rust, target, "msi").unwrap();
artifact.add_file(self, target, &msi);
}
});
manifest.add_artifact("installer-pkg", |artifact| {
for target in PKG_INSTALLERS {
let pkg = self.versions.archive_name(&PkgType::Rust, target, "pkg").unwrap();
artifact.add_file(self, target, &pkg);
}
});
}
fn add_profiles_to(&mut self, manifest: &mut Manifest) {
let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);

95
src/tools/build-manifest/src/manifest.rs
View file

@ -9,10 +9,19 @@ pub(crate) struct Manifest {
pub(crate) manifest_version: String,
pub(crate) date: String,
pub(crate) pkg: BTreeMap<String, Package>,
pub(crate) artifacts: BTreeMap<String, Artifact>,
pub(crate) renames: BTreeMap<String, Rename>,
pub(crate) profiles: BTreeMap<String, Vec<String>>,
}
impl Manifest {
pub(crate) fn add_artifact(&mut self, name: &str, f: impl FnOnce(&mut Artifact)) {
let mut artifact = Artifact { target: BTreeMap::new() };
f(&mut artifact);
self.artifacts.insert(name.to_string(), artifact);
}
}
#[derive(Serialize)]
pub(crate) struct Package {
pub(crate) version: String,
@ -25,6 +34,42 @@ pub(crate) struct Rename {
pub(crate) to: String,
}
#[derive(Serialize)]
pub(crate) struct Artifact {
pub(crate) target: BTreeMap<String, Vec<ArtifactFile>>,
}
impl Artifact {
pub(crate) fn add_file(&mut self, builder: &mut Builder, target: &str, path: &str) {
if let Some(path) = record_shipped_file(builder, builder.input.join(path)) {
self.target.entry(target.into()).or_insert_with(Vec::new).push(ArtifactFile {
url: builder.url(&path),
hash_sha256: FileHash::Missing(path),
});
}
}
pub(crate) fn add_tarball(&mut self, builder: &mut Builder, target: &str, base_path: &str) {
let files = self.target.entry(target.into()).or_insert_with(Vec::new);
let base_path = builder.input.join(base_path);
for compression in &["gz", "xz"] {
if let Some(tarball) = tarball_variant(builder, &base_path, compression) {
files.push(ArtifactFile {
url: builder.url(&tarball),
hash_sha256: FileHash::Missing(tarball),
});
}
}
}
}
#[derive(Serialize)]
#[serde(rename_all = "kebab-case")]
pub(crate) struct ArtifactFile {
pub(crate) url: String,
pub(crate) hash_sha256: FileHash,
}
#[derive(Serialize, Default)]
pub(crate) struct Target {
pub(crate) available: bool,
@ -39,8 +84,8 @@ pub(crate) struct Target {
impl Target {
pub(crate) fn from_compressed_tar(builder: &mut Builder, base_path: &str) -> Self {
let base_path = builder.input.join(base_path);
let gz = Self::tarball_variant(builder, &base_path, "gz");
let xz = Self::tarball_variant(builder, &base_path, "xz");
let gz = tarball_variant(builder, &base_path, "gz");
let xz = tarball_variant(builder, &base_path, "xz");
if gz.is_none() {
return Self::unavailable();
@ -59,23 +104,6 @@ impl Target {
}
}
fn tarball_variant(builder: &mut Builder, base: &Path, ext: &str) -> Option<PathBuf> {
let mut path = base.to_path_buf();
path.set_extension(ext);
if path.is_file() {
builder.shipped_files.insert(
path.file_name()
.expect("missing filename")
.to_str()
.expect("non-utf-8 filename")
.to_string(),
);
Some(path)
} else {
None
}
}
pub(crate) fn unavailable() -> Self {
Self::default()
}
@ -111,6 +139,27 @@ impl Serialize for FileHash {
}
}
fn tarball_variant(builder: &mut Builder, base: &Path, ext: &str) -> Option<PathBuf> {
let mut path = base.to_path_buf();
path.set_extension(ext);
record_shipped_file(builder, path)
}
fn record_shipped_file(builder: &mut Builder, path: PathBuf) -> Option<PathBuf> {
if path.is_file() {
builder.shipped_files.insert(
path.file_name()
.expect("missing filename")
.to_str()
.expect("non-utf-8 filename")
.to_string(),
);
Some(path)
} else {
None
}
}
pub(crate) fn visit_file_hashes(manifest: &mut Manifest, mut f: impl FnMut(&mut FileHash)) {
for pkg in manifest.pkg.values_mut() {
for target in pkg.target.values_mut() {
@ -122,4 +171,12 @@ pub(crate) fn visit_file_hashes(manifest: &mut Manifest, mut f: impl FnMut(&mut
}
}
}
for artifact in manifest.artifacts.values_mut() {
for target in artifact.target.values_mut() {
for file in target {
f(&mut file.hash_sha256);
}
}
}
}

19
src/tools/build-manifest/src/versions.rs
View file

@ -13,6 +13,7 @@ const RUSTC_VERSION: &str = include_str!("../../../version");
pub(crate) enum PkgType {
Rust,
RustSrc,
Rustc,
Cargo,
Rls,
RustAnalyzer,
@ -28,6 +29,7 @@ impl PkgType {
match component {
"rust" => PkgType::Rust,
"rust-src" => PkgType::RustSrc,
"rustc" => PkgType::Rustc,
"cargo" => PkgType::Cargo,
"rls" | "rls-preview" => PkgType::Rls,
"rust-analyzer" | "rust-analyzer-preview" => PkgType::RustAnalyzer,
@ -44,6 +46,7 @@ impl PkgType {
match self {
PkgType::Rust => "rust",
PkgType::RustSrc => "rust-src",
PkgType::Rustc => "rustc",
PkgType::Cargo => "cargo",
PkgType::Rls => "rls",
PkgType::RustAnalyzer => "rust-analyzer",
@ -69,6 +72,7 @@ impl PkgType {
PkgType::Rust => true,
PkgType::RustSrc => true,
PkgType::Rustc => true,
PkgType::Other(_) => true,
}
}
@ -165,10 +169,11 @@ impl Versions {
}
}
pub(crate) fn tarball_name(
pub(crate) fn archive_name(
&mut self,
package: &PkgType,
target: &str,
extension: &str,
) -> Result<String, Error> {
let component_name = package.tarball_component_name();
let version = match self.channel.as_str() {
@ -179,12 +184,20 @@ impl Versions {
};
if package.target_independent() {
Ok(format!("{}-{}.tar.gz", component_name, version))
Ok(format!("{}-{}.{}", component_name, version, extension))
} else {
Ok(format!("{}-{}-{}.tar.gz", component_name, version, target))
Ok(format!("{}-{}-{}.{}", component_name, version, target, extension))
}
}
pub(crate) fn tarball_name(
&mut self,
package: &PkgType,
target: &str,
) -> Result<String, Error> {
self.archive_name(package, target, "tar.gz")
}
pub(crate) fn rustc_version(&self) -> &str {
RUSTC_VERSION
}