squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

Rollup merge of #87282 - pietroalbini:refactor-extended, r=Mark-Simulacrum

Browse source 
Ensure `./x.py dist` adheres to `build.tools`

According to `config.toml.example`, the way to produce dist artifacts for both the compiler and a *subset* of tools would be to enable the extended build and manually specify the list of tools to build:

```toml
[build]
extended = true
tools = ["cargo", "rustfmt"]
```

This works as expected for `./x.py build` and `./x.py install`, but *not* for `./x.py dist`. Before this PR `./x.py dist` simply ignored the contents of `build.tools`, building just rustc/rustdoc if `build.extended = false` and all of the tools otherwise. This PR does two things:

* Changes `./x.py dist extended` to only build the tools defined in `build.tools`, if `build.tools` is not empty. The rest of the extended step was refactored to simplify the code.
* Changes how dist jobs for tools are gated: instead of `assert!(builder.config.extended)` to prevent tools from being built with `build.extended = false`, tools are simply built by default depending on `build.extended` and `build.tools`. This also enables to **explicitly** dist tools even with `build.extended = false`.

This PR is best reviewed commit-by-commit.

Fixes #86436
This commit is contained in:
Yuki Okushi 2021-08-02 11:03:23 +09:00 committed by GitHub
commit 46f01caab8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 154 additions and 135 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
src/bootstrap/builder.rs
View file

@ -163,14 +163,8 @@ impl StepDescription {
}
fn maybe_run(&self, builder: &Builder<'_>, pathset: &PathSet) {
if builder.config.exclude.iter().any(|e| pathset.has(e)) {
eprintln!("Skipping {:?} because it is excluded", pathset);
if self.is_excluded(builder, pathset) {
return;
} else if !builder.config.exclude.is_empty() {
eprintln!(
"{:?} not skipped for {:?} -- not in {:?}",
pathset, self.name, builder.config.exclude
);
}
// Determine the targets participating in this rule.
@ -182,6 +176,21 @@ impl StepDescription {
}
}
fn is_excluded(&self, builder: &Builder<'_>, pathset: &PathSet) -> bool {
if builder.config.exclude.iter().any(|e| pathset.has(e)) {
eprintln!("Skipping {:?} because it is excluded", pathset);
return true;
}
if !builder.config.exclude.is_empty() {
eprintln!(
"{:?} not skipped for {:?} -- not in {:?}",
pathset, self.name, builder.config.exclude
);
}
false
}
fn run(v: &[StepDescription], builder: &Builder<'_>, paths: &[PathBuf]) {
let should_runs =
v.iter().map(|desc| (desc.should_run)(ShouldRun::new(builder))).collect::<Vec<_>>();
@ -1579,6 +1588,27 @@ impl<'a> Builder<'a> {
self.cache.put(step, out.clone());
out
}
/// Ensure that a given step is built *only if it's supposed to be built by default*, returning
/// its output. This will cache the step, so it's safe (and good!) to call this as often as
/// needed to ensure that all dependencies are build.
pub(crate) fn ensure_if_default<T, S: Step<Output = Option<T>>>(
&'a self,
step: S,
) -> S::Output {
let desc = StepDescription::from::<S>();
let should_run = (desc.should_run)(ShouldRun::new(self));
// Avoid running steps contained in --exclude
for pathset in &should_run.paths {
if desc.is_excluded(self, pathset) {
return None;
}
}
// Only execute if it's supposed to run as default
if desc.default && should_run.is_really_default() { self.ensure(step) } else { None }
}
}
#[cfg(test)]

229
src/bootstrap/dist.rs
View file

@ -8,6 +8,7 @@
//! out to `rust-installer` still. This may one day be replaced with bits and
//! pieces of `rustup.rs`!
use std::collections::HashSet;
use std::env;
use std::fs;
use std::path::{Path, PathBuf};
@ -45,6 +46,13 @@ fn missing_tool(tool_name: &str, skip: bool) {
}
}
fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
if !builder.config.extended {
return false;
}
builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
}
#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
pub struct Docs {
pub host: TargetSelection,
@ -55,7 +63,8 @@ impl Step for Docs {
const DEFAULT: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/doc")
let default = run.builder.config.docs;
run.path("src/doc").default_condition(default)
}
fn make_run(run: RunConfig<'_>) {
@ -65,9 +74,6 @@ impl Step for Docs {
/// Builds the `rust-docs` installer component.
fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let host = self.host;
if !builder.config.docs {
return None;
}
builder.default_doc(&[]);
let dest = "share/doc/rust/html";
@ -676,8 +682,8 @@ impl Step for Analysis {
const DEFAULT: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
let builder = run.builder;
run.path("analysis").default_condition(builder.config.extended)
let default = should_build_extended_tool(&run.builder, "analysis");
run.path("analysis").default_condition(default)
}
fn make_run(run: RunConfig<'_>) {
@ -698,7 +704,6 @@ impl Step for Analysis {
fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let compiler = self.compiler;
let target = self.target;
assert!(builder.config.extended);
if compiler.host != builder.config.build {
return None;
}
@ -953,11 +958,13 @@ pub struct Cargo {
}
impl Step for Cargo {
type Output = GeneratedTarball;
type Output = Option<GeneratedTarball>;
const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("cargo")
let default = should_build_extended_tool(&run.builder, "cargo");
run.path("cargo").default_condition(default)
}
fn make_run(run: RunConfig<'_>) {
@ -971,7 +978,7 @@ impl Step for Cargo {
});
}
fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let compiler = self.compiler;
let target = self.target;
@ -996,7 +1003,7 @@ impl Step for Cargo {
}
}
tarball.generate()
Some(tarball.generate())
}
}
@ -1009,9 +1016,11 @@ pub struct Rls {
impl Step for Rls {
type Output = Option<GeneratedTarball>;
const ONLY_HOSTS: bool = true;
const DEFAULT: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("rls")
let default = should_build_extended_tool(&run.builder, "rls");
run.path("rls").default_condition(default)
}
fn make_run(run: RunConfig<'_>) {
@ -1028,7 +1037,6 @@ impl Step for Rls {
fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let compiler = self.compiler;
let target = self.target;
assert!(builder.config.extended);
let rls = builder
.ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
@ -1054,10 +1062,12 @@ pub struct RustAnalyzer {
impl Step for RustAnalyzer {
type Output = Option<GeneratedTarball>;
const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("rust-analyzer")
let default = should_build_extended_tool(&run.builder, "rust-analyzer");
run.path("rust-analyzer").default_condition(default)
}
fn make_run(run: RunConfig<'_>) {
@ -1080,7 +1090,6 @@ impl Step for RustAnalyzer {
}
let compiler = self.compiler;
let target = self.target;
assert!(builder.config.extended);
if target.contains("riscv64") {
// riscv64 currently has an LLVM bug that makes rust-analyzer unable
@ -1108,11 +1117,13 @@ pub struct Clippy {
}
impl Step for Clippy {
type Output = GeneratedTarball;
type Output = Option<GeneratedTarball>;
const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("clippy")
let default = should_build_extended_tool(&run.builder, "clippy");
run.path("clippy").default_condition(default)
}
fn make_run(run: RunConfig<'_>) {
@ -1126,10 +1137,9 @@ impl Step for Clippy {
});
}
fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let compiler = self.compiler;
let target = self.target;
assert!(builder.config.extended);
// Prepare the image directory
// We expect clippy to build, because we've exited this step above if tool
@ -1147,7 +1157,7 @@ impl Step for Clippy {
tarball.add_file(clippy, "bin", 0o755);
tarball.add_file(cargoclippy, "bin", 0o755);
tarball.add_legal_and_readme_to("share/doc/clippy");
tarball.generate()
Some(tarball.generate())
}
}
@ -1159,10 +1169,12 @@ pub struct Miri {
impl Step for Miri {
type Output = Option<GeneratedTarball>;
const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("miri")
let default = should_build_extended_tool(&run.builder, "miri");
run.path("miri").default_condition(default)
}
fn make_run(run: RunConfig<'_>) {
@ -1185,7 +1197,6 @@ impl Step for Miri {
}
let compiler = self.compiler;
let target = self.target;
assert!(builder.config.extended);
let miri = builder
.ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
@ -1218,10 +1229,12 @@ pub struct Rustfmt {
impl Step for Rustfmt {
type Output = Option<GeneratedTarball>;
const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("rustfmt")
let default = should_build_extended_tool(&run.builder, "rustfmt");
run.path("rustfmt").default_condition(default)
}
fn make_run(run: RunConfig<'_>) {
@ -1270,10 +1283,17 @@ pub struct RustDemangler {
impl Step for RustDemangler {
type Output = Option<GeneratedTarball>;
const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("rust-demangler")
// While other tools use `should_build_extended_tool` to decide whether to be run by
// default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
// the other ones or if `profiler = true`. Because we don't know the target at this stage
// we run the step by default when only `extended = true`, and decide whether to actually
// run it or not later.
let default = run.builder.config.extended;
run.path("rust-demangler").default_condition(default)
}
fn make_run(run: RunConfig<'_>) {
@ -1290,11 +1310,11 @@ impl Step for RustDemangler {
fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let compiler = self.compiler;
let target = self.target;
assert!(builder.config.extended);
// Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
let profiler = builder.config.profiler_enabled(target);
if !builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler")) {
let condition = should_build_extended_tool(builder, "rust-demangler")
|| builder.config.profiler_enabled(target);
if builder.config.extended && !condition {
return None;
}
@ -1345,51 +1365,44 @@ impl Step for Extended {
builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
let rustc_installer = builder.ensure(Rustc { compiler: builder.compiler(stage, target) });
let cargo_installer = builder.ensure(Cargo { compiler, target });
let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
let rust_demangler_installer = builder.ensure(RustDemangler { compiler, target });
let rls_installer = builder.ensure(Rls { compiler, target });
let rust_analyzer_installer = builder.ensure(RustAnalyzer { compiler, target });
let llvm_tools_installer = builder.ensure(LlvmTools { target });
let clippy_installer = builder.ensure(Clippy { compiler, target });
let miri_installer = builder.ensure(Miri { compiler, target });
let mingw_installer = builder.ensure(Mingw { host: target });
let analysis_installer = builder.ensure(Analysis { compiler, target });
let docs_installer = builder.ensure(Docs { host: target });
let std_installer = builder.ensure(Std { compiler, target });
let etc = builder.src.join("src/etc/installer");
// Avoid producing tarballs during a dry run.
if builder.config.dry_run {
return;
let mut tarballs = Vec::new();
let mut built_tools = HashSet::new();
macro_rules! add_component {
($name:expr => $step:expr) => {
if let Some(tarball) = builder.ensure_if_default($step) {
tarballs.push(tarball);
built_tools.insert($name);
}
};
}
// When rust-std package split from rustc, we needed to ensure that during
// upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
// the std files during uninstall. To do this ensure that rustc comes
// before rust-std in the list below.
let mut tarballs = Vec::new();
tarballs.push(rustc_installer);
tarballs.push(cargo_installer);
tarballs.push(clippy_installer);
tarballs.extend(rust_demangler_installer.clone());
tarballs.extend(rls_installer.clone());
tarballs.extend(rust_analyzer_installer.clone());
tarballs.extend(miri_installer.clone());
tarballs.extend(rustfmt_installer.clone());
tarballs.extend(llvm_tools_installer);
if let Some(analysis_installer) = analysis_installer {
tarballs.push(analysis_installer);
tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
if target.contains("windows-gnu") {
tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
}
tarballs.push(std_installer.expect("missing std"));
if let Some(docs_installer) = docs_installer {
tarballs.push(docs_installer);
}
if target.contains("pc-windows-gnu") {
tarballs.push(mingw_installer.unwrap());
add_component!("rust-docs" => Docs { host: target });
add_component!("rust-demangler"=> RustDemangler { compiler, target });
add_component!("cargo" => Cargo { compiler, target });
add_component!("rustfmt" => Rustfmt { compiler, target });
add_component!("rls" => Rls { compiler, target });
add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
add_component!("llvm-components" => LlvmTools { target });
add_component!("clippy" => Clippy { compiler, target });
add_component!("miri" => Miri { compiler, target });
add_component!("analysis" => Analysis { compiler, target });
let etc = builder.src.join("src/etc/installer");
// Avoid producing tarballs during a dry run.
if builder.config.dry_run {
return;
}
let tarball = Tarball::new(builder, "rust", &target.triple);
@ -1434,20 +1447,10 @@ impl Step for Extended {
let xform = |p: &Path| {
let mut contents = t!(fs::read_to_string(p));
if rust_demangler_installer.is_none() {
contents = filter(&contents, "rust-demangler");
}
if rls_installer.is_none() {
contents = filter(&contents, "rls");
}
if rust_analyzer_installer.is_none() {
contents = filter(&contents, "rust-analyzer");
}
if miri_installer.is_none() {
contents = filter(&contents, "miri");
}
if rustfmt_installer.is_none() {
contents = filter(&contents, "rustfmt");
for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
if !built_tools.contains(tool) {
contents = filter(&contents, tool);
}
}
let ret = tmp.join(p.file_name().unwrap());
t!(fs::write(&ret, &contents));
@ -1485,19 +1488,11 @@ impl Step for Extended {
prepare("rust-std");
prepare("rust-analysis");
prepare("clippy");
if rust_demangler_installer.is_some() {
prepare("rust-demangler");
for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
if built_tools.contains(tool) {
prepare(tool);
}
}
if rls_installer.is_some() {
prepare("rls");
}
if rust_analyzer_installer.is_some() {
prepare("rust-analyzer");
}
if miri_installer.is_some() {
prepare("miri");
}
// create an 'uninstall' package
builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
pkgbuild("uninstall");
@ -1554,17 +1549,10 @@ impl Step for Extended {
prepare("rust-docs");
prepare("rust-std");
prepare("clippy");
if rust_demangler_installer.is_some() {
prepare("rust-demangler");
}
if rls_installer.is_some() {
prepare("rls");
}
if rust_analyzer_installer.is_some() {
prepare("rust-analyzer");
}
if miri_installer.is_some() {
prepare("miri");
for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
if built_tools.contains(tool) {
prepare(tool);
}
}
if target.contains("windows-gnu") {
prepare("rust-mingw");
@ -1643,7 +1631,7 @@ impl Step for Extended {
.arg("-out")
.arg(exe.join("StdGroup.wxs")),
);
if rls_installer.is_some() {
if built_tools.contains("rls") {
builder.run(
Command::new(&heat)
.current_dir(&exe)
@ -1662,7 +1650,7 @@ impl Step for Extended {
.arg(etc.join("msi/remove-duplicates.xsl")),
);
}
if rust_analyzer_installer.is_some() {
if built_tools.contains("rust-analyzer") {
builder.run(
Command::new(&heat)
.current_dir(&exe)
@ -1698,7 +1686,7 @@ impl Step for Extended {
.arg("-t")
.arg(etc.join("msi/remove-duplicates.xsl")),
);
if rust_demangler_installer.is_some() {
if built_tools.contains("rust-demangler") {
builder.run(
Command::new(&heat)
.current_dir(&exe)
@ -1717,7 +1705,7 @@ impl Step for Extended {
.arg(etc.join("msi/remove-duplicates.xsl")),
);
}
if miri_installer.is_some() {
if built_tools.contains("miri") {
builder.run(
Command::new(&heat)
.current_dir(&exe)
@ -1790,16 +1778,16 @@ impl Step for Extended {
.arg(&input);
add_env(builder, &mut cmd, target);
if rust_demangler_installer.is_some() {
if built_tools.contains("rust-demangler") {
cmd.arg("-dRustDemanglerDir=rust-demangler");
}
if rls_installer.is_some() {
if built_tools.contains("rls") {
cmd.arg("-dRlsDir=rls");
}
if rust_analyzer_installer.is_some() {
if built_tools.contains("rust-analyzer") {
cmd.arg("-dRustAnalyzerDir=rust-analyzer");
}
if miri_installer.is_some() {
if built_tools.contains("miri") {
cmd.arg("-dMiriDir=miri");
}
if target.contains("windows-gnu") {
@ -1815,16 +1803,16 @@ impl Step for Extended {
candle("CargoGroup.wxs".as_ref());
candle("StdGroup.wxs".as_ref());
candle("ClippyGroup.wxs".as_ref());
if rust_demangler_installer.is_some() {
if built_tools.contains("rust-demangler") {
candle("RustDemanglerGroup.wxs".as_ref());
}
if rls_installer.is_some() {
if built_tools.contains("rls") {
candle("RlsGroup.wxs".as_ref());
}
if rust_analyzer_installer.is_some() {
if built_tools.contains("rust-analyzer") {
candle("RustAnalyzerGroup.wxs".as_ref());
}
if miri_installer.is_some() {
if built_tools.contains("miri") {
candle("MiriGroup.wxs".as_ref());
}
candle("AnalysisGroup.wxs".as_ref());
@ -1858,16 +1846,16 @@ impl Step for Extended {
.arg("ClippyGroup.wixobj")
.current_dir(&exe);
if rls_installer.is_some() {
if built_tools.contains("rls") {
cmd.arg("RlsGroup.wixobj");
}
if rust_analyzer_installer.is_some() {
if built_tools.contains("rust-analyzer") {
cmd.arg("RustAnalyzerGroup.wixobj");
}
if rust_demangler_installer.is_some() {
if built_tools.contains("rust-demangler") {
cmd.arg("RustDemanglerGroup.wixobj");
}
if miri_installer.is_some() {
if built_tools.contains("miri") {
cmd.arg("MiriGroup.wixobj");
}
@ -2003,9 +1991,11 @@ pub struct LlvmTools {
impl Step for LlvmTools {
type Output = Option<GeneratedTarball>;
const ONLY_HOSTS: bool = true;
const DEFAULT: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("llvm-tools")
let default = should_build_extended_tool(&run.builder, "llvm-tools");
run.path("llvm-tools").default_condition(default)
}
fn make_run(run: RunConfig<'_>) {
@ -2014,7 +2004,6 @@ impl Step for LlvmTools {
fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let target = self.target;
assert!(builder.config.extended);
/* run only if llvm-config isn't used */
if let Some(config) = builder.config.target_config.get(&target) {

16
src/bootstrap/install.rs
View file

@ -139,12 +139,8 @@ macro_rules! install {
install!((self, builder, _config),
Docs, "src/doc", _config.docs, only_hosts: false, {
if let Some(tarball) = builder.ensure(dist::Docs { host: self.target }) {
install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball);
} else {
panic!("docs are not available to install, \
check that `build.docs` is true in `config.toml`");
}
let tarball = builder.ensure(dist::Docs { host: self.target }).expect("missing docs");
install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball);
};
Std, "library/std", true, only_hosts: false, {
for target in &builder.targets {
@ -158,7 +154,9 @@ install!((self, builder, _config),
}
};
Cargo, "cargo", Self::should_build(_config), only_hosts: true, {
let tarball = builder.ensure(dist::Cargo { compiler: self.compiler, target: self.target });
let tarball = builder
.ensure(dist::Cargo { compiler: self.compiler, target: self.target })
.expect("missing cargo");
install_sh(builder, "cargo", self.compiler.stage, Some(self.target), &tarball);
};
Rls, "rls", Self::should_build(_config), only_hosts: true, {
@ -182,7 +180,9 @@ install!((self, builder, _config),
}
};
Clippy, "clippy", Self::should_build(_config), only_hosts: true, {
let tarball = builder.ensure(dist::Clippy { compiler: self.compiler, target: self.target });
let tarball = builder
.ensure(dist::Clippy { compiler: self.compiler, target: self.target })
.expect("missing clippy");
install_sh(builder, "clippy", self.compiler.stage, Some(self.target), &tarball);
};
Miri, "miri", Self::should_build(_config), only_hosts: true, {