From 150d644c21e39fd5ae8020ac9df625d2134fe734 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Perennou Date: Thu, 18 May 2017 22:48:14 +0200 Subject: [PATCH] rustbuild: split Install out of Dist subcommand only create source tarball for the Dist subcommand mark install rule as default for Kind::Install split install-docs split install-std factor out empty_dir handling split install-cargo split install-analysis split install-src rework install-rustc properly handle cross-compilation setups for install use pkgname in install split plain source tarball generation from rust-src dist document src-tarball in config.toml.exmaple Signed-off-by: Marc-Antoine Perennou --- README.md | 6 +- src/bootstrap/config.toml.example | 6 + src/bootstrap/dist.rs | 182 +++++++++++++++--------------- src/bootstrap/flags.rs | 17 ++- src/bootstrap/install.rs | 74 ++++++------ src/bootstrap/mk/Makefile.in | 2 +- src/bootstrap/step.rs | 68 +++++++++-- 7 files changed, 212 insertions(+), 143 deletions(-) diff --git a/README.md b/README.md index f387b4be600..dbb5bf9ce38 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Read ["Installing Rust"] from [The Book]. 3. Build and install: ```sh - $ ./x.py build && sudo ./x.py dist --install + $ ./x.py build && sudo ./x.py install ``` > ***Note:*** Install locations can be adjusted by copying the config file @@ -43,7 +43,7 @@ Read ["Installing Rust"] from [The Book]. > adjusting the `prefix` option under `[install]`. Various other options are > also supported, and are documented in the config file. - When complete, `sudo ./x.py dist --install` will place several programs into + When complete, `sudo ./x.py install` will place several programs into `/usr/local/bin`: `rustc`, the Rust compiler, and `rustdoc`, the API-documentation tool. This install does not include [Cargo], Rust's package manager, which you may also want to build. @@ -96,7 +96,7 @@ build. 4. Navigate to Rust's source code (or clone it), then build it: ```sh - $ ./x.py build && ./x.py dist --install + $ ./x.py build && ./x.py install ``` #### MSVC diff --git a/src/bootstrap/config.toml.example b/src/bootstrap/config.toml.example index 95cca96f7fc..0eb6c4c82c4 100644 --- a/src/bootstrap/config.toml.example +++ b/src/bootstrap/config.toml.example @@ -314,3 +314,9 @@ # Note that this address should not contain a trailing slash as file names will # be appended to it. #upload-addr = "https://example.com/folder" + +# Whether to build a plain source tarball to upload +# We disable that on Windows not to override the one already uploaded on S3 +# as the one built on Windows will contain backslashes in paths causing problems +# on linux +#src-tarball = true diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 511f2c9e80e..94189d7eb92 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -30,11 +30,11 @@ use {Build, Compiler, Mode}; use channel; use util::{cp_r, libdir, is_dylib, cp_filtered, copy, exe}; -fn pkgname(build: &Build, component: &str) -> String { +pub fn pkgname(build: &Build, component: &str) -> String { if component == "cargo" { format!("{}-{}", component, build.cargo_package_vers()) } else if component == "rls" { - format!("{}-{}", component, build.package_vers(&build.release_num("rls"))) + format!("{}-{}", component, build.rls_package_vers()) } else { assert!(component.starts_with("rust")); format!("{}-{}", component, build.rust_package_vers()) @@ -369,38 +369,7 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) { t!(fs::remove_dir_all(&image)); } -const CARGO_VENDOR_VERSION: &'static str = "0.1.4"; - -/// Creates the `rust-src` installer component and the plain source tarball -pub fn rust_src(build: &Build) { - if !build.config.rust_dist_src { - return - } - - println!("Dist src"); - - // Make sure that the root folder of tarball has the correct name - let plain_name = format!("rustc-{}-src", build.rust_package_vers()); - let plain_dst_src = tmpdir(build).join(&plain_name); - let _ = fs::remove_dir_all(&plain_dst_src); - t!(fs::create_dir_all(&plain_dst_src)); - - // This is the set of root paths which will become part of the source package - let src_files = [ - "COPYRIGHT", - "LICENSE-APACHE", - "LICENSE-MIT", - "CONTRIBUTING.md", - "README.md", - "RELEASES.md", - "configure", - "x.py", - ]; - let src_dirs = [ - "man", - "src", - ]; - +fn copy_src_dirs(build: &Build, src_dirs: &[&str], dst_dir: &Path) { let filter_fn = move |path: &Path| { let spath = match path.to_str() { Some(path) => path, @@ -429,60 +398,16 @@ pub fn rust_src(build: &Build) { }; // Copy the directories using our filter - for item in &src_dirs { - let dst = &plain_dst_src.join(item); - t!(fs::create_dir(dst)); + for item in src_dirs { + let dst = &dst_dir.join(item); + t!(fs::create_dir_all(dst)); cp_filtered(&build.src.join(item), dst, &filter_fn); } - // Copy the files normally - for item in &src_files { - copy(&build.src.join(item), &plain_dst_src.join(item)); - } - - // If we're building from git sources, we need to vendor a complete distribution. - if build.src_is_git { - // Get cargo-vendor installed, if it isn't already. - let mut has_cargo_vendor = false; - let mut cmd = Command::new(&build.cargo); - for line in output(cmd.arg("install").arg("--list")).lines() { - has_cargo_vendor |= line.starts_with("cargo-vendor "); - } - if !has_cargo_vendor { - let mut cmd = Command::new(&build.cargo); - cmd.arg("install") - .arg("--force") - .arg("--debug") - .arg("--vers").arg(CARGO_VENDOR_VERSION) - .arg("cargo-vendor") - .env("RUSTC", &build.rustc); - build.run(&mut cmd); - } - - // Vendor all Cargo dependencies - let mut cmd = Command::new(&build.cargo); - cmd.arg("vendor") - .current_dir(&plain_dst_src.join("src")); - build.run(&mut cmd); - } - - // Create the version file - write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes()); - - // Create plain source tarball - let mut tarball = rust_src_location(build); - tarball.set_extension(""); // strip .gz - tarball.set_extension(""); // strip .tar - if let Some(dir) = tarball.parent() { - t!(fs::create_dir_all(dir)); - } - let mut cmd = rust_installer(build); - cmd.arg("tarball") - .arg("--input").arg(&plain_name) - .arg("--output").arg(&tarball) - .arg("--work-dir=.") - .current_dir(tmpdir(build)); - build.run(&mut cmd); +} +/// Creates the `rust-src` installer component +pub fn rust_src(build: &Build) { + println!("Dist src"); let name = pkgname(build, "rust-src"); let image = tmpdir(build).join(format!("{}-image", name)); @@ -516,11 +441,7 @@ pub fn rust_src(build: &Build) { "src/rustc/libc_shim", ]; - for item in &std_src_dirs { - let dst = &dst_src.join(item); - t!(fs::create_dir_all(dst)); - cp_r(&plain_dst_src.join(item), dst); - } + copy_src_dirs(build, &std_src_dirs[..], &dst_src); // Create source tarball in rust-installer format let mut cmd = rust_installer(build); @@ -537,7 +458,86 @@ pub fn rust_src(build: &Build) { build.run(&mut cmd); t!(fs::remove_dir_all(&image)); - t!(fs::remove_dir_all(&plain_dst_src)); +} + +const CARGO_VENDOR_VERSION: &'static str = "0.1.4"; + +/// Creates the plain source tarball +pub fn plain_source_tarball(build: &Build) { + println!("Create plain source tarball"); + + // Make sure that the root folder of tarball has the correct name + let plain_name = format!("{}-src", pkgname(build, "rustc")); + let plain_dst_src = tmpdir(build).join(&plain_name); + let _ = fs::remove_dir_all(&plain_dst_src); + t!(fs::create_dir_all(&plain_dst_src)); + + // This is the set of root paths which will become part of the source package + let src_files = [ + "COPYRIGHT", + "LICENSE-APACHE", + "LICENSE-MIT", + "CONTRIBUTING.md", + "README.md", + "RELEASES.md", + "configure", + "x.py", + ]; + let src_dirs = [ + "man", + "src", + ]; + + copy_src_dirs(build, &src_dirs[..], &plain_dst_src); + + // Copy the files normally + for item in &src_files { + copy(&build.src.join(item), &plain_dst_src.join(item)); + } + + // Create the version file + write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes()); + + // If we're building from git sources, we need to vendor a complete distribution. + if build.src_is_git { + // Get cargo-vendor installed, if it isn't already. + let mut has_cargo_vendor = false; + let mut cmd = Command::new(&build.cargo); + for line in output(cmd.arg("install").arg("--list")).lines() { + has_cargo_vendor |= line.starts_with("cargo-vendor "); + } + if !has_cargo_vendor { + let mut cmd = Command::new(&build.cargo); + cmd.arg("install") + .arg("--force") + .arg("--debug") + .arg("--vers").arg(CARGO_VENDOR_VERSION) + .arg("cargo-vendor") + .env("RUSTC", &build.rustc); + build.run(&mut cmd); + } + + // Vendor all Cargo dependencies + let mut cmd = Command::new(&build.cargo); + cmd.arg("vendor") + .current_dir(&plain_dst_src.join("src")); + build.run(&mut cmd); + } + + // Create plain source tarball + let mut tarball = rust_src_location(build); + tarball.set_extension(""); // strip .gz + tarball.set_extension(""); // strip .tar + if let Some(dir) = tarball.parent() { + t!(fs::create_dir_all(dir)); + } + let mut cmd = rust_installer(build); + cmd.arg("tarball") + .arg("--input").arg(&plain_name) + .arg("--output").arg(&tarball) + .arg("--work-dir=.") + .current_dir(tmpdir(build)); + build.run(&mut cmd); } fn install(src: &Path, dstdir: &Path, perms: u32) { diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index a1466d68a13..fe4e18ab622 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -69,7 +69,9 @@ pub enum Subcommand { Clean, Dist { paths: Vec, - install: bool, + }, + Install { + paths: Vec, }, } @@ -85,7 +87,8 @@ Subcommands: bench Build and run some benchmarks doc Build documentation clean Clean out build directories - dist Build and/or install distribution artifacts + dist Build distribution artifacts + install Install distribution artifacts To learn more about a subcommand, run `./x.py -h`"); @@ -125,7 +128,8 @@ To learn more about a subcommand, run `./x.py -h`"); || (s == "bench") || (s == "doc") || (s == "clean") - || (s == "dist")); + || (s == "dist") + || (s == "install")); let subcommand = match possible_subcommands.first() { Some(s) => s, None => { @@ -139,7 +143,6 @@ To learn more about a subcommand, run `./x.py -h`"); match subcommand.as_str() { "test" => { opts.optmulti("", "test-args", "extra arguments", "ARGS"); }, "bench" => { opts.optmulti("", "test-args", "extra arguments", "ARGS"); }, - "dist" => { opts.optflag("", "install", "run installer as well"); }, _ => { }, }; @@ -281,7 +284,11 @@ Arguments: "dist" => { Subcommand::Dist { paths: paths, - install: matches.opt_present("install"), + } + } + "install" => { + Subcommand::Install { + paths: paths, } } _ => { diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs index dce0b1670e1..21e21628dc9 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/install.rs @@ -19,7 +19,7 @@ use std::path::{Path, PathBuf, Component}; use std::process::Command; use Build; -use dist::{sanitize_sh, tmpdir}; +use dist::{pkgname, sanitize_sh, tmpdir}; pub struct Installer<'a> { build: &'a Build, @@ -29,6 +29,13 @@ pub struct Installer<'a> { bindir: PathBuf, libdir: PathBuf, mandir: PathBuf, + empty_dir: PathBuf, +} + +impl<'a> Drop for Installer<'a> { + fn drop(&mut self) { + t!(fs::remove_dir_all(&self.empty_dir)); + } } impl<'a> Installer<'a> { @@ -61,6 +68,10 @@ impl<'a> Installer<'a> { let libdir = add_destdir(&libdir, &destdir); let mandir = add_destdir(&mandir, &destdir); + let empty_dir = build.out.join("tmp/empty_dir"); + + t!(fs::create_dir_all(&empty_dir)); + Installer { build, prefix, @@ -69,52 +80,49 @@ impl<'a> Installer<'a> { bindir, libdir, mandir, + empty_dir, } } - /// Installs everything. - pub fn install(&self, stage: u32, host: &str) { - let empty_dir = self.build.out.join("tmp/empty_dir"); - t!(fs::create_dir_all(&empty_dir)); - - if self.build.config.docs { - self.install_sh("docs", "rust-docs", &self.build.rust_package_vers(), - stage, Some(host), &empty_dir); - } + pub fn install_docs(&self, stage: u32, host: &str) { + self.install_sh("docs", "rust-docs", stage, Some(host)); + } + pub fn install_std(&self, stage: u32) { for target in self.build.config.target.iter() { - self.install_sh("std", "rust-std", &self.build.rust_package_vers(), - stage, Some(target), &empty_dir); + self.install_sh("std", "rust-std", stage, Some(target)); } - - if self.build.config.extended { - self.install_sh("cargo", "cargo", &self.build.cargo_package_vers(), - stage, Some(host), &empty_dir); - self.install_sh("rls", "rls", &self.build.rls_package_vers(), - stage, Some(host), &empty_dir); - self.install_sh("analysis", "rust-analysis", &self.build.rust_package_vers(), - stage, Some(host), &empty_dir); - self.install_sh("src", "rust-src", &self.build.rust_package_vers(), - stage, None, &empty_dir); - } - - self.install_sh("rustc", "rustc", &self.build.rust_package_vers(), - stage, Some(host), &empty_dir); - - t!(fs::remove_dir_all(&empty_dir)); } - fn install_sh(&self, package: &str, name: &str, version: &str, - stage: u32, host: Option<&str>, empty_dir: &Path) { + pub fn install_cargo(&self, stage: u32, host: &str) { + self.install_sh("cargo", "cargo", stage, Some(host)); + } + + pub fn install_rls(&self, stage: u32, host: &str) { + self.install_sh("rls", "rls", stage, Some(host)); + } + + pub fn install_analysis(&self, stage: u32, host: &str) { + self.install_sh("analysis", "rust-analysis", stage, Some(host)); + } + + pub fn install_src(&self, stage: u32) { + self.install_sh("src", "rust-src", stage, None); + } + pub fn install_rustc(&self, stage: u32, host: &str) { + self.install_sh("rustc", "rustc", stage, Some(host)); + } + + fn install_sh(&self, package: &str, name: &str, stage: u32, host: Option<&str>) { println!("Install {} stage{} ({:?})", package, stage, host); let package_name = if let Some(host) = host { - format!("{}-{}-{}", name, version, host) + format!("{}-{}", pkgname(self.build, name), host) } else { - format!("{}-{}", name, version) + pkgname(self.build, name) }; let mut cmd = Command::new("sh"); - cmd.current_dir(empty_dir) + cmd.current_dir(&self.empty_dir) .arg(sanitize_sh(&tmpdir(self.build).join(&package_name).join("install.sh"))) .arg(format!("--prefix={}", sanitize_sh(&self.prefix))) .arg(format!("--sysconfdir={}", sanitize_sh(&self.sysconfdir))) diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index a5df741e2bf..47c792a510b 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -69,7 +69,7 @@ distcheck: $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) $(Q)$(BOOTSTRAP) test distcheck $(BOOTSTRAP_ARGS) install: - $(Q)$(BOOTSTRAP) dist --install $(BOOTSTRAP_ARGS) + $(Q)$(BOOTSTRAP) install $(BOOTSTRAP_ARGS) tidy: $(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS) prepare: diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index 57915446e1d..db7a9aed91a 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -734,6 +734,13 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { dist::mingw(build, s.target) } }); + rules.dist("dist-plain-source-tarball", "src") + .default(build.config.rust_dist_src) + .host(true) + .only_build(true) + .only_host_build(true) + .dep(move |s| tool_rust_installer(build, s)) + .run(move |_| dist::plain_source_tarball(build)); rules.dist("dist-src", "src") .default(true) .host(true) @@ -759,9 +766,6 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { .dep(|s| s.name("tool-rls")) .dep(move |s| tool_rust_installer(build, s)) .run(move |s| dist::rls(build, s.stage, s.target)); - rules.dist("install", "path/to/nowhere") - .dep(|s| s.name("default:dist")) - .run(move |s| install::Installer::new(build).install(s.stage, s.target)); rules.dist("dist-cargo", "cargo") .host(true) .only_host_build(true) @@ -789,6 +793,47 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { .dep(move |s| s.name("tool-build-manifest").target(&build.config.build).stage(0)) .run(move |_| dist::hash_and_sign(build)); + rules.install("install-docs", "src/doc") + .default(build.config.docs) + .only_host_build(true) + .dep(|s| s.name("dist-docs")) + .run(move |s| install::Installer::new(build).install_docs(s.stage, s.target)); + rules.install("install-std", "src/libstd") + .default(true) + .only_host_build(true) + .dep(|s| s.name("dist-std")) + .run(move |s| install::Installer::new(build).install_std(s.stage)); + rules.install("install-cargo", "cargo") + .default(build.config.extended) + .host(true) + .only_host_build(true) + .dep(|s| s.name("dist-cargo")) + .run(move |s| install::Installer::new(build).install_cargo(s.stage, s.target)); + rules.install("install-rls", "rls") + .default(build.config.extended) + .host(true) + .only_host_build(true) + .dep(|s| s.name("dist-rls")) + .run(move |s| install::Installer::new(build).install_rls(s.stage, s.target)); + rules.install("install-analysis", "analysis") + .default(build.config.extended) + .only_host_build(true) + .dep(|s| s.name("dist-analysis")) + .run(move |s| install::Installer::new(build).install_analysis(s.stage, s.target)); + rules.install("install-src", "src") + .default(build.config.extended) + .host(true) + .only_build(true) + .only_host_build(true) + .dep(|s| s.name("dist-src")) + .run(move |s| install::Installer::new(build).install_src(s.stage)); + rules.install("install-rustc", "src/librustc") + .default(true) + .host(true) + .only_host_build(true) + .dep(|s| s.name("dist-rustc")) + .run(move |s| install::Installer::new(build).install_rustc(s.stage, s.target)); + rules.verify(); return rules; @@ -902,6 +947,7 @@ enum Kind { Bench, Dist, Doc, + Install, } impl<'a> Rule<'a> { @@ -1033,6 +1079,12 @@ impl<'a> Rules<'a> { self.rule(name, path, Kind::Dist) } + /// Same as `build`, but for `Kind::Install`. + fn install<'b>(&'b mut self, name: &'a str, path: &'a str) + -> RuleBuilder<'a, 'b> { + self.rule(name, path, Kind::Install) + } + fn rule<'b>(&'b mut self, name: &'a str, path: &'a str, @@ -1073,6 +1125,7 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? "test" => Kind::Test, "bench" => Kind::Bench, "dist" => Kind::Dist, + "install" => Kind::Install, _ => return None, }; let rules = self.rules.values().filter(|r| r.kind == kind); @@ -1122,13 +1175,8 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]), Subcommand::Test { ref paths, test_args: _ } => (Kind::Test, &paths[..]), Subcommand::Bench { ref paths, test_args: _ } => (Kind::Bench, &paths[..]), - Subcommand::Dist { ref paths, install } => { - if install { - return vec![self.sbuild.name("install")] - } else { - (Kind::Dist, &paths[..]) - } - } + Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]), + Subcommand::Install { ref paths } => (Kind::Install, &paths[..]), Subcommand::Clean => panic!(), };