Auto merge of #65474 - Mark-Simulacrum:rustc-dev-split, r=pietroalbini

Split the rustc target libraries into separate rustc-dev component

This is re-applies a squashed version of #64823 as well as including #65337 to fix bugs noted after merging the first PR.

The second PR is confirmed as fixing windows-gnu, and presumably also fixes other platforms, such as musl (i.e. #65335 should be fixed); `RUSTUP_DIST_SERVER=https://dev-static.rust-lang.org rustup toolchain install nightly-2019-10-16` can be installed to confirm that this is indeed the case.

src/bootstrap/builder.rs

@@ -443,6 +443,7 @@ impl<'a> Builder<'a> {
                 dist::Rustc,
                 dist::DebuggerScripts,
                 dist::Std,
+                dist::RustcDev,
                 dist::Analysis,
                 dist::Src,
                 dist::PlainSourceTarball,

src/bootstrap/check.rs

@@ -55,6 +55,7 @@ impl Step for Std {
                   cargo,
                   args(builder.kind),
                   &libstd_stamp(builder, compiler, target),
+                  vec![],
                   true);
 
         let libdir = builder.sysroot_libdir(compiler, target);
@@ -103,6 +104,7 @@ impl Step for Rustc {
                   cargo,
                   args(builder.kind),
                   &librustc_stamp(builder, compiler, target),
+                  vec![],
                   true);
 
         let libdir = builder.sysroot_libdir(compiler, target);
@@ -155,6 +157,7 @@ impl Step for CodegenBackend {
                   cargo,
                   args(builder.kind),
                   &codegen_backend_stamp(builder, compiler, target, backend),
+                  vec![],
                   true);
     }
 }
@@ -199,6 +202,7 @@ impl Step for Rustdoc {
                   cargo,
                   args(builder.kind),
                   &rustdoc_stamp(builder, compiler, target),
+                  vec![],
                   true);
 
         let libdir = builder.sysroot_libdir(compiler, target);

src/bootstrap/compile.rs

@@ -69,7 +69,7 @@ impl Step for Std {
             return;
         }
 
-        builder.ensure(StartupObjects { compiler, target });
+        let mut target_deps = builder.ensure(StartupObjects { compiler, target });
 
         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
         if compiler_to_use != compiler {
@@ -91,7 +91,7 @@ impl Step for Std {
             return;
         }
 
-        copy_third_party_objects(builder, &compiler, target);
+        target_deps.extend(copy_third_party_objects(builder, &compiler, target).into_iter());
 
         let mut cargo = builder.cargo(compiler, Mode::Std, target, "build");
         std_cargo(builder, &compiler, target, &mut cargo);
@@ -102,6 +102,7 @@ impl Step for Std {
                   cargo,
                   vec![],
                   &libstd_stamp(builder, compiler, target),
+                  target_deps,
                   false);
 
         builder.ensure(StdLink {
@@ -113,9 +114,22 @@ impl Step for Std {
 }
 
 /// Copies third pary objects needed by various targets.
-fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target: Interned<String>) {
+fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target: Interned<String>)
+    -> Vec<PathBuf>
+{
     let libdir = builder.sysroot_libdir(*compiler, target);
 
+    let mut target_deps = vec![];
+
+    let mut copy_and_stamp = |sourcedir: &Path, name: &str| {
+        let target = libdir.join(name);
+        builder.copy(
+            &sourcedir.join(name),
+            &target,
+        );
+        target_deps.push(target);
+    };
+
     // Copies the crt(1,i,n).o startup objects
     //
     // Since musl supports fully static linking, we can cross link for it even
@@ -123,19 +137,13 @@ fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target:
     // files. As those shipped with glibc won't work, copy the ones provided by
     // musl so we have them on linux-gnu hosts.
     if target.contains("musl") {
+        let srcdir = builder.musl_root(target).unwrap().join("lib");
         for &obj in &["crt1.o", "crti.o", "crtn.o"] {
-            builder.copy(
+            copy_and_stamp(&srcdir, obj);
-                &builder.musl_root(target).unwrap().join("lib").join(obj),
-                &libdir.join(obj),
-            );
         }
     } else if target.ends_with("-wasi") {
-        for &obj in &["crt1.o"] {
+        let srcdir = builder.wasi_root(target).unwrap().join("lib/wasm32-wasi");
-            builder.copy(
+        copy_and_stamp(&srcdir, "crt1.o");
-                &builder.wasi_root(target).unwrap().join("lib/wasm32-wasi").join(obj),
-                &libdir.join(obj),
-            );
-        }
     }
 
     // Copies libunwind.a compiled to be linked wit x86_64-fortanix-unknown-sgx.
@@ -145,11 +153,11 @@ fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target:
     // which is provided by std for this target.
     if target == "x86_64-fortanix-unknown-sgx" {
         let src_path_env = "X86_FORTANIX_SGX_LIBS";
-        let obj = "libunwind.a";
         let src = env::var(src_path_env).expect(&format!("{} not found in env", src_path_env));
-        let src = Path::new(&src).join(obj);
+        copy_and_stamp(Path::new(&src), "libunwind.a");
-        builder.copy(&src, &libdir.join(obj));
     }
+
+    target_deps
 }
 
 /// Configure cargo to compile the standard library, adding appropriate env vars
@@ -306,7 +314,7 @@ pub struct StartupObjects {
 }
 
 impl Step for StartupObjects {
-    type Output = ();
+    type Output = Vec<PathBuf>;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
         run.path("src/rtstartup")
@@ -325,13 +333,15 @@ impl Step for StartupObjects {
     /// They don't require any library support as they're just plain old object
     /// files, so we just use the nightly snapshot compiler to always build them (as
     /// no other compilers are guaranteed to be available).
-    fn run(self, builder: &Builder<'_>) {
+    fn run(self, builder: &Builder<'_>) -> Vec<PathBuf> {
         let for_compiler = self.compiler;
         let target = self.target;
         if !target.contains("windows-gnu") {
-            return
+            return vec![]
         }
 
+        let mut target_deps = vec![];
+
         let src_dir = &builder.src.join("src/rtstartup");
         let dst_dir = &builder.native_dir(target).join("rtstartup");
         let sysroot_dir = &builder.sysroot_libdir(for_compiler, target);
@@ -350,7 +360,9 @@ impl Step for StartupObjects {
                             .arg(src_file));
             }
 
-            builder.copy(dst_file, &sysroot_dir.join(file.to_string() + ".o"));
+            let target = sysroot_dir.join(file.to_string() + ".o");
+            builder.copy(dst_file, &target);
+            target_deps.push(target);
         }
 
         for obj in ["crt2.o", "dllcrt2.o"].iter() {
@@ -358,8 +370,12 @@ impl Step for StartupObjects {
                                     builder.cc(target),
                                     target,
                                     obj);
-            builder.copy(&src, &sysroot_dir.join(obj));
+            let target = sysroot_dir.join(obj);
+            builder.copy(&src, &target);
+            target_deps.push(target);
         }
+
+        target_deps
     }
 }
 
@@ -437,6 +453,7 @@ impl Step for Rustc {
                   cargo,
                   vec![],
                   &librustc_stamp(builder, compiler, target),
+                  vec![],
                   false);
 
         builder.ensure(RustcLink {
@@ -585,7 +602,7 @@ impl Step for CodegenBackend {
 
         let tmp_stamp = out_dir.join(".tmp.stamp");
 
-        let files = run_cargo(builder, cargo, vec![], &tmp_stamp, false);
+        let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false);
         if builder.config.dry_run {
             return;
         }
@@ -941,6 +958,7 @@ pub fn run_cargo(builder: &Builder<'_>,
                  cargo: Cargo,
                  tail_args: Vec<String>,
                  stamp: &Path,
+                 additional_target_deps: Vec<PathBuf>,
                  is_check: bool)
     -> Vec<PathBuf>
 {
@@ -1057,6 +1075,7 @@ pub fn run_cargo(builder: &Builder<'_>,
         deps.push((path_to_add.into(), false));
     }
 
+    deps.extend(additional_target_deps.into_iter().map(|d| (d, false)));
     deps.sort();
     let mut new_contents = Vec::new();
     for (dep, proc_macro) in deps.iter() {

src/bootstrap/dist.rs

@@ -637,6 +637,28 @@ impl Step for DebuggerScripts {
     }
 }
 
+fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
+    // The only true set of target libraries came from the build triple, so
+    // let's reduce redundant work by only producing archives from that host.
+    if compiler.host != builder.config.build {
+        builder.info("\tskipping, not a build host");
+        true
+    } else {
+        false
+    }
+}
+
+/// Copy stamped files into an image's `target/lib` directory.
+fn copy_target_libs(builder: &Builder<'_>, target: &str, image: &Path, stamp: &Path) {
+    let dst = image.join("lib/rustlib").join(target).join("lib");
+    t!(fs::create_dir_all(&dst));
+    for (path, host) in builder.read_stamp_file(stamp) {
+        if !host || builder.config.build == target {
+            builder.copy(&path, &dst.join(path.file_name().unwrap()));
+        }
+    }
+}
+
 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Std {
     pub compiler: Compiler,
@@ -667,44 +689,19 @@ impl Step for Std {
         let target = self.target;
 
         let name = pkgname(builder, "rust-std");
-
+        let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));
-        // The only true set of target libraries came from the build triple, so
+        if skip_host_target_lib(builder, compiler) {
-        // let's reduce redundant work by only producing archives from that host.
+            return archive;
-        if compiler.host != builder.config.build {
-            builder.info("\tskipping, not a build host");
-            return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
         }
 
-        // We want to package up as many target libraries as possible
-        // for the `rust-std` package, so if this is a host target we
-        // depend on librustc and otherwise we just depend on libtest.
-        if builder.hosts.iter().any(|t| t == target) {
-            builder.ensure(compile::Rustc { compiler, target });
-        } else {
         builder.ensure(compile::Std { compiler, target });
-        }
 
         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
         let _ = fs::remove_dir_all(&image);
 
-        let dst = image.join("lib/rustlib").join(target);
+        let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
-        t!(fs::create_dir_all(&dst));
+        let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
-        let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
+        copy_target_libs(builder, &target, &image, &stamp);
-        src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
-        builder.cp_filtered(&src, &dst, &|path| {
-            if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
-                if name == builder.config.rust_codegen_backends_dir.as_str() {
-                    return false
-                }
-                if name == "bin" {
-                    return false
-                }
-                if name.contains("LLVM") {
-                    return false
-                }
-            }
-            true
-        });
 
         let mut cmd = rust_installer(builder);
         cmd.arg("generate")
@@ -723,7 +720,73 @@ impl Step for Std {
         let _time = timeit(builder);
         builder.run(&mut cmd);
         builder.remove_dir(&image);
-        distdir(builder).join(format!("{}-{}.tar.gz", name, target))
+        archive
+    }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustcDev {
+    pub compiler: Compiler,
+    pub target: Interned<String>,
+}
+
+impl Step for RustcDev {
+    type Output = PathBuf;
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("rustc-dev")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(RustcDev {
+            compiler: run.builder.compiler_for(
+                run.builder.top_stage,
+                run.builder.config.build,
+                run.target,
+            ),
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> PathBuf {
+        let compiler = self.compiler;
+        let target = self.target;
+
+        let name = pkgname(builder, "rustc-dev");
+        let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));
+        if skip_host_target_lib(builder, compiler) {
+            return archive;
+        }
+
+        builder.ensure(compile::Rustc { compiler, target });
+
+        let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
+        let _ = fs::remove_dir_all(&image);
+
+        let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
+        let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
+        copy_target_libs(builder, &target, &image, &stamp);
+
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+           .arg("--product-name=Rust")
+           .arg("--rel-manifest-dir=rustlib")
+           .arg("--success-message=Rust-is-ready-to-develop.")
+           .arg("--image-dir").arg(&image)
+           .arg("--work-dir").arg(&tmpdir(builder))
+           .arg("--output-dir").arg(&distdir(builder))
+           .arg(format!("--package-name={}-{}", name, target))
+           .arg(format!("--component-name=rustc-dev-{}", target))
+           .arg("--legacy-manifest-dirs=rustlib,cargo");
+
+        builder.info(&format!("Dist rustc-dev stage{} ({} -> {})",
+            compiler.stage, &compiler.host, target));
+        let _time = timeit(builder);
+        builder.run(&mut cmd);
+        builder.remove_dir(&image);
+        archive
     }
 }
 

src/bootstrap/lib.rs

@@ -1137,6 +1137,7 @@ impl Build {
     pub fn copy(&self, src: &Path, dst: &Path) {
         if self.config.dry_run { return; }
         self.verbose_than(1, &format!("Copy {:?} to {:?}", src, dst));
+        if src == dst { return; }
         let _ = fs::remove_file(&dst);
         let metadata = t!(src.symlink_metadata());
         if metadata.file_type().is_symlink() {

src/tools/build-manifest/src/main.rs

@@ -399,6 +399,7 @@ impl Builder {
     fn add_packages_to(&mut self, manifest: &mut Manifest) {
         let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets);
         package("rustc", HOSTS);
+        package("rustc-dev", HOSTS);
         package("cargo", HOSTS);
         package("rust-mingw", MINGW);
         package("rust-std", TARGETS);
@@ -426,6 +427,13 @@ impl Builder {
             "rls-preview", "rust-src", "llvm-tools-preview",
             "lldb-preview", "rust-analysis", "miri-preview"
         ]);
+
+        // The compiler libraries are not stable for end users, but `rustc-dev` was only recently
+        // split out of `rust-std`. We'll include it by default as a transition for nightly users.
+        if self.rust_release == "nightly" {
+            self.extend_profile("default", &mut manifest.profiles, &["rustc-dev"]);
+            self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
+        }
     }
 
     fn add_renames_to(&self, manifest: &mut Manifest) {
@@ -481,6 +489,15 @@ impl Builder {
             components.push(host_component("rust-mingw"));
         }
 
+        // The compiler libraries are not stable for end users, but `rustc-dev` was only recently
+        // split out of `rust-std`. We'll include it by default as a transition for nightly users,
+        // but ship it as an optional component on the beta and stable channels.
+        if self.rust_release == "nightly" {
+            components.push(host_component("rustc-dev"));
+        } else {
+            extensions.push(host_component("rustc-dev"));
+        }
+
         // Tools are always present in the manifest,
         // but might be marked as unavailable if they weren't built.
         extensions.extend(vec![
@@ -498,6 +515,11 @@ impl Builder {
                 .filter(|&&target| target != host)
                 .map(|target| Component::from_str("rust-std", target))
         );
+        extensions.extend(
+            HOSTS.iter()
+                .filter(|&&target| target != host)
+                .map(|target| Component::from_str("rustc-dev", target))
+        );
         extensions.push(Component::from_str("rust-src", "*"));
 
         // If the components/extensions don't actually exist for this
@@ -534,6 +556,14 @@ impl Builder {
         dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
     }
 
+    fn extend_profile(&mut self,
+               profile_name: &str,
+               dst: &mut BTreeMap<String, Vec<String>>,
+               pkgs: &[&str]) {
+        dst.get_mut(profile_name).expect("existing profile")
+            .extend(pkgs.iter().map(|s| (*s).to_owned()));
+    }
+
     fn package(&mut self,
                pkgname: &str,
                dst: &mut BTreeMap<String, Package>,