Auto merge of #64823 - cuviper:min-std, r=Mark-Simulacrum

minimize the rust-std component

This changes the `rust-std` dist component to only include the artifacts of compiling the `libstd` step, as listed in `.libstd.stamp`. This does include `test` and `proc-macro` as well. The remaining _unstable_ libraries that are built as part of `rustc` are packaged into a new `rustc-dev` component, intended for use in the development of closely related tools (clippy, miri, rls).

Here are the component sizes from the [try build](https://dev-static.rust-lang.org/dist/2019-10-07/index.html):

| Name | Size
| --- | ---
| rust-std-nightly-x86_64-unknown-linux-gnu.tar.gz | 23.94 MiB
| rust-std-nightly-x86_64-unknown-linux-gnu.tar.xz | 17.4 MiB
| rustc-dev-nightly-x86_64-unknown-linux-gnu.tar.gz | 182.03 MiB
| rustc-dev-nightly-x86_64-unknown-linux-gnu.tar.xz | 157.91 MiB

Fixes #61978
Fixes #62486

src/bootstrap/builder.rs

@@ -443,6 +443,7 @@ impl<'a> Builder<'a> {
                 dist::Rustc,
                 dist::DebuggerScripts,
                 dist::Std,
+                dist::RustcDev,
                 dist::Analysis,
                 dist::Src,
                 dist::PlainSourceTarball,

src/bootstrap/dist.rs

@@ -637,6 +637,28 @@ impl Step for DebuggerScripts {
     }
 }
 
+fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
+    // The only true set of target libraries came from the build triple, so
+    // let's reduce redundant work by only producing archives from that host.
+    if compiler.host != builder.config.build {
+        builder.info("\tskipping, not a build host");
+        true
+    } else {
+        false
+    }
+}
+
+/// Copy stamped files into an image's `target/lib` directory.
+fn copy_target_libs(builder: &Builder<'_>, target: &str, image: &Path, stamp: &Path) {
+    let dst = image.join("lib/rustlib").join(target).join("lib");
+    t!(fs::create_dir_all(&dst));
+    for (path, host) in builder.read_stamp_file(stamp) {
+        if !host || builder.config.build == target {
+            builder.copy(&path, &dst.join(path.file_name().unwrap()));
+        }
+    }
+}
+
 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Std {
     pub compiler: Compiler,
@@ -667,44 +689,19 @@ impl Step for Std {
         let target = self.target;
 
         let name = pkgname(builder, "rust-std");
-
+        let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));
-        // The only true set of target libraries came from the build triple, so
+        if skip_host_target_lib(builder, compiler) {
-        // let's reduce redundant work by only producing archives from that host.
+            return archive;
-        if compiler.host != builder.config.build {
-            builder.info("\tskipping, not a build host");
-            return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
         }
 
-        // We want to package up as many target libraries as possible
+        builder.ensure(compile::Std { compiler, target });
-        // for the `rust-std` package, so if this is a host target we
-        // depend on librustc and otherwise we just depend on libtest.
-        if builder.hosts.iter().any(|t| t == target) {
-            builder.ensure(compile::Rustc { compiler, target });
-        } else {
-            builder.ensure(compile::Std { compiler, target });
-        }
 
         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
         let _ = fs::remove_dir_all(&image);
 
-        let dst = image.join("lib/rustlib").join(target);
+        let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
-        t!(fs::create_dir_all(&dst));
+        let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
-        let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
+        copy_target_libs(builder, &target, &image, &stamp);
-        src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
-        builder.cp_filtered(&src, &dst, &|path| {
-            if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
-                if name == builder.config.rust_codegen_backends_dir.as_str() {
-                    return false
-                }
-                if name == "bin" {
-                    return false
-                }
-                if name.contains("LLVM") {
-                    return false
-                }
-            }
-            true
-        });
 
         let mut cmd = rust_installer(builder);
         cmd.arg("generate")
@@ -723,7 +720,73 @@ impl Step for Std {
         let _time = timeit(builder);
         builder.run(&mut cmd);
         builder.remove_dir(&image);
-        distdir(builder).join(format!("{}-{}.tar.gz", name, target))
+        archive
+    }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustcDev {
+    pub compiler: Compiler,
+    pub target: Interned<String>,
+}
+
+impl Step for RustcDev {
+    type Output = PathBuf;
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("rustc-dev")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(RustcDev {
+            compiler: run.builder.compiler_for(
+                run.builder.top_stage,
+                run.builder.config.build,
+                run.target,
+            ),
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> PathBuf {
+        let compiler = self.compiler;
+        let target = self.target;
+
+        let name = pkgname(builder, "rustc-dev");
+        let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));
+        if skip_host_target_lib(builder, compiler) {
+            return archive;
+        }
+
+        builder.ensure(compile::Rustc { compiler, target });
+
+        let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
+        let _ = fs::remove_dir_all(&image);
+
+        let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
+        let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
+        copy_target_libs(builder, &target, &image, &stamp);
+
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+           .arg("--product-name=Rust")
+           .arg("--rel-manifest-dir=rustlib")
+           .arg("--success-message=Rust-is-ready-to-develop.")
+           .arg("--image-dir").arg(&image)
+           .arg("--work-dir").arg(&tmpdir(builder))
+           .arg("--output-dir").arg(&distdir(builder))
+           .arg(format!("--package-name={}-{}", name, target))
+           .arg(format!("--component-name=rustc-dev-{}", target))
+           .arg("--legacy-manifest-dirs=rustlib,cargo");
+
+        builder.info(&format!("Dist rustc-dev stage{} ({} -> {})",
+            compiler.stage, &compiler.host, target));
+        let _time = timeit(builder);
+        builder.run(&mut cmd);
+        builder.remove_dir(&image);
+        archive
     }
 }
 

src/tools/build-manifest/src/main.rs

@@ -399,6 +399,7 @@ impl Builder {
     fn add_packages_to(&mut self, manifest: &mut Manifest) {
         let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets);
         package("rustc", HOSTS);
+        package("rustc-dev", HOSTS);
         package("cargo", HOSTS);
         package("rust-mingw", MINGW);
         package("rust-std", TARGETS);
@@ -426,6 +427,13 @@ impl Builder {
             "rls-preview", "rust-src", "llvm-tools-preview",
             "lldb-preview", "rust-analysis", "miri-preview"
         ]);
+
+        // The compiler libraries are not stable for end users, but `rustc-dev` was only recently
+        // split out of `rust-std`. We'll include it by default as a transition for nightly users.
+        if self.rust_release == "nightly" {
+            self.extend_profile("default", &mut manifest.profiles, &["rustc-dev"]);
+            self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
+        }
     }
 
     fn add_renames_to(&self, manifest: &mut Manifest) {
@@ -481,6 +489,15 @@ impl Builder {
             components.push(host_component("rust-mingw"));
         }
 
+        // The compiler libraries are not stable for end users, but `rustc-dev` was only recently
+        // split out of `rust-std`. We'll include it by default as a transition for nightly users,
+        // but ship it as an optional component on the beta and stable channels.
+        if self.rust_release == "nightly" {
+            components.push(host_component("rustc-dev"));
+        } else {
+            extensions.push(host_component("rustc-dev"));
+        }
+
         // Tools are always present in the manifest,
         // but might be marked as unavailable if they weren't built.
         extensions.extend(vec![
@@ -498,6 +515,11 @@ impl Builder {
                 .filter(|&&target| target != host)
                 .map(|target| Component::from_str("rust-std", target))
         );
+        extensions.extend(
+            HOSTS.iter()
+                .filter(|&&target| target != host)
+                .map(|target| Component::from_str("rustc-dev", target))
+        );
         extensions.push(Component::from_str("rust-src", "*"));
 
         // If the components/extensions don't actually exist for this
@@ -534,6 +556,14 @@ impl Builder {
         dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
     }
 
+    fn extend_profile(&mut self,
+               profile_name: &str,
+               dst: &mut BTreeMap<String, Vec<String>>,
+               pkgs: &[&str]) {
+        dst.get_mut(profile_name).expect("existing profile")
+            .extend(pkgs.iter().map(|s| (*s).to_owned()));
+    }
+
     fn package(&mut self,
                pkgname: &str,
                dst: &mut BTreeMap<String, Package>,