squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

Auto merge of #76349 - Mark-Simulacrum:dl-llvm, r=alexcrichton

Browse source 
Download LLVM from CI to bootstrap (linux-only to start)

This follows #76332, adding support for using CI-built LLVM rather than building it locally. This should essentially "just work," but is left off by default in this PR.

While we can support downloading LLVM for multiple host triples, this currently only downloads it for the build triple. That said, it should be possible to expand this relatively easily should multiple host triples be desired. Most people shouldn't be adjusting host/target triples though, so this should cover most use cases.

Currently this downloads LLVM for the last bors-authored commit in the `git log`. This is a bit suboptimal -- we want the last bors-authored commit that touched the llvm-project submodule in basically all cases. But for now this just adds an extra ~20 MB download when rebasing atop latest master. Once we have a submodule bump landing after #76332, we can fix this behavior to reduce downloads further.
This commit is contained in:
bors 2020-09-13 02:16:18 +00:00
parent 498dab0256 2e87a6e78d
commit 04b72b4697
6 changed files with 186 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
config.toml.example
View file

@ -14,6 +14,21 @@
# =============================================================================
[llvm]
# Whether to use Rust CI built LLVM instead of locally building it.
#
# Unless you're developing for a target where Rust CI doesn't build a compiler
# toolchain or changing LLVM locally, you probably want to set this to true.
#
# It's currently false by default due to being newly added; please file bugs if
# enabling this did not work for you on Linux (macOS and Windows support is
# coming soon).
#
# We also currently only support this when building LLVM for the build triple.
#
# Note that many of the LLVM options are not currently supported for
# downloading. Currently only the "assertions" option can be toggled.
#download-ci-llvm = false
# Indicates whether LLVM rebuild should be skipped when running bootstrap. If
# this is `false` then the compiler's LLVM will be rebuilt whenever the built
# version doesn't have the correct hash. If it is `true` then LLVM will never

105
src/bootstrap/bootstrap.py
View file

@ -14,8 +14,17 @@ import tempfile
from time import time
def support_xz():
try:
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
temp_path = temp_file.name
with tarfile.open(temp_path, "w:xz"):
pass
return True
except tarfile.CompressionError:
return False
def get(url, path, verbose=False):
def get(url, path, verbose=False, do_verify=True):
suffix = '.sha256'
sha_url = url + suffix
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
@ -24,19 +33,20 @@ def get(url, path, verbose=False):
sha_path = sha_file.name
try:
download(sha_path, sha_url, False, verbose)
if os.path.exists(path):
if verify(path, sha_path, False):
if verbose:
print("using already-download file", path)
return
else:
if verbose:
print("ignoring already-download file",
path, "due to failed verification")
os.unlink(path)
if do_verify:
download(sha_path, sha_url, False, verbose)
if os.path.exists(path):
if verify(path, sha_path, False):
if verbose:
print("using already-download file", path)
return
else:
if verbose:
print("ignoring already-download file",
path, "due to failed verification")
os.unlink(path)
download(temp_path, url, True, verbose)
if not verify(temp_path, sha_path, verbose):
if do_verify and not verify(temp_path, sha_path, verbose):
raise RuntimeError("failed verification")
if verbose:
print("moving {} to {}".format(temp_path, path))
@ -365,16 +375,6 @@ class RustBuild(object):
cargo_channel = self.cargo_channel
rustfmt_channel = self.rustfmt_channel
def support_xz():
try:
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
temp_path = temp_file.name
with tarfile.open(temp_path, "w:xz"):
pass
return True
except tarfile.CompressionError:
return False
if self.rustc().startswith(self.bin_root()) and \
(not os.path.exists(self.rustc()) or
self.program_out_of_date(self.rustc_stamp())):
@ -423,6 +423,19 @@ class RustBuild(object):
with output(self.rustfmt_stamp()) as rustfmt_stamp:
rustfmt_stamp.write(self.date + self.rustfmt_channel)
if self.downloading_llvm():
llvm_sha = subprocess.check_output(["git", "log", "--author=bors",
"--format=%H", "-n1"]).decode(sys.getdefaultencoding()).strip()
llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
self._download_ci_llvm(llvm_sha, llvm_assertions)
with output(self.llvm_stamp()) as llvm_stamp:
llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
def downloading_llvm(self):
opt = self.get_toml('download-ci-llvm', 'llvm')
return opt == "true"
def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
if date is None:
date = self.date
@ -437,6 +450,25 @@ class RustBuild(object):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
def _download_ci_llvm(self, llvm_sha, llvm_assertions):
cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, cache_prefix)
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)
url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
if llvm_assertions:
url = url.replace('rustc-builds', 'rustc-builds-alt')
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
filename = "rust-dev-nightly-" + self.build + tarball_suffix
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
unpack(tarball, tarball_suffix, self.llvm_root(),
match="rust-dev",
verbose=self.verbose)
def fix_bin_or_dylib(self, fname):
"""Modifies the interpreter section of 'fname' to fix the dynamic linker,
or the RPATH section, to fix the dynamic library search path
@ -558,6 +590,17 @@ class RustBuild(object):
"""
return os.path.join(self.bin_root(), '.rustfmt-stamp')
def llvm_stamp(self):
"""Return the path for .rustfmt-stamp
>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
True
"""
return os.path.join(self.llvm_root(), '.llvm-stamp')
def program_out_of_date(self, stamp_path, extra=""):
"""Check if the given program stamp is out of date"""
if not os.path.exists(stamp_path) or self.clean:
@ -581,6 +624,22 @@ class RustBuild(object):
"""
return os.path.join(self.build_dir, self.build, "stage0")
def llvm_root(self):
"""Return the CI LLVM root directory
>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.llvm_root() == os.path.join("build", "ci-llvm")
True
When the 'build' property is given should be a nested directory:
>>> rb.build = "devel"
>>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
True
"""
return os.path.join(self.build_dir, self.build, "ci-llvm")
def get_toml(self, key, section=None):
"""Returns the value of the given key in config.toml, otherwise returns None

2
src/bootstrap/compile.rs
View file

@ -593,7 +593,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS
let file = compiler_file(builder, builder.cxx(target).unwrap(), target, "libstdc++.a");
cargo.env("LLVM_STATIC_STDCPP", file);
}
if builder.config.llvm_link_shared || builder.config.llvm_thin_lto {
if builder.config.llvm_link_shared {
cargo.env("LLVM_LINK_SHARED", "1");
}
if builder.config.llvm_use_libcxx {

64
src/bootstrap/config.rs
View file

@ -15,9 +15,20 @@ use std::process;
use crate::cache::{Interned, INTERNER};
use crate::flags::Flags;
pub use crate::flags::Subcommand;
use crate::util::exe;
use build_helper::t;
use serde::Deserialize;
macro_rules! check_ci_llvm {
($name:expr) => {
assert!(
$name.is_none(),
"setting {} is incompatible with download-ci-llvm.",
stringify!($name)
);
};
}
/// Global configuration for the entire build and/or bootstrap.
///
/// This structure is derived from a combination of both `config.toml` and
@ -84,6 +95,7 @@ pub struct Config {
pub llvm_version_suffix: Option<String>,
pub llvm_use_linker: Option<String>,
pub llvm_allow_old_toolchain: Option<bool>,
pub llvm_from_ci: bool,
pub use_lld: bool,
pub lld_enabled: bool,
@ -344,6 +356,7 @@ struct Llvm {
use_libcxx: Option<bool>,
use_linker: Option<String>,
allow_old_toolchain: Option<bool>,
download_ci_llvm: Option<bool>,
}
#[derive(Deserialize, Default, Clone)]
@ -624,6 +637,43 @@ impl Config {
set(&mut config.llvm_use_libcxx, llvm.use_libcxx);
config.llvm_use_linker = llvm.use_linker.clone();
config.llvm_allow_old_toolchain = llvm.allow_old_toolchain;
config.llvm_from_ci = llvm.download_ci_llvm.unwrap_or(false);
if config.llvm_from_ci {
// None of the LLVM options, except assertions, are supported
// when using downloaded LLVM. We could just ignore these but
// that's potentially confusing, so force them to not be
// explicitly set. The defaults and CI defaults don't
// necessarily match but forcing people to match (somewhat
// arbitrary) CI configuration locally seems bad/hard.
check_ci_llvm!(llvm.optimize);
check_ci_llvm!(llvm.thin_lto);
check_ci_llvm!(llvm.release_debuginfo);
check_ci_llvm!(llvm.link_shared);
check_ci_llvm!(llvm.static_libstdcpp);
check_ci_llvm!(llvm.targets);
check_ci_llvm!(llvm.experimental_targets);
check_ci_llvm!(llvm.link_jobs);
check_ci_llvm!(llvm.link_shared);
check_ci_llvm!(llvm.clang_cl);
check_ci_llvm!(llvm.version_suffix);
check_ci_llvm!(llvm.cflags);
check_ci_llvm!(llvm.cxxflags);
check_ci_llvm!(llvm.ldflags);
check_ci_llvm!(llvm.use_libcxx);
check_ci_llvm!(llvm.use_linker);
check_ci_llvm!(llvm.allow_old_toolchain);
// CI-built LLVM is shared
config.llvm_link_shared = true;
}
if config.llvm_thin_lto {
// If we're building with ThinLTO on, we want to link to LLVM
// shared, to avoid re-doing ThinLTO (which happens in the link
// step) with each stage.
config.llvm_link_shared = true;
}
}
if let Some(ref rust) = toml.rust {
@ -706,6 +756,20 @@ impl Config {
}
}
if config.llvm_from_ci {
let triple = &config.build.triple;
let mut build_target = config
.target_config
.entry(config.build)
.or_insert_with(|| Target::from_triple(&triple));
check_ci_llvm!(build_target.llvm_config);
check_ci_llvm!(build_target.llvm_filecheck);
let ci_llvm_bin = config.out.join(&*config.build.triple).join("ci-llvm/bin");
build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build)));
build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build)));
}
if let Some(ref t) = toml.dist {
config.dist_sign_folder = t.sign_folder.clone().map(PathBuf::from);
config.dist_gpg_password_file = t.gpg_password_file.clone().map(PathBuf::from);

34
src/bootstrap/dist.rs
View file

@ -2382,26 +2382,32 @@ impl Step for HashSign {
/// Note: This function does not yet support Windows, but we also don't support
/// linking LLVM tools dynamically on Windows yet.
fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) {
let src_libdir = builder.llvm_out(target).join("lib");
if !builder.config.llvm_link_shared {
// We do not need to copy LLVM files into the sysroot if it is not
// dynamically linked; it is already included into librustc_llvm
// statically.
return;
}
// On macOS for some reason the llvm-config binary behaves differently and
// and fails on missing .a files if run without --link-shared. If run with
// that option, it still fails, but because we only ship a libLLVM.dylib
// rather than libLLVM-11-rust-....dylib file.
//
// For now just don't use llvm-config here on macOS; that will fail to
// support CI-built LLVM, but until we work out the different behavior that
// is fine as it is off by default.
if target.contains("apple-darwin") {
let src_libdir = builder.llvm_out(target).join("lib");
let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
if llvm_dylib_path.exists() {
builder.install(&llvm_dylib_path, dst_libdir, 0o644);
}
return;
}
// Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
// Since tools link to the latter rather than the former, we have to
// follow the symlink to find out what to distribute.
let llvm_dylib_path = src_libdir.join("libLLVM.so");
if llvm_dylib_path.exists() {
let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
panic!("dist: Error calling canonicalize path `{}`: {}", llvm_dylib_path.display(), e);
});
builder.install(&llvm_dylib_path, dst_libdir, 0o644);
} else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
let files = output(Command::new(llvm_config).arg("--libfiles"));
for file in files.lines() {
builder.install(Path::new(file), dst_libdir, 0o644);
}
}
}

4
src/bootstrap/lib.rs
View file

@ -611,6 +611,10 @@ impl Build {
///
/// If no custom `llvm-config` was specified then Rust's llvm will be used.
fn is_rust_llvm(&self, target: TargetSelection) -> bool {
if self.config.llvm_from_ci && target == self.config.build {
return true;
}
match self.config.target_config.get(&target) {
Some(ref c) => c.llvm_config.is_none(),
None => true,