squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

Download LLVM from CI to bootstrap

Browse source
This commit is contained in:
Mark Rousskov 2020-09-04 19:00:04 -04:00
parent 2d6cbd21b2
commit a7b092f418
5 changed files with 178 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
config.toml.example
View file

@ -14,6 +14,21 @@
# =============================================================================
[llvm]
# Whether to use Rust CI built LLVM instead of locally building it.
#
# Unless you're developing for a target where Rust CI doesn't build a compiler
# toolchain or changing LLVM locally, you probably want to set this to true.
#
# It's currently false by default due to being newly added; please file bugs if
# enabling this did not work for you on Linux (macOS and Windows support is
# coming soon).
#
# We also currently only support this when building LLVM for the build triple.
#
# Note that many of the LLVM options are not currently supported for
# downloading. Currently only the "assertions" option can be toggled.
#download-ci-llvm = false
# Indicates whether LLVM rebuild should be skipped when running bootstrap. If
# this is `false` then the compiler's LLVM will be rebuilt whenever the built
# version doesn't have the correct hash. If it is `true` then LLVM will never

105
src/bootstrap/bootstrap.py
View file

@ -14,8 +14,17 @@ import tempfile
from time import time
def support_xz():
try:
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
temp_path = temp_file.name
with tarfile.open(temp_path, "w:xz"):
pass
return True
except tarfile.CompressionError:
return False
def get(url, path, verbose=False):
def get(url, path, verbose=False, do_verify=True):
suffix = '.sha256'
sha_url = url + suffix
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
@ -24,19 +33,20 @@ def get(url, path, verbose=False):
sha_path = sha_file.name
try:
download(sha_path, sha_url, False, verbose)
if os.path.exists(path):
if verify(path, sha_path, False):
if verbose:
print("using already-download file", path)
return
else:
if verbose:
print("ignoring already-download file",
path, "due to failed verification")
os.unlink(path)
if do_verify:
download(sha_path, sha_url, False, verbose)
if os.path.exists(path):
if verify(path, sha_path, False):
if verbose:
print("using already-download file", path)
return
else:
if verbose:
print("ignoring already-download file",
path, "due to failed verification")
os.unlink(path)
download(temp_path, url, True, verbose)
if not verify(temp_path, sha_path, verbose):
if do_verify and not verify(temp_path, sha_path, verbose):
raise RuntimeError("failed verification")
if verbose:
print("moving {} to {}".format(temp_path, path))
@ -365,16 +375,6 @@ class RustBuild(object):
cargo_channel = self.cargo_channel
rustfmt_channel = self.rustfmt_channel
def support_xz():
try:
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
temp_path = temp_file.name
with tarfile.open(temp_path, "w:xz"):
pass
return True
except tarfile.CompressionError:
return False
if self.rustc().startswith(self.bin_root()) and \
(not os.path.exists(self.rustc()) or
self.program_out_of_date(self.rustc_stamp())):
@ -423,6 +423,19 @@ class RustBuild(object):
with output(self.rustfmt_stamp()) as rustfmt_stamp:
rustfmt_stamp.write(self.date + self.rustfmt_channel)
if self.downloading_llvm():
llvm_sha = subprocess.check_output(["git", "log", "--author=bors",
"--format=%H", "-n1"]).decode(sys.getdefaultencoding()).strip()
llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
self._download_ci_llvm(llvm_sha, llvm_assertions)
with output(self.llvm_stamp()) as llvm_stamp:
llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
def downloading_llvm(self):
opt = self.get_toml('download-ci-llvm', 'llvm')
return opt == "true"
def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
if date is None:
date = self.date
@ -437,6 +450,25 @@ class RustBuild(object):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
def _download_ci_llvm(self, llvm_sha, llvm_assertions):
cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, cache_prefix)
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)
url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
if llvm_assertions:
url = url.replace('rustc-builds', 'rustc-builds-alt')
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
filename = "rust-dev-nightly-" + self.build + tarball_suffix
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
unpack(tarball, tarball_suffix, self.llvm_root(),
match="rust-dev",
verbose=self.verbose)
def fix_bin_or_dylib(self, fname):
"""Modifies the interpreter section of 'fname' to fix the dynamic linker,
or the RPATH section, to fix the dynamic library search path
@ -558,6 +590,17 @@ class RustBuild(object):
"""
return os.path.join(self.bin_root(), '.rustfmt-stamp')
def llvm_stamp(self):
"""Return the path for .rustfmt-stamp
>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
True
"""
return os.path.join(self.llvm_root(), '.llvm-stamp')
def program_out_of_date(self, stamp_path, extra=""):
"""Check if the given program stamp is out of date"""
if not os.path.exists(stamp_path) or self.clean:
@ -581,6 +624,22 @@ class RustBuild(object):
"""
return os.path.join(self.build_dir, self.build, "stage0")
def llvm_root(self):
"""Return the CI LLVM root directory
>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.llvm_root() == os.path.join("build", "ci-llvm")
True
When the 'build' property is given should be a nested directory:
>>> rb.build = "devel"
>>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
True
"""
return os.path.join(self.build_dir, self.build, "ci-llvm")
def get_toml(self, key, section=None):
"""Returns the value of the given key in config.toml, otherwise returns None

57
src/bootstrap/config.rs
View file

@ -15,9 +15,20 @@ use std::process;
use crate::cache::{Interned, INTERNER};
use crate::flags::Flags;
pub use crate::flags::Subcommand;
use crate::util::exe;
use build_helper::t;
use serde::Deserialize;
macro_rules! check_ci_llvm {
($name:expr) => {
assert!(
$name.is_none(),
"setting {} is incompatible with download-ci-llvm.",
stringify!($name)
);
};
}
/// Global configuration for the entire build and/or bootstrap.
///
/// This structure is derived from a combination of both `config.toml` and
@ -84,6 +95,7 @@ pub struct Config {
pub llvm_version_suffix: Option<String>,
pub llvm_use_linker: Option<String>,
pub llvm_allow_old_toolchain: Option<bool>,
pub llvm_from_ci: bool,
pub use_lld: bool,
pub lld_enabled: bool,
@ -344,6 +356,7 @@ struct Llvm {
use_libcxx: Option<bool>,
use_linker: Option<String>,
allow_old_toolchain: Option<bool>,
download_ci_llvm: Option<bool>,
}
#[derive(Deserialize, Default, Clone)]
@ -624,6 +637,36 @@ impl Config {
set(&mut config.llvm_use_libcxx, llvm.use_libcxx);
config.llvm_use_linker = llvm.use_linker.clone();
config.llvm_allow_old_toolchain = llvm.allow_old_toolchain;
config.llvm_from_ci = llvm.download_ci_llvm.unwrap_or(false);
if config.llvm_from_ci {
// None of the LLVM options, except assertions, are supported
// when using downloaded LLVM. We could just ignore these but
// that's potentially confusing, so force them to not be
// explicitly set. The defaults and CI defaults don't
// necessarily match but forcing people to match (somewhat
// arbitrary) CI configuration locally seems bad/hard.
check_ci_llvm!(llvm.optimize);
check_ci_llvm!(llvm.thin_lto);
check_ci_llvm!(llvm.release_debuginfo);
check_ci_llvm!(llvm.link_shared);
check_ci_llvm!(llvm.static_libstdcpp);
check_ci_llvm!(llvm.targets);
check_ci_llvm!(llvm.experimental_targets);
check_ci_llvm!(llvm.link_jobs);
check_ci_llvm!(llvm.link_shared);
check_ci_llvm!(llvm.clang_cl);
check_ci_llvm!(llvm.version_suffix);
check_ci_llvm!(llvm.cflags);
check_ci_llvm!(llvm.cxxflags);
check_ci_llvm!(llvm.ldflags);
check_ci_llvm!(llvm.use_libcxx);
check_ci_llvm!(llvm.use_linker);
check_ci_llvm!(llvm.allow_old_toolchain);
// CI-built LLVM is shared
config.llvm_link_shared = true;
}
}
if let Some(ref rust) = toml.rust {
@ -706,6 +749,20 @@ impl Config {
}
}
if config.llvm_from_ci {
let triple = &config.build.triple;
let mut build_target = config
.target_config
.entry(config.build)
.or_insert_with(|| Target::from_triple(&triple));
check_ci_llvm!(build_target.llvm_config);
check_ci_llvm!(build_target.llvm_filecheck);
let ci_llvm_bin = config.out.join(&*config.build.triple).join("ci-llvm/bin");
build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build)));
build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build)));
}
if let Some(ref t) = toml.dist {
config.dist_sign_folder = t.sign_folder.clone().map(PathBuf::from);
config.dist_gpg_password_file = t.gpg_password_file.clone().map(PathBuf::from);

34
src/bootstrap/dist.rs
View file

@ -2382,26 +2382,32 @@ impl Step for HashSign {
/// Note: This function does not yet support Windows, but we also don't support
/// linking LLVM tools dynamically on Windows yet.
fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) {
let src_libdir = builder.llvm_out(target).join("lib");
if !builder.config.llvm_link_shared {
// We do not need to copy LLVM files into the sysroot if it is not
// dynamically linked; it is already included into librustc_llvm
// statically.
return;
}
// On macOS for some reason the llvm-config binary behaves differently and
// and fails on missing .a files if run without --link-shared. If run with
// that option, it still fails, but because we only ship a libLLVM.dylib
// rather than libLLVM-11-rust-....dylib file.
//
// For now just don't use llvm-config here on macOS; that will fail to
// support CI-built LLVM, but until we work out the different behavior that
// is fine as it is off by default.
if target.contains("apple-darwin") {
let src_libdir = builder.llvm_out(target).join("lib");
let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
if llvm_dylib_path.exists() {
builder.install(&llvm_dylib_path, dst_libdir, 0o644);
}
return;
}
// Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
// Since tools link to the latter rather than the former, we have to
// follow the symlink to find out what to distribute.
let llvm_dylib_path = src_libdir.join("libLLVM.so");
if llvm_dylib_path.exists() {
let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
panic!("dist: Error calling canonicalize path `{}`: {}", llvm_dylib_path.display(), e);
});
builder.install(&llvm_dylib_path, dst_libdir, 0o644);
} else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
let files = output(Command::new(llvm_config).arg("--libfiles"));
for file in files.lines() {
builder.install(Path::new(file), dst_libdir, 0o644);
}
}
}

4
src/bootstrap/lib.rs
View file

@ -611,6 +611,10 @@ impl Build {
///
/// If no custom `llvm-config` was specified then Rust's llvm will be used.
fn is_rust_llvm(&self, target: TargetSelection) -> bool {
if self.config.llvm_from_ci && target == self.config.build {
return true;
}
match self.config.target_config.get(&target) {
Some(ref c) => c.llvm_config.is_none(),
None => true,