squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

Rollup merge of #74441 - eddyb:zlib-on-nixos, r=nagisa

Browse source 
bootstrap.py: patch RPATH on NixOS to handle the new zlib dependency.

This is a stop-gap until #74420 is resolved (assuming we'll patch beta to statically link zlib).

However, I've been meaning to rewrite the NixOS support we have in `bootstrap.py` for a while now, and had to in order to cleanly add zlib as a dependency (the second commit is a relatively small delta in functionality, compared to the first).

Previously, we would extract the `ld-linux.so` path from the output of `ldd /run/current-system/sw/bin/sh`, which assumes a lot. On top of that we didn't use any symlinks, which meant if the user ran  GC (`nix-collect-garbage`), e.g. after updating their system, their `stage0` binaries would suddenly be broken (i.e. referring to files that no longer exist).
We were also using `patchelf` directly, assuming it can be found in `$PATH` (which is not necessarily true).

My new approach relies on using `nix-build` to get the following "derivations" (packages, more or less):
* `stdenv.cc.bintools`, which has a `nix-support/dynamic-linker` file containing the path to `ld-linux.so`
  * reading this file is [the canonical way to run `patchelf --set-interpreter`](https://github.com/NixOS/nixpkgs/search?l=Nix&q=%22--set-interpreter+%24%28cat+%24NIX_CC%2Fnix-support%2Fdynamic-linker%29%22)
* `patchelf` (so that the user doesn't need to have it installed)
* `zlib`, for the `libz.so` dependency of `libLLVM-*.so` (until #74420 is resolved, presumably)

This is closer to how software is built on Nix, but I've tried to keep it as simple as possible (and not add e.g. a `stage0.nix` file).
Symlinks to each of those dependencies are kept in `stage0/.nix-deps`, which prevents GC from invalidating `stage0` binaries.

r? @nagisa cc @Mark-Simulacrum @oli-obk @davidtwco
This commit is contained in:
Manish Goregaokar 2020-07-17 18:13:44 -07:00 committed by GitHub
commit 8d1bb0e748
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 63 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

98
src/bootstrap/bootstrap.py
View file

@ -349,6 +349,7 @@ class RustBuild(object):
self.use_vendored_sources = ''
self.verbose = False
self.git_version = None
self.nix_deps_dir = None
def download_stage0(self):
"""Fetch the build system for Rust, written in Rust
@ -388,8 +389,12 @@ class RustBuild(object):
filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix)
self._download_stage0_helper(filename, "rustc", tarball_suffix)
self.fix_executable("{}/bin/rustc".format(self.bin_root()))
self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
lib_dir = "{}/lib".format(self.bin_root())
for lib in os.listdir(lib_dir):
if lib.endswith(".so"):
self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
with output(self.rustc_stamp()) as rust_stamp:
rust_stamp.write(self.date)
@ -408,7 +413,7 @@ class RustBuild(object):
filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
tarball_suffix)
self._download_stage0_helper(filename, "cargo", tarball_suffix)
self.fix_executable("{}/bin/cargo".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
with output(self.cargo_stamp()) as cargo_stamp:
cargo_stamp.write(self.date)
@ -421,8 +426,8 @@ class RustBuild(object):
[channel, date] = rustfmt_channel.split('-', 1)
filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
self.fix_executable("{}/bin/rustfmt".format(self.bin_root()))
self.fix_executable("{}/bin/cargo-fmt".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
with output(self.rustfmt_stamp()) as rustfmt_stamp:
rustfmt_stamp.write(self.date + self.rustfmt_channel)
@ -440,12 +445,12 @@ class RustBuild(object):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
@staticmethod
def fix_executable(fname):
"""Modifies the interpreter section of 'fname' to fix the dynamic linker
def fix_bin_or_dylib(self, fname):
"""Modifies the interpreter section of 'fname' to fix the dynamic linker,
or the RPATH section, to fix the dynamic library search path
This method is only required on NixOS and uses the PatchELF utility to
change the dynamic linker of ELF executables.
change the interpreter/RPATH of ELF executables.
Please see https://nixos.org/patchelf.html for more information
"""
@ -472,38 +477,61 @@ class RustBuild(object):
nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
print(nix_os_msg, fname)
try:
interpreter = subprocess.check_output(
["patchelf", "--print-interpreter", fname])
interpreter = interpreter.strip().decode(default_encoding)
except subprocess.CalledProcessError as reason:
print("warning: failed to call patchelf:", reason)
return
# Only build `stage0/.nix-deps` once.
nix_deps_dir = self.nix_deps_dir
if not nix_deps_dir:
nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
if not os.path.exists(nix_deps_dir):
os.makedirs(nix_deps_dir)
loader = interpreter.split("/")[-1]
nix_deps = [
# Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
"stdenv.cc.bintools",
try:
ldd_output = subprocess.check_output(
['ldd', '/run/current-system/sw/bin/sh'])
ldd_output = ldd_output.strip().decode(default_encoding)
except subprocess.CalledProcessError as reason:
print("warning: unable to call ldd:", reason)
return
# Needed as a system dependency of `libLLVM-*.so`.
"zlib",
for line in ldd_output.splitlines():
libname = line.split()[0]
if libname.endswith(loader):
loader_path = libname[:len(libname) - len(loader)]
break
# Needed for patching ELF binaries (see doc comment above).
"patchelf",
]
# Run `nix-build` to "build" each dependency (which will likely reuse
# the existing `/nix/store` copy, or at most download a pre-built copy).
# Importantly, we don't rely on `nix-build` printing the `/nix/store`
# path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
# ensuring garbage collection will never remove the `/nix/store` path
# (which would break our patched binaries that hardcode those paths).
for dep in nix_deps:
try:
subprocess.check_output([
"nix-build", "<nixpkgs>",
"-A", dep,
"-o", "{}/{}".format(nix_deps_dir, dep),
])
except subprocess.CalledProcessError as reason:
print("warning: failed to call nix-build:", reason)
return
self.nix_deps_dir = nix_deps_dir
patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
if fname.endswith(".so"):
# Dynamic library, patch RPATH to point to system dependencies.
dylib_deps = ["zlib"]
rpath_entries = [
# Relative default, all binary and dynamic libraries we ship
# appear to have this (even when `../lib` is redundant).
"$ORIGIN/../lib",
] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
else:
print("warning: unable to find the path to the dynamic linker")
return
correct_interpreter = loader_path + loader
bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
try:
subprocess.check_output(
["patchelf", "--set-interpreter", correct_interpreter, fname])
subprocess.check_output([patchelf] + patchelf_args + [fname])
except subprocess.CalledProcessError as reason:
print("warning: failed to call patchelf:", reason)
return