Auto merge of #59926 - pietroalbini:android-sdk-manager, r=alexcrichton
ci: use a custom android sdk manager with pinning and mirroring Google's own sdkmanager has two issues that make it unsuitable for us: * Mirroring has to be done manually, which is annoying because we need to figure out on our own all the URLs to copy (I couldn't find any documentation when building this PR, had to use mitmproxy). * There is no support for pinning, which means an update on Google's side can break our CI, as it happened multiple times. This PR replaces all our usage of sdkmanager with a custom Python script which mimics its behavior, but with the two issues fixes. sdkmanager's logic for installing packages is thankfully very simple: the package name (like `system-images;android-18;default;armeabi-v7a`) is the directory where the package should live (with `;` replaced with `/`), so to install a package we only need to extract its contents in the right directory. r? @alexcrichton cc @kennytm fixes https://github.com/rust-lang/rust/issues/59778
This commit is contained in:
bors
commit
2975a3c4be
4 changed files with 225 additions and 69 deletions
|
|
@ -7,23 +7,21 @@ COPY scripts/android-ndk.sh /scripts/
|
|||
RUN . /scripts/android-ndk.sh && \
|
||||
download_and_make_toolchain android-ndk-r15c-linux-x86_64.zip arm 14
|
||||
|
||||
# Note:
|
||||
# Do not upgrade to `openjdk-9-jre-headless`, as it will cause certificate error
|
||||
# when installing the Android SDK (see PR #45193). This is unfortunate, but
|
||||
# every search result suggested either disabling HTTPS or replacing JDK 9 by
|
||||
# JDK 8 as the solution (e.g. https://stackoverflow.com/q/41421340). :|
|
||||
RUN dpkg --add-architecture i386 && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
libgl1-mesa-glx \
|
||||
libpulse0 \
|
||||
libstdc++6:i386 \
|
||||
openjdk-8-jre-headless \
|
||||
tzdata
|
||||
openjdk-9-jre-headless \
|
||||
tzdata \
|
||||
wget \
|
||||
python3
|
||||
|
||||
COPY scripts/android-sdk.sh /scripts/
|
||||
RUN . /scripts/android-sdk.sh && \
|
||||
download_and_create_avd 4333796 armeabi-v7a 18 5264690
|
||||
COPY scripts/android-sdk-manager.py /scripts/
|
||||
COPY arm-android/android-sdk.lock /android/sdk/android-sdk.lock
|
||||
RUN /scripts/android-sdk.sh
|
||||
|
||||
ENV PATH=$PATH:/android/sdk/emulator
|
||||
ENV PATH=$PATH:/android/sdk/tools
|
||||
|
|
|
|||
6
src/ci/docker/arm-android/android-sdk.lock
Normal file
6
src/ci/docker/arm-android/android-sdk.lock
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
emulator emulator-linux-5264690.zip 48c1cda2bdf3095d9d9d5c010fbfb3d6d673e3ea
|
||||
patcher;v4 3534162-studio.sdk-patcher.zip 046699c5e2716ae11d77e0bad814f7f33fab261e
|
||||
platform-tools platform-tools_r28.0.2-linux.zip 46a4c02a9b8e4e2121eddf6025da3c979bf02e28
|
||||
platforms;android-18 android-18_r03.zip e6b09b3505754cbbeb4a5622008b907262ee91cb
|
||||
system-images;android-18;default;armeabi-v7a sys-img/android/armeabi-v7a-18_r05.zip 580b583720f7de671040d5917c8c9db0c7aa03fd
|
||||
tools sdk-tools-linux-4333796.zip 8c7c28554a32318461802c1291d76fccfafde054
|
||||
190
src/ci/docker/scripts/android-sdk-manager.py
Executable file
190
src/ci/docker/scripts/android-sdk-manager.py
Executable file
|
|
@ -0,0 +1,190 @@
|
|||
#!/usr/bin/env python3
|
||||
# Simpler reimplementation of Android's sdkmanager
|
||||
# Extra features of this implementation are pinning and mirroring
|
||||
|
||||
# These URLs are the Google repositories containing the list of available
|
||||
# packages and their versions. The list has been generated by listing the URLs
|
||||
# fetched while executing `tools/bin/sdkmanager --list`
|
||||
BASE_REPOSITORY = "https://dl.google.com/android/repository/"
|
||||
REPOSITORIES = [
|
||||
"sys-img/android/sys-img2-1.xml",
|
||||
"sys-img/android-wear/sys-img2-1.xml",
|
||||
"sys-img/android-wear-cn/sys-img2-1.xml",
|
||||
"sys-img/android-tv/sys-img2-1.xml",
|
||||
"sys-img/google_apis/sys-img2-1.xml",
|
||||
"sys-img/google_apis_playstore/sys-img2-1.xml",
|
||||
"addon2-1.xml",
|
||||
"glass/addon2-1.xml",
|
||||
"extras/intel/addon2-1.xml",
|
||||
"repository2-1.xml",
|
||||
]
|
||||
|
||||
# Available hosts: linux, macosx and windows
|
||||
HOST_OS = "linux"
|
||||
|
||||
# Mirroring options
|
||||
MIRROR_BUCKET = "rust-lang-ci2"
|
||||
MIRROR_BASE_DIR = "rust-ci-mirror/android/"
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
class Package:
|
||||
def __init__(self, path, url, sha1, deps=None):
|
||||
if deps is None:
|
||||
deps = []
|
||||
self.path = path.strip()
|
||||
self.url = url.strip()
|
||||
self.sha1 = sha1.strip()
|
||||
self.deps = deps
|
||||
|
||||
def download(self, base_url):
|
||||
_, file = tempfile.mkstemp()
|
||||
url = base_url + self.url
|
||||
subprocess.run(["curl", "-o", file, url], check=True)
|
||||
# Ensure there are no hash mismatches
|
||||
with open(file, "rb") as f:
|
||||
sha1 = hashlib.sha1(f.read()).hexdigest()
|
||||
if sha1 != self.sha1:
|
||||
raise RuntimeError(
|
||||
"hash mismatch for package " + self.path + ": " +
|
||||
sha1 + " vs " + self.sha1 + " (known good)"
|
||||
)
|
||||
return file
|
||||
|
||||
def __repr__(self):
|
||||
return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")"
|
||||
|
||||
def fetch_url(url):
|
||||
page = urllib.request.urlopen(url)
|
||||
return page.read()
|
||||
|
||||
def fetch_repository(base, repo_url):
|
||||
packages = {}
|
||||
root = ET.fromstring(fetch_url(base + repo_url))
|
||||
for package in root:
|
||||
if package.tag != "remotePackage":
|
||||
continue
|
||||
path = package.attrib["path"]
|
||||
|
||||
for archive in package.find("archives"):
|
||||
host_os = archive.find("host-os")
|
||||
if host_os is not None and host_os.text != HOST_OS:
|
||||
continue
|
||||
complete = archive.find("complete")
|
||||
url = os.path.join(os.path.dirname(repo_url), complete.find("url").text)
|
||||
sha1 = complete.find("checksum").text
|
||||
|
||||
deps = []
|
||||
dependencies = package.find("dependencies")
|
||||
if dependencies is not None:
|
||||
for dep in dependencies:
|
||||
deps.append(dep.attrib["path"])
|
||||
|
||||
packages[path] = Package(path, url, sha1, deps)
|
||||
break
|
||||
|
||||
return packages
|
||||
|
||||
def fetch_repositories():
|
||||
packages = {}
|
||||
for repo in REPOSITORIES:
|
||||
packages.update(fetch_repository(BASE_REPOSITORY, repo))
|
||||
return packages
|
||||
|
||||
class Lockfile:
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.packages = {}
|
||||
if os.path.exists(path):
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
path, url, sha1 = line.split(" ")
|
||||
self.packages[path] = Package(path, url, sha1)
|
||||
|
||||
def add(self, packages, name, *, update=True):
|
||||
if name not in packages:
|
||||
raise NameError("package not found: " + name)
|
||||
if not update and name in self.packages:
|
||||
return
|
||||
self.packages[name] = packages[name]
|
||||
for dep in packages[name].deps:
|
||||
self.add(packages, dep, update=False)
|
||||
|
||||
def save(self):
|
||||
packages = list(sorted(self.packages.values(), key=lambda p: p.path))
|
||||
with open(self.path, "w") as f:
|
||||
for package in packages:
|
||||
f.write(package.path + " " + package.url + " " + package.sha1 + "\n")
|
||||
|
||||
def cli_add_to_lockfile(args):
|
||||
lockfile = Lockfile(args.lockfile)
|
||||
packages = fetch_repositories()
|
||||
for package in args.packages:
|
||||
lockfile.add(packages, package)
|
||||
lockfile.save()
|
||||
|
||||
def cli_update_mirror(args):
|
||||
lockfile = Lockfile(args.lockfile)
|
||||
for package in lockfile.packages.values():
|
||||
path = package.download(BASE_REPOSITORY)
|
||||
subprocess.run([
|
||||
"aws", "s3", "mv", path,
|
||||
"s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url,
|
||||
"--profile=" + args.awscli_profile,
|
||||
], check=True)
|
||||
|
||||
def cli_install(args):
|
||||
lockfile = Lockfile(args.lockfile)
|
||||
for package in lockfile.packages.values():
|
||||
# Download the file from the mirror into a temp file
|
||||
url = "https://" + MIRROR_BUCKET + ".s3.amazonaws.com/" + MIRROR_BASE_DIR
|
||||
downloaded = package.download(url)
|
||||
# Extract the file in a temporary directory
|
||||
extract_dir = tempfile.mkdtemp()
|
||||
subprocess.run([
|
||||
"unzip", "-q", downloaded, "-d", extract_dir,
|
||||
], check=True)
|
||||
# Figure out the prefix used in the zip
|
||||
subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")]
|
||||
if len(subdirs) != 1:
|
||||
raise RuntimeError("extracted directory contains more than one dir")
|
||||
# Move the extracted files in the proper directory
|
||||
dest = os.path.join(args.dest, package.path.replace(";", "/"))
|
||||
os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True)
|
||||
os.rename(os.path.join(extract_dir, subdirs[0]), dest)
|
||||
os.unlink(downloaded)
|
||||
|
||||
def cli():
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers()
|
||||
|
||||
add_to_lockfile = subparsers.add_parser("add-to-lockfile")
|
||||
add_to_lockfile.add_argument("lockfile")
|
||||
add_to_lockfile.add_argument("packages", nargs="+")
|
||||
add_to_lockfile.set_defaults(func=cli_add_to_lockfile)
|
||||
|
||||
update_mirror = subparsers.add_parser("update-mirror")
|
||||
update_mirror.add_argument("lockfile")
|
||||
update_mirror.add_argument("--awscli-profile", default="default")
|
||||
update_mirror.set_defaults(func=cli_update_mirror)
|
||||
|
||||
install = subparsers.add_parser("install")
|
||||
install.add_argument("lockfile")
|
||||
install.add_argument("dest")
|
||||
install.set_defaults(func=cli_install)
|
||||
|
||||
args = parser.parse_args()
|
||||
if not hasattr(args, "func"):
|
||||
print("error: a subcommand is required (see --help)")
|
||||
exit(1)
|
||||
args.func(args)
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
82
src/ci/docker/scripts/android-sdk.sh
Normal file → Executable file
82
src/ci/docker/scripts/android-sdk.sh
Normal file → Executable file
|
|
@ -2,66 +2,28 @@ set -ex
|
|||
|
||||
export ANDROID_HOME=/android/sdk
|
||||
PATH=$PATH:"${ANDROID_HOME}/tools/bin"
|
||||
LOCKFILE="${ANDROID_HOME}/android-sdk.lock"
|
||||
|
||||
download_sdk() {
|
||||
mkdir -p /android
|
||||
curl -fo sdk.zip "https://dl.google.com/android/repository/sdk-tools-linux-$1.zip"
|
||||
unzip -q sdk.zip -d "$ANDROID_HOME"
|
||||
rm -f sdk.zip
|
||||
}
|
||||
|
||||
download_sysimage() {
|
||||
abi=$1
|
||||
api=$2
|
||||
|
||||
# See https://developer.android.com/studio/command-line/sdkmanager.html for
|
||||
# usage of `sdkmanager`.
|
||||
#
|
||||
# The output from sdkmanager is so noisy that it will occupy all of the 4 MB
|
||||
# log extremely quickly. Thus we must silence all output.
|
||||
yes | sdkmanager --licenses > /dev/null
|
||||
yes | sdkmanager platform-tools \
|
||||
"platforms;android-$api" \
|
||||
"system-images;android-$api;default;$abi" > /dev/null
|
||||
}
|
||||
|
||||
download_emulator() {
|
||||
# Download a pinned version of the emulator since upgrades can cause issues
|
||||
curl -fo emulator.zip "https://dl.google.com/android/repository/emulator-linux-$1.zip"
|
||||
rm -rf "${ANDROID_HOME}/emulator"
|
||||
unzip -q emulator.zip -d "${ANDROID_HOME}"
|
||||
rm -f emulator.zip
|
||||
}
|
||||
|
||||
create_avd() {
|
||||
abi=$1
|
||||
api=$2
|
||||
|
||||
# See https://developer.android.com/studio/command-line/avdmanager.html for
|
||||
# usage of `avdmanager`.
|
||||
echo no | avdmanager create avd \
|
||||
-n "$abi-$api" \
|
||||
-k "system-images;android-$api;default;$abi"
|
||||
}
|
||||
|
||||
download_and_create_avd() {
|
||||
download_sdk $1
|
||||
download_sysimage $2 $3
|
||||
create_avd $2 $3
|
||||
download_emulator $4
|
||||
}
|
||||
|
||||
# Usage:
|
||||
# To add a new packages to the SDK or to update an existing one you need to
|
||||
# run the command:
|
||||
#
|
||||
# download_and_create_avd 4333796 armeabi-v7a 18 5264690
|
||||
# android-sdk-manager.py add-to-lockfile $LOCKFILE <package-name>
|
||||
#
|
||||
# 4333796 =>
|
||||
# SDK tool version.
|
||||
# Copy from https://developer.android.com/studio/index.html#command-tools
|
||||
# armeabi-v7a =>
|
||||
# System image ABI
|
||||
# 18 =>
|
||||
# Android API Level (18 = Android 4.3 = Jelly Bean MR2)
|
||||
# 5264690 =>
|
||||
# Android Emulator version.
|
||||
# Copy from the "build_id" in the `/android/sdk/emulator/emulator -version` output
|
||||
# Then, after every lockfile update the mirror has to be synchronized as well:
|
||||
#
|
||||
# android-sdk-manager.py update-mirror $LOCKFILE
|
||||
#
|
||||
/scripts/android-sdk-manager.py install "${LOCKFILE}" "${ANDROID_HOME}"
|
||||
|
||||
details=$(cat "${LOCKFILE}" \
|
||||
| grep system-images \
|
||||
| sed 's/^system-images;android-\([0-9]\+\);default;\([a-z0-9-]\+\) /\1 \2 /g')
|
||||
api="$(echo "${details}" | awk '{print($1)}')"
|
||||
abi="$(echo "${details}" | awk '{print($2)}')"
|
||||
|
||||
# See https://developer.android.com/studio/command-line/avdmanager.html for
|
||||
# usage of `avdmanager`.
|
||||
echo no | avdmanager create avd \
|
||||
-n "$abi-$api" \
|
||||
-k "system-images;android-$api;default;$abi"
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue