More hacking on the snapshot system.
This commit is contained in:
parent
ed40c85af5
commit
d987b49a4b
|
@ -108,7 +108,6 @@ GENERATED :=
|
||||||
%:: s.%
|
%:: s.%
|
||||||
%:: SCCS/s.%
|
%:: SCCS/s.%
|
||||||
|
|
||||||
|
|
||||||
######################################################################
|
######################################################################
|
||||||
# Standard library variables
|
# Standard library variables
|
||||||
######################################################################
|
######################################################################
|
||||||
|
@ -135,6 +134,13 @@ SREQ1 := stage1/rustc$(X) $(LREQ) stage2/glue.o stage2/$(CFG_STDLIB)
|
||||||
SREQ2 := stage2/rustc$(X) $(LREQ) stage3/glue.o stage3/$(CFG_STDLIB)
|
SREQ2 := stage2/rustc$(X) $(LREQ) stage3/glue.o stage3/$(CFG_STDLIB)
|
||||||
|
|
||||||
|
|
||||||
|
######################################################################
|
||||||
|
# Exports for sub-utilities
|
||||||
|
######################################################################
|
||||||
|
|
||||||
|
export CFG_SRC_DIR
|
||||||
|
|
||||||
|
|
||||||
######################################################################
|
######################################################################
|
||||||
# Single-target rules
|
# Single-target rules
|
||||||
######################################################################
|
######################################################################
|
||||||
|
@ -179,5 +185,6 @@ include $(CFG_SRC_DIR)/mk/rustllvm.mk
|
||||||
include $(CFG_SRC_DIR)/mk/docs.mk
|
include $(CFG_SRC_DIR)/mk/docs.mk
|
||||||
include $(CFG_SRC_DIR)/mk/tests.mk
|
include $(CFG_SRC_DIR)/mk/tests.mk
|
||||||
include $(CFG_SRC_DIR)/mk/dist.mk
|
include $(CFG_SRC_DIR)/mk/dist.mk
|
||||||
|
include $(CFG_SRC_DIR)/mk/snap.mk
|
||||||
include $(CFG_SRC_DIR)/mk/clean.mk
|
include $(CFG_SRC_DIR)/mk/clean.mk
|
||||||
include $(CFG_SRC_DIR)/mk/autodep.mk
|
include $(CFG_SRC_DIR)/mk/autodep.mk
|
||||||
|
|
|
@ -9,34 +9,15 @@ def snap_filename_hash_part(snap):
|
||||||
raise Exception("unable to find hash in filename: " + snap)
|
raise Exception("unable to find hash in filename: " + snap)
|
||||||
return match.group(1)
|
return match.group(1)
|
||||||
|
|
||||||
def get_snapshot_and_check_hash(snap):
|
|
||||||
|
|
||||||
hsh = snap_filename_hash_part(snap)
|
|
||||||
|
|
||||||
h = hashlib.sha1()
|
|
||||||
url = download_url_base + "/" + snap
|
|
||||||
print "downloading " + url
|
|
||||||
u = urllib2.urlopen(url)
|
|
||||||
print "checking hash on download"
|
|
||||||
data = u.read()
|
|
||||||
h.update(data)
|
|
||||||
if h.hexdigest() != hsh:
|
|
||||||
raise Exception("hash check failed on " + snap)
|
|
||||||
|
|
||||||
print "hash ok"
|
|
||||||
with open(os.path.join(download_dir_base, snap), "w+b") as f:
|
|
||||||
f.write(data)
|
|
||||||
return True
|
|
||||||
|
|
||||||
def unpack_snapshot(snap):
|
def unpack_snapshot(snap):
|
||||||
dl_path = os.path.join(download_dir_base, snap)
|
dl_path = os.path.join(download_dir_base, snap)
|
||||||
print "opening snapshot " + dl_path
|
print("opening snapshot " + dl_path)
|
||||||
tar = tarfile.open(dl_path)
|
tar = tarfile.open(dl_path)
|
||||||
kernel = get_kernel()
|
kernel = get_kernel()
|
||||||
for name in snapshot_files[kernel]:
|
for name in snapshot_files[kernel]:
|
||||||
p = os.path.join("rust-stage0", name)
|
p = os.path.join("rust-stage0", name)
|
||||||
fp = os.path.join("stage0", name)
|
fp = os.path.join("stage0", name)
|
||||||
print "extracting " + fp
|
print("extracting " + fp)
|
||||||
tar.extract(p, download_unpack_base)
|
tar.extract(p, download_unpack_base)
|
||||||
tp = os.path.join(download_unpack_base, p)
|
tp = os.path.join(download_unpack_base, p)
|
||||||
shutil.move(tp, fp)
|
shutil.move(tp, fp)
|
||||||
|
@ -80,16 +61,16 @@ def determine_last_snapshot_for_platform():
|
||||||
# Main
|
# Main
|
||||||
|
|
||||||
snap = determine_last_snapshot_for_platform()
|
snap = determine_last_snapshot_for_platform()
|
||||||
print "determined most recent snapshot: " + snap
|
|
||||||
dl = os.path.join(download_dir_base, snap)
|
dl = os.path.join(download_dir_base, snap)
|
||||||
if (os.path.exists(dl)):
|
url = download_url_base + "/" + snap
|
||||||
|
print("determined most recent snapshot: " + snap)
|
||||||
|
|
||||||
|
if (not os.path.exists(dl)):
|
||||||
|
get_url_to_file(url, dl)
|
||||||
|
|
||||||
if (snap_filename_hash_part(snap) == hash_file(dl)):
|
if (snap_filename_hash_part(snap) == hash_file(dl)):
|
||||||
print "found existing download with ok hash"
|
print("got download with ok hash")
|
||||||
else:
|
else:
|
||||||
print "bad hash on existing download, re-fetching"
|
raise Exception("bad hash on download")
|
||||||
get_snapshot_and_check_hash(snap)
|
|
||||||
else:
|
|
||||||
print "no cached download, fetching"
|
|
||||||
get_snapshot_and_check_hash(snap)
|
|
||||||
|
|
||||||
unpack_snapshot(snap)
|
unpack_snapshot(snap)
|
||||||
|
|
|
@ -21,4 +21,4 @@ file1 = full_snapshot_name(date, rev, kernel, cpu, h)
|
||||||
|
|
||||||
shutil.move(file0, file1)
|
shutil.move(file0, file1)
|
||||||
|
|
||||||
print file1
|
print(file1)
|
||||||
|
|
|
@ -1,6 +1,10 @@
|
||||||
import re, os, sys, hashlib, tarfile, shutil, subprocess, urllib2, tempfile
|
import re, os, sys, hashlib, tarfile, shutil, subprocess, tempfile
|
||||||
|
|
||||||
snapshotfile = "snapshots.txt"
|
src_dir = os.getenv("CFG_SRC_DIR")
|
||||||
|
if not src_dir:
|
||||||
|
raise Exception("missing env var CFG_SRC_DIR")
|
||||||
|
|
||||||
|
snapshotfile = os.path.join(src_dir, "snapshots.txt")
|
||||||
download_url_base = "http://dl.rust-lang.org/stage0-snapshots"
|
download_url_base = "http://dl.rust-lang.org/stage0-snapshots"
|
||||||
download_dir_base = "dl"
|
download_dir_base = "dl"
|
||||||
download_unpack_base = os.path.join(download_dir_base, "unpack")
|
download_unpack_base = os.path.join(download_dir_base, "unpack")
|
||||||
|
@ -59,11 +63,16 @@ def get_cpu():
|
||||||
def get_platform():
|
def get_platform():
|
||||||
return "%s-%s" % (get_kernel(), get_cpu())
|
return "%s-%s" % (get_kernel(), get_cpu())
|
||||||
|
|
||||||
|
def scrub(b):
|
||||||
|
if sys.version_info >= (3,) and type(b) == bytes:
|
||||||
|
return b.decode('ascii')
|
||||||
|
else:
|
||||||
|
return b
|
||||||
|
|
||||||
def cmd_out(cmdline):
|
def cmd_out(cmdline):
|
||||||
p = subprocess.Popen(cmdline,
|
p = subprocess.Popen(cmdline,
|
||||||
stdout=subprocess.PIPE)
|
stdout=subprocess.PIPE)
|
||||||
return p.communicate()[0].strip()
|
return scrub(p.communicate()[0].strip())
|
||||||
|
|
||||||
|
|
||||||
def local_rev_info(field):
|
def local_rev_info(field):
|
||||||
|
@ -82,8 +91,10 @@ def local_rev_short_sha():
|
||||||
def local_rev_committer_date():
|
def local_rev_committer_date():
|
||||||
return local_rev_info("ci")
|
return local_rev_info("ci")
|
||||||
|
|
||||||
|
def get_url_to_file(u,f):
|
||||||
|
subprocess.check_call(["curl", "-o", f, u])
|
||||||
|
|
||||||
def hash_file(x):
|
def hash_file(x):
|
||||||
h = hashlib.sha1()
|
h = hashlib.sha1()
|
||||||
h.update(open(x).read())
|
h.update(open(x, "rb").read())
|
||||||
return h.hexdigest()
|
return scrub(h.hexdigest())
|
||||||
|
|
|
@ -1,3 +1,6 @@
|
||||||
|
S 2011-05-02 ed40c85
|
||||||
|
winnt-i386 e69c11fbc62639ac3a3eef7ea36c9ad77209e2b1
|
||||||
|
|
||||||
S 2011-04-29 7b95b5c
|
S 2011-04-29 7b95b5c
|
||||||
linux-i386 f0e166816ce34adc9f7202bd3cfbd80623505f28
|
linux-i386 f0e166816ce34adc9f7202bd3cfbd80623505f28
|
||||||
macos-i386 abf2ee279da63676ca17c9dc9e54d04d8f752b00
|
macos-i386 abf2ee279da63676ca17c9dc9e54d04d8f752b00
|
||||||
|
|
Loading…
Reference in a new issue