squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

cargo dev crater: cleanup, don't re-download and reextract crates on every run

Browse source
This commit is contained in:
Matthias Krüger 2020-12-18 20:58:46 +01:00
parent 734d2052df
commit 7314133722
1 changed files with 51 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
clippy_dev/src/crater.rs
View file

@ -13,6 +13,7 @@ struct KrateSource {
struct Krate {
version: String,
name: String,
// path to the extracted sources that clippy can check
path: PathBuf,
}
@ -23,37 +24,44 @@ impl KrateSource {
name: name.into(),
}
}
fn download_and_extract(&self) -> Krate {
let extract_dir = PathBuf::from("target/crater/crates");
// download
let krate_download_dir = PathBuf::from("target/crater/downloads");
// url to download the crate from crates.io
let url = format!(
"https://crates.io/api/v1/crates/{}/{}/download",
self.name, self.version
);
println!("Downloading {}, {} / {}", self.name, self.version, url);
println!("Downloading and extracting {} {} from {}", self.name, self.version, url);
let _ = std::fs::create_dir("target/crater/");
let _ = std::fs::create_dir(&krate_download_dir);
let _ = std::fs::create_dir(&extract_dir);
let krate_name = format!("{}-{}.crate.tar.gz", &self.name, &self.version);
let krate_file_path = krate_download_dir.join(krate_name);
let mut krate_dest = std::fs::File::create(&krate_file_path).unwrap();
let mut krate_req = ureq::get(&url).call().unwrap().into_reader();
std::io::copy(&mut krate_req, &mut krate_dest).unwrap();
// unzip the tarball
let dl = std::fs::File::open(krate_file_path).unwrap();
let krate_file_path = krate_download_dir.join(format!("{}-{}.crate.tar.gz", &self.name, &self.version));
// don't download/extract if we already have done so
if !krate_file_path.is_file() {
// create a file path to download and write the crate data into
let mut krate_dest = std::fs::File::create(&krate_file_path).unwrap();
let mut krate_req = ureq::get(&url).call().unwrap().into_reader();
// copy the crate into the file
std::io::copy(&mut krate_req, &mut krate_dest).unwrap();
let ungz_tar = flate2::read::GzDecoder::new(dl);
// extract the tar archive
let mut archiv = tar::Archive::new(ungz_tar);
let extract_path = extract_dir.clone();
archiv.unpack(&extract_path).expect("Failed to extract!");
// extracted
dbg!(&extract_path);
// unzip the tarball
let ungz_tar = flate2::read::GzDecoder::new(std::fs::File::open(&krate_file_path).unwrap());
// extract the tar archive
let mut archiv = tar::Archive::new(ungz_tar);
archiv.unpack(&extract_dir).expect("Failed to extract!");
// unzip the tarball
let ungz_tar = flate2::read::GzDecoder::new(std::fs::File::open(&krate_file_path).unwrap());
// extract the tar archive
let mut archiv = tar::Archive::new(ungz_tar);
archiv.unpack(&extract_dir).expect("Failed to extract!");
}
// crate is extracted, return a new Krate object which contains the path to the extracted
// sources that clippy can check
Krate {
version: self.version.clone(),
name: self.name.clone(),
@ -64,24 +72,37 @@ impl KrateSource {
impl Krate {
fn run_clippy_lints(&self, cargo_clippy_path: &PathBuf) -> Vec<String> {
println!("Linting {} {}...", &self.name, &self.version);
let cargo_clippy_path = std::fs::canonicalize(cargo_clippy_path).unwrap();
let project_root = &self.path;
dbg!(&cargo_clippy_path);
dbg!(&project_root);
let output = std::process::Command::new(cargo_clippy_path)
let all_output = std::process::Command::new(cargo_clippy_path)
// lint warnings will look like this:
// src/cargo/ops/cargo_compile.rs:127:35: warning: usage of `FromIterator::from_iter`
.args(&["--", "--message-format=short", "--", "--cap-lints=warn"])
.current_dir(project_root)
.current_dir(&self.path)
.output()
.unwrap();
let mut output = String::from_utf8_lossy(&output.stderr);
let output: Vec<&str> = output.lines().collect();
let mut output: Vec<String> = output
let stderr = String::from_utf8_lossy(&all_output.stderr);
let output_lines = stderr.lines();
let mut output: Vec<String> = output_lines
.into_iter()
.filter(|line| line.contains(": warning: "))
.map(|l| l.to_string())
// prefix with the crate name and version
// cargo-0.49.0/src/cargo/ops/cargo_compile.rs:127:35: warning: usage of `FromIterator::from_iter`
.map(|line| format!("{}-{}/{}", self.name, self.version, line))
// remove the "warning: "
.map(|line| {
let remove_pat = "warning: ";
let pos = line
.find(&remove_pat)
.expect("clippy output did not contain \"warning: \"");
let mut new = line[0..pos].to_string();
new.push_str(&line[pos + remove_pat.len()..]);
new
})
.collect();
// sort messages alphabtically to avoid noise in the logs
output.sort();
output
}
@ -97,7 +118,6 @@ fn build_clippy() {
// the main fn
pub fn run() {
let cargo_clippy_path: PathBuf = PathBuf::from("target/debug/cargo-clippy");
let clippy_driver_path: PathBuf = PathBuf::from("target/debug/clippy-driver");
// crates we want to check:
let krates: Vec<KrateSource> = vec![KrateSource::new("regex", "1.4.2"), KrateSource::new("cargo", "0.49.0")];
@ -112,11 +132,6 @@ pub fn run() {
"target/debug/cargo-clippy binary not found! {}",
cargo_clippy_path.display()
);
assert!(
clippy_driver_path.is_file(),
"target/debug/clippy-driver binary not found! {}",
clippy_driver_path.display()
);
// download and extract the crates, then run clippy on them and collect clippys warnings
let clippy_lint_results: Vec<Vec<String>> = krates
@ -125,7 +140,8 @@ pub fn run() {
.map(|krate| krate.run_clippy_lints(&cargo_clippy_path))
.collect();
let results: Vec<String> = clippy_lint_results.into_iter().flatten().collect();
let all_warnings: Vec<String> = clippy_lint_results.into_iter().flatten().collect();
results.iter().for_each(|l| println!("{}", l));
// TODO: save these into a file
all_warnings.iter().for_each(|l| println!("{}", l));
}