From a422d480a188a28c6b5e7862fbf07817eb2c7447 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 18 Dec 2018 16:38:05 +0300 Subject: [PATCH] implement vfs events handling --- Cargo.lock | 81 ++++++++++ .../tests/heavy_tests/support.rs | 4 +- crates/ra_vfs/Cargo.toml | 3 + crates/ra_vfs/src/arena.rs | 7 +- crates/ra_vfs/src/io.rs | 43 ++--- crates/ra_vfs/src/lib.rs | 153 ++++++++++++++++-- crates/ra_vfs/tests/vfs.rs | 101 ++++++++++++ 7 files changed, 349 insertions(+), 43 deletions(-) create mode 100644 crates/ra_vfs/tests/vfs.rs diff --git a/Cargo.lock b/Cargo.lock index c15955f4cd5..aac4d91b36a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -752,6 +752,7 @@ dependencies = [ "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "relative-path 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "thread_worker 0.1.0", "walkdir 2.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -778,6 +779,33 @@ dependencies = [ "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rand" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_pcg 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_xorshift 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_chacha" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rand_core" version = "0.2.2" @@ -791,6 +819,39 @@ name = "rand_core" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_pcg" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rayon" version = "1.0.3" @@ -1068,6 +1129,19 @@ dependencies = [ "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tempfile" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tera" version = "0.11.20" @@ -1431,8 +1505,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "53fa22a1994bd0f9372d7a816207d8a2677ad0325b073f5c5332760f0fb62b5c" "checksum rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8356f47b32624fef5b3301c1be97e5944ecdd595409cc5da11d05f211db6cfbd" "checksum rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e464cd887e869cddcae8792a4ee31d23c7edd516700695608f5b98c67ee0131c" +"checksum rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae9d223d52ae411a33cf7e54ec6034ec165df296ccd23533d671a28252b6f66a" +"checksum rand_chacha 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "771b009e3a508cb67e8823dda454aaa5368c7bc1c16829fb77d3e980440dd34a" "checksum rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1961a422c4d189dfb50ffa9320bf1f2a9bd54ecb92792fb9477f99a1045f3372" "checksum rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0905b6b7079ec73b314d4c748701f6931eb79fd97c668caa3f1899b22b32c6db" +"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +"checksum rand_pcg 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "086bd09a33c7044e56bb44d5bdde5a60e7f119a9e95b0775f545de759a32fe05" +"checksum rand_xorshift 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "effa3fcaa47e18db002bdde6060944b6d2f9cfd8db471c30e873448ad9187be3" "checksum rayon 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "373814f27745b2686b350dd261bfd24576a6fb0e2c5919b3a2b6005f820b0473" "checksum rayon-core 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b055d1e92aba6877574d8fe604a63c8b5df60f60e5982bf7ccbb1338ea527356" "checksum redox_syscall 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)" = "a84bcd297b87a545980a2d25a0beb72a1f490c31f0a9fde52fca35bfbb1ceb70" @@ -1467,6 +1547,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum syn 0.15.23 (registry+https://github.com/rust-lang/crates.io-index)" = "9545a6a093a3f0bd59adb472700acc08cad3776f860f16a897dfce8c88721cbc" "checksum synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +"checksum tempfile 3.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "7e91405c14320e5c79b3d148e1c86f40749a36e490642202a31689cb1a3452b2" "checksum tera 0.11.20 (registry+https://github.com/rust-lang/crates.io-index)" = "4b505279e19d8f7d24b1a9dc58327c9c36174b1a2c7ebdeac70792d017cb64f3" "checksum teraron 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0d89ad4617d1dec55331067fadaa041e813479e1779616f3d3ce9308bf46184e" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" diff --git a/crates/ra_lsp_server/tests/heavy_tests/support.rs b/crates/ra_lsp_server/tests/heavy_tests/support.rs index 07a878a2648..c14d287cac2 100644 --- a/crates/ra_lsp_server/tests/heavy_tests/support.rs +++ b/crates/ra_lsp_server/tests/heavy_tests/support.rs @@ -174,11 +174,11 @@ impl Server { impl Drop for Server { fn drop(&mut self) { self.send_request::(666, ()); - let receiver = self.worker.take().unwrap().stop(); + let receiver = self.worker.take().unwrap().shutdown(); while let Some(msg) = recv_timeout(&receiver) { drop(msg); } - self.watcher.take().unwrap().stop().unwrap(); + self.watcher.take().unwrap().shutdown().unwrap(); } } diff --git a/crates/ra_vfs/Cargo.toml b/crates/ra_vfs/Cargo.toml index 9ce619a777d..ccea8a866fb 100644 --- a/crates/ra_vfs/Cargo.toml +++ b/crates/ra_vfs/Cargo.toml @@ -12,3 +12,6 @@ crossbeam-channel = "0.2.4" log = "0.4.6" thread_worker = { path = "../thread_worker" } + +[dev-dependencies] +tempfile = "3" diff --git a/crates/ra_vfs/src/arena.rs b/crates/ra_vfs/src/arena.rs index d6fad753bdb..6b42ae26d22 100644 --- a/crates/ra_vfs/src/arena.rs +++ b/crates/ra_vfs/src/arena.rs @@ -1,5 +1,4 @@ use std::{ - hash::{Hash, Hasher}, marker::PhantomData, ops::{Index, IndexMut}, }; @@ -21,6 +20,12 @@ impl Arena { self.data.push(value); ID::from_u32(id) } + pub fn iter<'a>(&'a self) -> impl Iterator { + self.data + .iter() + .enumerate() + .map(|(idx, value)| (ID::from_u32(idx as u32), value)) + } } impl Default for Arena { diff --git a/crates/ra_vfs/src/io.rs b/crates/ra_vfs/src/io.rs index c467605835b..178c9beff43 100644 --- a/crates/ra_vfs/src/io.rs +++ b/crates/ra_vfs/src/io.rs @@ -1,35 +1,26 @@ use std::{ fs, path::{Path, PathBuf}, - thread::JoinHandle, }; use walkdir::{DirEntry, WalkDir}; -use crossbeam_channel::{Sender, Receiver}; use thread_worker::{WorkerHandle}; +use relative_path::RelativePathBuf; use crate::VfsRoot; -pub(crate) enum Task { - ScanRoot { - root: VfsRoot, - path: PathBuf, - filter: Box bool + Send>, - }, -} - -#[derive(Debug)] -pub(crate) struct FileEvent { +pub(crate) struct Task { + pub(crate) root: VfsRoot, pub(crate) path: PathBuf, - pub(crate) kind: FileEventKind, + pub(crate) filter: Box bool + Send>, } -#[derive(Debug)] -pub(crate) enum FileEventKind { - Add(String), +pub struct TaskResult { + pub(crate) root: VfsRoot, + pub(crate) files: Vec<(RelativePathBuf, String)>, } -pub(crate) type Worker = thread_worker::Worker)>; +pub(crate) type Worker = thread_worker::Worker; pub(crate) fn start() -> (Worker, WorkerHandle) { thread_worker::spawn("vfs", 128, |input_receiver, output_sender| { @@ -39,17 +30,17 @@ pub(crate) fn start() -> (Worker, WorkerHandle) { }) } -fn handle_task(task: Task) -> (PathBuf, Vec) { - let Task::ScanRoot { path, .. } = task; +fn handle_task(task: Task) -> TaskResult { + let Task { root, path, filter } = task; log::debug!("loading {} ...", path.as_path().display()); - let events = load_root(path.as_path()); + let files = load_root(path.as_path(), &*filter); log::debug!("... loaded {}", path.as_path().display()); - (path, events) + TaskResult { root, files } } -fn load_root(path: &Path) -> Vec { +fn load_root(root: &Path, filter: &dyn Fn(&DirEntry) -> bool) -> Vec<(RelativePathBuf, String)> { let mut res = Vec::new(); - for entry in WalkDir::new(path) { + for entry in WalkDir::new(root).into_iter().filter_entry(filter) { let entry = match entry { Ok(entry) => entry, Err(e) => { @@ -71,10 +62,8 @@ fn load_root(path: &Path) -> Vec { continue; } }; - res.push(FileEvent { - path: path.to_owned(), - kind: FileEventKind::Add(text), - }) + let path = RelativePathBuf::from_path(path.strip_prefix(root).unwrap()).unwrap(); + res.push((path.to_owned(), text)) } res } diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index 8ce6b6ee0a7..792f722a7f9 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -15,14 +15,19 @@ mod arena; mod io; use std::{ + mem, thread, cmp::Reverse, path::{Path, PathBuf}, ffi::OsStr, sync::Arc, + fs, }; +use rustc_hash::{FxHashMap, FxHashSet}; use relative_path::RelativePathBuf; +use crossbeam_channel::Receiver; +use walkdir::DirEntry; use thread_worker::{WorkerHandle}; use crate::{ @@ -40,15 +45,25 @@ impl RootFilter { fn new(root: PathBuf) -> RootFilter { RootFilter { root, - file_filter: rs_extension_filter, + file_filter: has_rs_extension, } } - fn can_contain(&self, path: &Path) -> bool { - (self.file_filter)(path) && path.starts_with(&self.root) + /// Check if this root can contain `path`. NB: even if this returns + /// true, the `path` might actually be conained in some nested root. + fn can_contain(&self, path: &Path) -> Option { + if !(self.file_filter)(path) { + return None; + } + if !(path.starts_with(&self.root)) { + return None; + } + let path = path.strip_prefix(&self.root).unwrap(); + let path = RelativePathBuf::from_path(path).unwrap(); + Some(path) } } -fn rs_extension_filter(p: &Path) -> bool { +fn has_rs_extension(p: &Path) -> bool { p.extension() == Some(OsStr::new("rs")) } @@ -82,10 +97,11 @@ struct VfsFileData { text: Arc, } -struct Vfs { +pub struct Vfs { roots: Arena, files: Arena, - // pending_changes: Vec, + root2files: FxHashMap>, + pending_changes: Vec, worker: io::Worker, worker_handle: WorkerHandle, } @@ -97,33 +113,144 @@ impl Vfs { let mut res = Vfs { roots: Arena::default(), files: Arena::default(), + root2files: FxHashMap::default(), worker, worker_handle, + pending_changes: Vec::new(), }; // A hack to make nesting work. roots.sort_by_key(|it| Reverse(it.as_os_str().len())); - - for path in roots { - res.roots.alloc(RootFilter::new(path)); + for (i, path) in roots.iter().enumerate() { + let root = res.roots.alloc(RootFilter::new(path.clone())); + let nested = roots[..i] + .iter() + .filter(|it| it.starts_with(path)) + .map(|it| it.clone()) + .collect::>(); + let filter = move |entry: &DirEntry| { + if entry.file_type().is_file() { + has_rs_extension(entry.path()) + } else { + nested.iter().all(|it| it != entry.path()) + } + }; + let task = io::Task { + root, + path: path.clone(), + filter: Box::new(filter), + }; + res.worker.inp.send(task); } res } - pub fn add_file_overlay(&mut self, path: &Path, content: String) {} + pub fn task_receiver(&self) -> &Receiver { + &self.worker.out + } - pub fn change_file_overlay(&mut self, path: &Path, new_content: String) {} + pub fn handle_task(&mut self, task: io::TaskResult) { + let mut files = Vec::new(); + for (path, text) in task.files { + let text = Arc::new(text); + let file = self.add_file(task.root, path.clone(), Arc::clone(&text)); + files.push((file, path, text)); + } + let change = VfsChange::AddRoot { + root: task.root, + files, + }; + self.pending_changes.push(change); + } - pub fn remove_file_overlay(&mut self, path: &Path) {} + pub fn add_file_overlay(&mut self, path: &Path, text: String) { + if let Some((root, path, file)) = self.find_root(path) { + let text = Arc::new(text); + let change = if let Some(file) = file { + self.change_file(file, Arc::clone(&text)); + VfsChange::ChangeFile { file, text } + } else { + let file = self.add_file(root, path.clone(), Arc::clone(&text)); + VfsChange::AddFile { + file, + text, + root, + path, + } + }; + self.pending_changes.push(change); + } + } + + pub fn change_file_overlay(&mut self, path: &Path, new_text: String) { + if let Some((_root, _path, file)) = self.find_root(path) { + let file = file.expect("can't change a file which wasn't added"); + let text = Arc::new(new_text); + self.change_file(file, Arc::clone(&text)); + let change = VfsChange::ChangeFile { file, text }; + self.pending_changes.push(change); + } + } + + pub fn remove_file_overlay(&mut self, path: &Path) { + if let Some((root, path, file)) = self.find_root(path) { + let file = file.expect("can't remove a file which wasn't added"); + let full_path = path.to_path(&self.roots[root].root); + let change = if let Ok(text) = fs::read_to_string(&full_path) { + let text = Arc::new(text); + self.change_file(file, Arc::clone(&text)); + VfsChange::ChangeFile { file, text } + } else { + self.remove_file(file); + VfsChange::RemoveFile { file } + }; + self.pending_changes.push(change); + } + } pub fn commit_changes(&mut self) -> Vec { - unimplemented!() + mem::replace(&mut self.pending_changes, Vec::new()) } pub fn shutdown(self) -> thread::Result<()> { let _ = self.worker.shutdown(); self.worker_handle.shutdown() } + + fn add_file(&mut self, root: VfsRoot, path: RelativePathBuf, text: Arc) -> VfsFile { + let data = VfsFileData { root, path, text }; + let file = self.files.alloc(data); + self.root2files + .entry(root) + .or_insert_with(FxHashSet::default) + .insert(file); + file + } + + fn change_file(&mut self, file: VfsFile, new_text: Arc) { + self.files[file].text = new_text; + } + + fn remove_file(&mut self, file: VfsFile) { + //FIXME: use arena with removal + self.files[file].text = Default::default(); + self.files[file].path = Default::default(); + let root = self.files[file].root; + let removed = self.root2files.get_mut(&root).unwrap().remove(&file); + assert!(removed); + } + + fn find_root(&self, path: &Path) -> Option<(VfsRoot, RelativePathBuf, Option)> { + let (root, path) = self + .roots + .iter() + .find_map(|(root, data)| data.can_contain(path).map(|it| (root, it)))?; + let file = self.root2files[&root] + .iter() + .map(|&it| it) + .find(|&file| self.files[file].path == path); + Some((root, path, file)) + } } #[derive(Debug, Clone)] diff --git a/crates/ra_vfs/tests/vfs.rs b/crates/ra_vfs/tests/vfs.rs new file mode 100644 index 00000000000..4f44215c8b1 --- /dev/null +++ b/crates/ra_vfs/tests/vfs.rs @@ -0,0 +1,101 @@ +use std::{ + fs, + collections::HashSet, +}; + +use tempfile::tempdir; + +use ra_vfs::{Vfs, VfsChange}; + +#[test] +fn test_vfs_works() -> std::io::Result<()> { + let files = [ + ("a/foo.rs", "hello"), + ("a/bar.rs", "world"), + ("a/b/baz.rs", "nested hello"), + ]; + + let dir = tempdir()?; + for (path, text) in files.iter() { + let file_path = dir.path().join(path); + fs::create_dir_all(file_path.parent().unwrap())?; + fs::write(file_path, text)? + } + + let a_root = dir.path().join("a"); + let b_root = dir.path().join("a/b"); + + let mut vfs = Vfs::new(vec![a_root, b_root]); + for _ in 0..2 { + let task = vfs.task_receiver().recv().unwrap(); + vfs.handle_task(task); + } + { + let files = vfs + .commit_changes() + .into_iter() + .flat_map(|change| { + let files = match change { + VfsChange::AddRoot { files, .. } => files, + _ => panic!("unexpected change"), + }; + files.into_iter().map(|(_id, path, text)| { + let text: String = (&*text).clone(); + (format!("{}", path.display()), text) + }) + }) + .collect::>(); + + let expected_files = [ + ("foo.rs", "hello"), + ("bar.rs", "world"), + ("baz.rs", "nested hello"), + ] + .iter() + .map(|(path, text)| (path.to_string(), text.to_string())) + .collect::>(); + + assert_eq!(files, expected_files); + } + + vfs.add_file_overlay(&dir.path().join("a/b/baz.rs"), "quux".to_string()); + let change = vfs.commit_changes().pop().unwrap(); + match change { + VfsChange::ChangeFile { text, .. } => assert_eq!(&*text, "quux"), + _ => panic!("unexpected change"), + } + + vfs.change_file_overlay(&dir.path().join("a/b/baz.rs"), "m".to_string()); + let change = vfs.commit_changes().pop().unwrap(); + match change { + VfsChange::ChangeFile { text, .. } => assert_eq!(&*text, "m"), + _ => panic!("unexpected change"), + } + + vfs.remove_file_overlay(&dir.path().join("a/b/baz.rs")); + let change = vfs.commit_changes().pop().unwrap(); + match change { + VfsChange::ChangeFile { text, .. } => assert_eq!(&*text, "nested hello"), + _ => panic!("unexpected change"), + } + + vfs.add_file_overlay(&dir.path().join("a/b/spam.rs"), "spam".to_string()); + let change = vfs.commit_changes().pop().unwrap(); + match change { + VfsChange::AddFile { text, path, .. } => { + assert_eq!(&*text, "spam"); + assert_eq!(path, "spam.rs"); + } + _ => panic!("unexpected change"), + } + + vfs.remove_file_overlay(&dir.path().join("a/b/spam.rs")); + let change = vfs.commit_changes().pop().unwrap(); + match change { + VfsChange::RemoveFile { .. } => (), + _ => panic!("unexpected change"), + } + + vfs.shutdown().unwrap(); + Ok(()) +}