implement vfs events handling

This commit is contained in:
Aleksey Kladov 2018-12-18 16:38:05 +03:00
parent e69b05781f
commit a422d480a1
7 changed files with 349 additions and 43 deletions

81
Cargo.lock generated
View file

@ -752,6 +752,7 @@ dependencies = [
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
"relative-path 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tempfile 3.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"thread_worker 0.1.0",
"walkdir 2.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -778,6 +779,33 @@ dependencies = [
"winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rand"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_chacha 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_pcg 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_xorshift 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rand_chacha"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rand_core"
version = "0.2.2"
@ -791,6 +819,39 @@ name = "rand_core"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "rand_hc"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rand_isaac"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rand_pcg"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rand_xorshift"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rayon"
version = "1.0.3"
@ -1068,6 +1129,19 @@ dependencies = [
"remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tempfile"
version = "3.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)",
"remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tera"
version = "0.11.20"
@ -1431,8 +1505,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "53fa22a1994bd0f9372d7a816207d8a2677ad0325b073f5c5332760f0fb62b5c"
"checksum rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8356f47b32624fef5b3301c1be97e5944ecdd595409cc5da11d05f211db6cfbd"
"checksum rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e464cd887e869cddcae8792a4ee31d23c7edd516700695608f5b98c67ee0131c"
"checksum rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae9d223d52ae411a33cf7e54ec6034ec165df296ccd23533d671a28252b6f66a"
"checksum rand_chacha 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "771b009e3a508cb67e8823dda454aaa5368c7bc1c16829fb77d3e980440dd34a"
"checksum rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1961a422c4d189dfb50ffa9320bf1f2a9bd54ecb92792fb9477f99a1045f3372"
"checksum rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0905b6b7079ec73b314d4c748701f6931eb79fd97c668caa3f1899b22b32c6db"
"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
"checksum rand_pcg 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "086bd09a33c7044e56bb44d5bdde5a60e7f119a9e95b0775f545de759a32fe05"
"checksum rand_xorshift 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "effa3fcaa47e18db002bdde6060944b6d2f9cfd8db471c30e873448ad9187be3"
"checksum rayon 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "373814f27745b2686b350dd261bfd24576a6fb0e2c5919b3a2b6005f820b0473"
"checksum rayon-core 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b055d1e92aba6877574d8fe604a63c8b5df60f60e5982bf7ccbb1338ea527356"
"checksum redox_syscall 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)" = "a84bcd297b87a545980a2d25a0beb72a1f490c31f0a9fde52fca35bfbb1ceb70"
@ -1467,6 +1547,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum syn 0.15.23 (registry+https://github.com/rust-lang/crates.io-index)" = "9545a6a093a3f0bd59adb472700acc08cad3776f860f16a897dfce8c88721cbc"
"checksum synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015"
"checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
"checksum tempfile 3.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "7e91405c14320e5c79b3d148e1c86f40749a36e490642202a31689cb1a3452b2"
"checksum tera 0.11.20 (registry+https://github.com/rust-lang/crates.io-index)" = "4b505279e19d8f7d24b1a9dc58327c9c36174b1a2c7ebdeac70792d017cb64f3"
"checksum teraron 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0d89ad4617d1dec55331067fadaa041e813479e1779616f3d3ce9308bf46184e"
"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"

View file

@ -174,11 +174,11 @@ impl Server {
impl Drop for Server {
fn drop(&mut self) {
self.send_request::<Shutdown>(666, ());
let receiver = self.worker.take().unwrap().stop();
let receiver = self.worker.take().unwrap().shutdown();
while let Some(msg) = recv_timeout(&receiver) {
drop(msg);
}
self.watcher.take().unwrap().stop().unwrap();
self.watcher.take().unwrap().shutdown().unwrap();
}
}

View file

@ -12,3 +12,6 @@ crossbeam-channel = "0.2.4"
log = "0.4.6"
thread_worker = { path = "../thread_worker" }
[dev-dependencies]
tempfile = "3"

View file

@ -1,5 +1,4 @@
use std::{
hash::{Hash, Hasher},
marker::PhantomData,
ops::{Index, IndexMut},
};
@ -21,6 +20,12 @@ impl<ID: ArenaId, T> Arena<ID, T> {
self.data.push(value);
ID::from_u32(id)
}
pub fn iter<'a>(&'a self) -> impl Iterator<Item = (ID, &'a T)> {
self.data
.iter()
.enumerate()
.map(|(idx, value)| (ID::from_u32(idx as u32), value))
}
}
impl<ID: ArenaId, T> Default for Arena<ID, T> {

View file

@ -1,35 +1,26 @@
use std::{
fs,
path::{Path, PathBuf},
thread::JoinHandle,
};
use walkdir::{DirEntry, WalkDir};
use crossbeam_channel::{Sender, Receiver};
use thread_worker::{WorkerHandle};
use relative_path::RelativePathBuf;
use crate::VfsRoot;
pub(crate) enum Task {
ScanRoot {
root: VfsRoot,
path: PathBuf,
filter: Box<FnMut(&DirEntry) -> bool + Send>,
},
}
#[derive(Debug)]
pub(crate) struct FileEvent {
pub(crate) struct Task {
pub(crate) root: VfsRoot,
pub(crate) path: PathBuf,
pub(crate) kind: FileEventKind,
pub(crate) filter: Box<Fn(&DirEntry) -> bool + Send>,
}
#[derive(Debug)]
pub(crate) enum FileEventKind {
Add(String),
pub struct TaskResult {
pub(crate) root: VfsRoot,
pub(crate) files: Vec<(RelativePathBuf, String)>,
}
pub(crate) type Worker = thread_worker::Worker<Task, (PathBuf, Vec<FileEvent>)>;
pub(crate) type Worker = thread_worker::Worker<Task, TaskResult>;
pub(crate) fn start() -> (Worker, WorkerHandle) {
thread_worker::spawn("vfs", 128, |input_receiver, output_sender| {
@ -39,17 +30,17 @@ pub(crate) fn start() -> (Worker, WorkerHandle) {
})
}
fn handle_task(task: Task) -> (PathBuf, Vec<FileEvent>) {
let Task::ScanRoot { path, .. } = task;
fn handle_task(task: Task) -> TaskResult {
let Task { root, path, filter } = task;
log::debug!("loading {} ...", path.as_path().display());
let events = load_root(path.as_path());
let files = load_root(path.as_path(), &*filter);
log::debug!("... loaded {}", path.as_path().display());
(path, events)
TaskResult { root, files }
}
fn load_root(path: &Path) -> Vec<FileEvent> {
fn load_root(root: &Path, filter: &dyn Fn(&DirEntry) -> bool) -> Vec<(RelativePathBuf, String)> {
let mut res = Vec::new();
for entry in WalkDir::new(path) {
for entry in WalkDir::new(root).into_iter().filter_entry(filter) {
let entry = match entry {
Ok(entry) => entry,
Err(e) => {
@ -71,10 +62,8 @@ fn load_root(path: &Path) -> Vec<FileEvent> {
continue;
}
};
res.push(FileEvent {
path: path.to_owned(),
kind: FileEventKind::Add(text),
})
let path = RelativePathBuf::from_path(path.strip_prefix(root).unwrap()).unwrap();
res.push((path.to_owned(), text))
}
res
}

View file

@ -15,14 +15,19 @@ mod arena;
mod io;
use std::{
mem,
thread,
cmp::Reverse,
path::{Path, PathBuf},
ffi::OsStr,
sync::Arc,
fs,
};
use rustc_hash::{FxHashMap, FxHashSet};
use relative_path::RelativePathBuf;
use crossbeam_channel::Receiver;
use walkdir::DirEntry;
use thread_worker::{WorkerHandle};
use crate::{
@ -40,15 +45,25 @@ impl RootFilter {
fn new(root: PathBuf) -> RootFilter {
RootFilter {
root,
file_filter: rs_extension_filter,
file_filter: has_rs_extension,
}
}
fn can_contain(&self, path: &Path) -> bool {
(self.file_filter)(path) && path.starts_with(&self.root)
/// Check if this root can contain `path`. NB: even if this returns
/// true, the `path` might actually be conained in some nested root.
fn can_contain(&self, path: &Path) -> Option<RelativePathBuf> {
if !(self.file_filter)(path) {
return None;
}
if !(path.starts_with(&self.root)) {
return None;
}
let path = path.strip_prefix(&self.root).unwrap();
let path = RelativePathBuf::from_path(path).unwrap();
Some(path)
}
}
fn rs_extension_filter(p: &Path) -> bool {
fn has_rs_extension(p: &Path) -> bool {
p.extension() == Some(OsStr::new("rs"))
}
@ -82,10 +97,11 @@ struct VfsFileData {
text: Arc<String>,
}
struct Vfs {
pub struct Vfs {
roots: Arena<VfsRoot, RootFilter>,
files: Arena<VfsFile, VfsFileData>,
// pending_changes: Vec<PendingChange>,
root2files: FxHashMap<VfsRoot, FxHashSet<VfsFile>>,
pending_changes: Vec<VfsChange>,
worker: io::Worker,
worker_handle: WorkerHandle,
}
@ -97,33 +113,144 @@ impl Vfs {
let mut res = Vfs {
roots: Arena::default(),
files: Arena::default(),
root2files: FxHashMap::default(),
worker,
worker_handle,
pending_changes: Vec::new(),
};
// A hack to make nesting work.
roots.sort_by_key(|it| Reverse(it.as_os_str().len()));
for path in roots {
res.roots.alloc(RootFilter::new(path));
for (i, path) in roots.iter().enumerate() {
let root = res.roots.alloc(RootFilter::new(path.clone()));
let nested = roots[..i]
.iter()
.filter(|it| it.starts_with(path))
.map(|it| it.clone())
.collect::<Vec<_>>();
let filter = move |entry: &DirEntry| {
if entry.file_type().is_file() {
has_rs_extension(entry.path())
} else {
nested.iter().all(|it| it != entry.path())
}
};
let task = io::Task {
root,
path: path.clone(),
filter: Box::new(filter),
};
res.worker.inp.send(task);
}
res
}
pub fn add_file_overlay(&mut self, path: &Path, content: String) {}
pub fn task_receiver(&self) -> &Receiver<io::TaskResult> {
&self.worker.out
}
pub fn change_file_overlay(&mut self, path: &Path, new_content: String) {}
pub fn handle_task(&mut self, task: io::TaskResult) {
let mut files = Vec::new();
for (path, text) in task.files {
let text = Arc::new(text);
let file = self.add_file(task.root, path.clone(), Arc::clone(&text));
files.push((file, path, text));
}
let change = VfsChange::AddRoot {
root: task.root,
files,
};
self.pending_changes.push(change);
}
pub fn remove_file_overlay(&mut self, path: &Path) {}
pub fn add_file_overlay(&mut self, path: &Path, text: String) {
if let Some((root, path, file)) = self.find_root(path) {
let text = Arc::new(text);
let change = if let Some(file) = file {
self.change_file(file, Arc::clone(&text));
VfsChange::ChangeFile { file, text }
} else {
let file = self.add_file(root, path.clone(), Arc::clone(&text));
VfsChange::AddFile {
file,
text,
root,
path,
}
};
self.pending_changes.push(change);
}
}
pub fn change_file_overlay(&mut self, path: &Path, new_text: String) {
if let Some((_root, _path, file)) = self.find_root(path) {
let file = file.expect("can't change a file which wasn't added");
let text = Arc::new(new_text);
self.change_file(file, Arc::clone(&text));
let change = VfsChange::ChangeFile { file, text };
self.pending_changes.push(change);
}
}
pub fn remove_file_overlay(&mut self, path: &Path) {
if let Some((root, path, file)) = self.find_root(path) {
let file = file.expect("can't remove a file which wasn't added");
let full_path = path.to_path(&self.roots[root].root);
let change = if let Ok(text) = fs::read_to_string(&full_path) {
let text = Arc::new(text);
self.change_file(file, Arc::clone(&text));
VfsChange::ChangeFile { file, text }
} else {
self.remove_file(file);
VfsChange::RemoveFile { file }
};
self.pending_changes.push(change);
}
}
pub fn commit_changes(&mut self) -> Vec<VfsChange> {
unimplemented!()
mem::replace(&mut self.pending_changes, Vec::new())
}
pub fn shutdown(self) -> thread::Result<()> {
let _ = self.worker.shutdown();
self.worker_handle.shutdown()
}
fn add_file(&mut self, root: VfsRoot, path: RelativePathBuf, text: Arc<String>) -> VfsFile {
let data = VfsFileData { root, path, text };
let file = self.files.alloc(data);
self.root2files
.entry(root)
.or_insert_with(FxHashSet::default)
.insert(file);
file
}
fn change_file(&mut self, file: VfsFile, new_text: Arc<String>) {
self.files[file].text = new_text;
}
fn remove_file(&mut self, file: VfsFile) {
//FIXME: use arena with removal
self.files[file].text = Default::default();
self.files[file].path = Default::default();
let root = self.files[file].root;
let removed = self.root2files.get_mut(&root).unwrap().remove(&file);
assert!(removed);
}
fn find_root(&self, path: &Path) -> Option<(VfsRoot, RelativePathBuf, Option<VfsFile>)> {
let (root, path) = self
.roots
.iter()
.find_map(|(root, data)| data.can_contain(path).map(|it| (root, it)))?;
let file = self.root2files[&root]
.iter()
.map(|&it| it)
.find(|&file| self.files[file].path == path);
Some((root, path, file))
}
}
#[derive(Debug, Clone)]

101
crates/ra_vfs/tests/vfs.rs Normal file
View file

@ -0,0 +1,101 @@
use std::{
fs,
collections::HashSet,
};
use tempfile::tempdir;
use ra_vfs::{Vfs, VfsChange};
#[test]
fn test_vfs_works() -> std::io::Result<()> {
let files = [
("a/foo.rs", "hello"),
("a/bar.rs", "world"),
("a/b/baz.rs", "nested hello"),
];
let dir = tempdir()?;
for (path, text) in files.iter() {
let file_path = dir.path().join(path);
fs::create_dir_all(file_path.parent().unwrap())?;
fs::write(file_path, text)?
}
let a_root = dir.path().join("a");
let b_root = dir.path().join("a/b");
let mut vfs = Vfs::new(vec![a_root, b_root]);
for _ in 0..2 {
let task = vfs.task_receiver().recv().unwrap();
vfs.handle_task(task);
}
{
let files = vfs
.commit_changes()
.into_iter()
.flat_map(|change| {
let files = match change {
VfsChange::AddRoot { files, .. } => files,
_ => panic!("unexpected change"),
};
files.into_iter().map(|(_id, path, text)| {
let text: String = (&*text).clone();
(format!("{}", path.display()), text)
})
})
.collect::<HashSet<_>>();
let expected_files = [
("foo.rs", "hello"),
("bar.rs", "world"),
("baz.rs", "nested hello"),
]
.iter()
.map(|(path, text)| (path.to_string(), text.to_string()))
.collect::<HashSet<_>>();
assert_eq!(files, expected_files);
}
vfs.add_file_overlay(&dir.path().join("a/b/baz.rs"), "quux".to_string());
let change = vfs.commit_changes().pop().unwrap();
match change {
VfsChange::ChangeFile { text, .. } => assert_eq!(&*text, "quux"),
_ => panic!("unexpected change"),
}
vfs.change_file_overlay(&dir.path().join("a/b/baz.rs"), "m".to_string());
let change = vfs.commit_changes().pop().unwrap();
match change {
VfsChange::ChangeFile { text, .. } => assert_eq!(&*text, "m"),
_ => panic!("unexpected change"),
}
vfs.remove_file_overlay(&dir.path().join("a/b/baz.rs"));
let change = vfs.commit_changes().pop().unwrap();
match change {
VfsChange::ChangeFile { text, .. } => assert_eq!(&*text, "nested hello"),
_ => panic!("unexpected change"),
}
vfs.add_file_overlay(&dir.path().join("a/b/spam.rs"), "spam".to_string());
let change = vfs.commit_changes().pop().unwrap();
match change {
VfsChange::AddFile { text, path, .. } => {
assert_eq!(&*text, "spam");
assert_eq!(path, "spam.rs");
}
_ => panic!("unexpected change"),
}
vfs.remove_file_overlay(&dir.path().join("a/b/spam.rs"));
let change = vfs.commit_changes().pop().unwrap();
match change {
VfsChange::RemoveFile { .. } => (),
_ => panic!("unexpected change"),
}
vfs.shutdown().unwrap();
Ok(())
}