678: WIP: automatically collect garbage r=matklad a=matklad



Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>
This commit is contained in:
bors[bot] 2019-01-28 11:39:02 +00:00
commit 7abe1f422c
9 changed files with 152 additions and 31 deletions

57
Cargo.lock generated
View file

@ -380,6 +380,11 @@ name = "fnv"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "fs_extra"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "fsevent"
version = "0.2.17"
@ -551,6 +556,34 @@ name = "itoa"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "jemalloc-ctl"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"jemalloc-sys 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "jemalloc-sys"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cc 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
"fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "jemallocator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"jemalloc-sys 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "join_to_string"
version = "0.1.3"
@ -890,7 +923,7 @@ dependencies = [
"ra_syntax 0.1.0",
"relative-path 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"salsa 0.10.0-alpha5 (registry+https://github.com/rust-lang/crates.io-index)",
"salsa 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
"test_utils 0.1.0",
]
@ -920,6 +953,8 @@ dependencies = [
"fst 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"insta 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"jemalloc-ctl 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"jemallocator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"join_to_string 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1061,7 +1096,7 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -1090,7 +1125,7 @@ name = "rand_hc"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -1128,7 +1163,7 @@ name = "rand_xorshift"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -1266,7 +1301,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "salsa"
version = "0.10.0-alpha5"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"derive-new 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1275,13 +1310,13 @@ dependencies = [
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"salsa-macros 0.10.0-alpha5 (registry+https://github.com/rust-lang/crates.io-index)",
"salsa-macros 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "salsa-macros"
version = "0.10.0-alpha5"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1868,6 +1903,7 @@ dependencies = [
"checksum filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a2df5c1a8c4be27e7707789dc42ae65976e60b394afd293d1419ab915833e646"
"checksum flexi_logger 0.10.5 (registry+https://github.com/rust-lang/crates.io-index)" = "bbd731387787f54fa333fa426e173fe42ea3d1123636b2b27ad802025fc5d182"
"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
"checksum fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674"
"checksum fsevent 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "c4bbbf71584aeed076100b5665ac14e3d85eeb31fdbb45fbd41ef9a682b5ec05"
"checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874"
"checksum fst 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "db72126ca7dff566cdbbdd54af44668c544897d9d3862b198141f176f1238bdf"
@ -1887,6 +1923,9 @@ dependencies = [
"checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08"
"checksum itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b8467d9c1cebe26feb08c640139247fac215782d35371ade9a2136ed6085358"
"checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b"
"checksum jemalloc-ctl 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4e93b0f37e7d735c6b610176d5b1bde8e1621ff3f6f7ac23cdfa4e7f7d0111b5"
"checksum jemalloc-sys 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "bfc62c8e50e381768ce8ee0428ee53741929f7ebd73e4d83f669bcf7693e00ae"
"checksum jemallocator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "9f0cd42ac65f758063fea55126b0148b1ce0a6354ff78e07a4d6806bc65c4ab3"
"checksum join_to_string 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4dc7a5290e8c2606ce2be49f456d50f69173cb96d1541e4f66e34ac8b331a98f"
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
"checksum lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a374c89b9db55895453a74c1e38861d9deec0b01b405a82516e9d5de4820dea1"
@ -1950,8 +1989,8 @@ dependencies = [
"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
"checksum rusty-fork 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9591f190d2852720b679c21f66ad929f9f1d7bb09d1193c26167586029d8489c"
"checksum ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "eb9e9b8cde282a9fe6a42dd4681319bfb63f121b8a8ee9439c6f4107e58a46f7"
"checksum salsa 0.10.0-alpha5 (registry+https://github.com/rust-lang/crates.io-index)" = "8b5e2535d707dc5ced81106d3b71d806cfeef8a6e8a567472fde7ffd56b770dd"
"checksum salsa-macros 0.10.0-alpha5 (registry+https://github.com/rust-lang/crates.io-index)" = "e7c5da4c649f6d4fc1864fcd9a379b1f7c6d570b278559c84a6e15981c949cc6"
"checksum salsa 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cad0e2348e2f80725b2980914a08a00267136c3ecf720896d3f7f08eef51e08f"
"checksum salsa-macros 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7c79744109df21b80aef1367669b0a9e4985bc966e76bf0e9321b222ec0b9fbb"
"checksum same-file 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8f20c4be53a8a1ff4c1f1b2bd14570d2f634628709752f0702ecdd2b3f9a5267"
"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"

View file

@ -5,8 +5,8 @@ version = "0.1.0"
authors = ["Aleksey Kladov <aleksey.kladov@gmail.com>"]
[dependencies]
salsa = "0.10.0"
relative-path = "0.4.0"
salsa = "0.10.0-alpha5"
rustc-hash = "1.0"
parking_lot = "0.7.0"

View file

@ -14,6 +14,8 @@ fst = "0.3.1"
rustc-hash = "1.0"
parking_lot = "0.7.0"
unicase = "2.2.0"
jemallocator = "0.1.9"
jemalloc-ctl = "0.2.0"
ra_syntax = { path = "../ra_syntax" }
ra_ide_api_light = { path = "../ra_ide_api_light" }

View file

@ -1,4 +1,7 @@
use std::sync::Arc;
use std::{
sync::Arc,
time,
};
use ra_db::{
CheckCanceled, FileId, Canceled, SourceDatabase,
@ -17,6 +20,8 @@ use crate::{LineIndex, symbol_index::{self, SymbolsDatabase}};
pub(crate) struct RootDatabase {
runtime: salsa::Runtime<RootDatabase>,
interner: Arc<hir::HirInterner>,
pub(crate) last_gc: time::Instant,
pub(crate) last_gc_check: time::Instant,
}
impl salsa::Database for RootDatabase {
@ -33,6 +38,8 @@ impl Default for RootDatabase {
let mut db = RootDatabase {
runtime: salsa::Runtime::default(),
interner: Default::default(),
last_gc: time::Instant::now(),
last_gc_check: time::Instant::now(),
};
db.set_crate_graph(Default::default());
db.set_local_roots(Default::default());
@ -46,6 +53,8 @@ impl salsa::ParallelDatabase for RootDatabase {
salsa::Snapshot::new(RootDatabase {
runtime: self.runtime.snapshot(self),
interner: Arc::clone(&self.interner),
last_gc: self.last_gc.clone(),
last_gc_check: self.last_gc_check.clone(),
})
}
}

View file

@ -1,4 +1,7 @@
use std::sync::Arc;
use std::{
sync::Arc,
time,
};
use hir::{
self, Problem, source_binder
@ -19,12 +22,14 @@ use crate::{
CrateId, db, Diagnostic, FileId, FilePosition, FileRange, FileSystemEdit,
Query, RootChange, SourceChange, SourceFileEdit,
symbol_index::{FileSymbol, SymbolsDatabase},
status::syntax_tree_stats
};
const GC_COOLDOWN: time::Duration = time::Duration::from_millis(100);
impl db::RootDatabase {
pub(crate) fn apply_change(&mut self, change: AnalysisChange) {
log::info!("apply_change {:?}", change);
// self.gc_syntax_trees();
if !change.new_roots.is_empty() {
let mut local_roots = Vec::clone(&self.local_roots());
for (root_id, is_local) in change.new_roots {
@ -72,18 +77,36 @@ impl db::RootDatabase {
self.set_source_root(root_id, Arc::new(source_root));
}
/// Ideally, we should call this function from time to time to collect heavy
/// syntax trees. However, if we actually do that, everything is recomputed
/// for some reason. Needs investigation.
pub(crate) fn maybe_collect_garbage(&mut self) {
if self.last_gc_check.elapsed() > GC_COOLDOWN {
self.last_gc_check = time::Instant::now();
let retained_trees = syntax_tree_stats(self).retained;
if retained_trees > 100 {
log::info!(
"automatic garbadge collection, {} retained trees",
retained_trees
);
self.collect_garbage();
}
}
}
pub(crate) fn collect_garbage(&mut self) {
self.query(ra_db::ParseQuery)
.sweep(SweepStrategy::default().discard_values());
self.query(hir::db::HirParseQuery)
.sweep(SweepStrategy::default().discard_values());
self.query(hir::db::FileItemsQuery)
.sweep(SweepStrategy::default().discard_values());
self.query(hir::db::FileItemQuery)
.sweep(SweepStrategy::default().discard_values());
self.last_gc = time::Instant::now();
let sweep = SweepStrategy::default()
.discard_values()
.sweep_all_revisions();
self.query(ra_db::ParseQuery).sweep(sweep);
self.query(hir::db::HirParseQuery).sweep(sweep);
self.query(hir::db::FileItemsQuery).sweep(sweep);
self.query(hir::db::FileItemQuery).sweep(sweep);
self.query(hir::db::LowerModuleQuery).sweep(sweep);
self.query(hir::db::LowerModuleSourceMapQuery).sweep(sweep);
self.query(hir::db::BodySyntaxMappingQuery).sweep(sweep);
}
}

View file

@ -59,6 +59,11 @@ pub use ra_db::{
Canceled, CrateGraph, CrateId, FileId, FilePosition, FileRange, SourceRootId
};
// We use jemalloc mainly to get heap usage statistics, actual performance
// differnece is not measures.
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
pub type Cancelable<T> = Result<T, Canceled>;
#[derive(Default)]
@ -286,6 +291,10 @@ impl AnalysisHost {
self.db.apply_change(change)
}
pub fn maybe_collect_garbage(&mut self) {
self.db.maybe_collect_garbage();
}
pub fn collect_garbage(&mut self) {
self.db.collect_garbage();
}

View file

@ -15,9 +15,13 @@ use crate::{
symbol_index::{SymbolIndex, LibrarySymbolsQuery},
};
pub(crate) fn syntax_tree_stats(db: &RootDatabase) -> SyntaxTreeStats {
db.query(ParseQuery).entries::<SyntaxTreeStats>()
}
pub(crate) fn status(db: &RootDatabase) -> String {
let files_stats = db.query(FileTextQuery).entries::<FilesStats>();
let syntax_tree_stats = db.query(ParseQuery).entries::<SyntaxTreeStats>();
let syntax_tree_stats = syntax_tree_stats(db);
let symbols_stats = db
.query(LibrarySymbolsQuery)
.entries::<LibrarySymbolsStats>();
@ -26,8 +30,13 @@ pub(crate) fn status(db: &RootDatabase) -> String {
interner.len()
};
format!(
"{}\n{}\n{}\nn_defs {}\n",
files_stats, symbols_stats, syntax_tree_stats, n_defs
"{}\n{}\n{}\n{} defs\n\nmemory:\n{}\ngc {:?} seconds ago",
files_stats,
symbols_stats,
syntax_tree_stats,
n_defs,
MemoryStats::current(),
db.last_gc.elapsed().as_secs(),
)
}
@ -58,9 +67,9 @@ impl FromIterator<TableEntry<FileId, Arc<String>>> for FilesStats {
}
#[derive(Default)]
struct SyntaxTreeStats {
pub(crate) struct SyntaxTreeStats {
total: usize,
retained: usize,
pub(crate) retained: usize,
retained_size: Bytes,
}
@ -118,6 +127,31 @@ impl FromIterator<TableEntry<SourceRootId, Arc<SymbolIndex>>> for LibrarySymbols
}
}
struct MemoryStats {
allocated: Bytes,
resident: Bytes,
}
impl MemoryStats {
fn current() -> MemoryStats {
jemalloc_ctl::epoch().unwrap();
MemoryStats {
allocated: Bytes(jemalloc_ctl::stats::allocated().unwrap()),
resident: Bytes(jemalloc_ctl::stats::resident().unwrap()),
}
}
}
impl fmt::Display for MemoryStats {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(
fmt,
"{} allocated {} resident",
self.allocated, self.resident,
)
}
}
#[derive(Default)]
struct Bytes(usize);

View file

@ -172,6 +172,7 @@ fn main_loop_inner(
let (libdata_sender, libdata_receiver) = unbounded();
loop {
state.maybe_collect_garbage();
log::trace!("selecting");
let event = select! {
recv(msg_receiver) -> msg => match msg {
@ -207,7 +208,7 @@ fn main_loop_inner(
};
match req.cast::<req::CollectGarbage>() {
Ok((id, ())) => {
state.collect_garbadge();
state.collect_garbage();
let resp = RawResponse::ok::<req::CollectGarbage>(id, &());
msg_sender.send(RawMessage::Response(resp)).unwrap()
}

View file

@ -232,7 +232,11 @@ impl ServerWorldState {
}
}
pub fn collect_garbadge(&mut self) {
pub fn maybe_collect_garbage(&mut self) {
self.analysis_host.maybe_collect_garbage()
}
pub fn collect_garbage(&mut self) {
self.analysis_host.collect_garbage()
}
}