incr.comp.: Store DepNode colors in a dense array instead of a hashmap.
This commit is contained in:
parent
4d2d3fc5da
commit
d4b847504c
3 changed files with 112 additions and 47 deletions
|
@ -74,7 +74,7 @@ struct DepGraphData {
|
||||||
/// nodes and edges as well as all fingerprints of nodes that have them.
|
/// nodes and edges as well as all fingerprints of nodes that have them.
|
||||||
previous: PreviousDepGraph,
|
previous: PreviousDepGraph,
|
||||||
|
|
||||||
colors: RefCell<FxHashMap<DepNode, DepNodeColor>>,
|
colors: RefCell<DepNodeColorMap>,
|
||||||
|
|
||||||
/// When we load, there may be `.o` files, cached mir, or other such
|
/// When we load, there may be `.o` files, cached mir, or other such
|
||||||
/// things available to us. If we find that they are not dirty, we
|
/// things available to us. If we find that they are not dirty, we
|
||||||
|
@ -97,8 +97,10 @@ impl DepGraph {
|
||||||
// Pre-allocate the fingerprints array. We over-allocate a little so
|
// Pre-allocate the fingerprints array. We over-allocate a little so
|
||||||
// that we hopefully don't have to re-allocate during this compilation
|
// that we hopefully don't have to re-allocate during this compilation
|
||||||
// session.
|
// session.
|
||||||
|
let prev_graph_node_count = prev_graph.node_count();
|
||||||
|
|
||||||
let fingerprints = IndexVec::from_elem_n(Fingerprint::ZERO,
|
let fingerprints = IndexVec::from_elem_n(Fingerprint::ZERO,
|
||||||
(prev_graph.node_count() * 115) / 100);
|
(prev_graph_node_count * 115) / 100);
|
||||||
DepGraph {
|
DepGraph {
|
||||||
data: Some(Rc::new(DepGraphData {
|
data: Some(Rc::new(DepGraphData {
|
||||||
previous_work_products: RefCell::new(FxHashMap()),
|
previous_work_products: RefCell::new(FxHashMap()),
|
||||||
|
@ -106,7 +108,7 @@ impl DepGraph {
|
||||||
dep_node_debug: RefCell::new(FxHashMap()),
|
dep_node_debug: RefCell::new(FxHashMap()),
|
||||||
current: RefCell::new(CurrentDepGraph::new()),
|
current: RefCell::new(CurrentDepGraph::new()),
|
||||||
previous: prev_graph,
|
previous: prev_graph,
|
||||||
colors: RefCell::new(FxHashMap()),
|
colors: RefCell::new(DepNodeColorMap::new(prev_graph_node_count)),
|
||||||
loaded_from_cache: RefCell::new(FxHashMap()),
|
loaded_from_cache: RefCell::new(FxHashMap()),
|
||||||
})),
|
})),
|
||||||
fingerprints: Rc::new(RefCell::new(fingerprints)),
|
fingerprints: Rc::new(RefCell::new(fingerprints)),
|
||||||
|
@ -213,8 +215,6 @@ impl DepGraph {
|
||||||
R: HashStable<HCX>,
|
R: HashStable<HCX>,
|
||||||
{
|
{
|
||||||
if let Some(ref data) = self.data {
|
if let Some(ref data) = self.data {
|
||||||
debug_assert!(!data.colors.borrow().contains_key(&key));
|
|
||||||
|
|
||||||
push(&data.current, key);
|
push(&data.current, key);
|
||||||
if cfg!(debug_assertions) {
|
if cfg!(debug_assertions) {
|
||||||
profq_msg(ProfileQueriesMsg::TaskBegin(key.clone()))
|
profq_msg(ProfileQueriesMsg::TaskBegin(key.clone()))
|
||||||
|
@ -254,19 +254,21 @@ impl DepGraph {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Determine the color of the new DepNode.
|
// Determine the color of the new DepNode.
|
||||||
{
|
if let Some(prev_index) = data.previous.node_to_index_opt(&key) {
|
||||||
let prev_fingerprint = data.previous.fingerprint_of(&key);
|
let prev_fingerprint = data.previous.fingerprint_by_index(prev_index);
|
||||||
|
|
||||||
let color = if Some(current_fingerprint) == prev_fingerprint {
|
let color = if current_fingerprint == prev_fingerprint {
|
||||||
DepNodeColor::Green(dep_node_index)
|
DepNodeColor::Green(dep_node_index)
|
||||||
} else {
|
} else {
|
||||||
DepNodeColor::Red
|
DepNodeColor::Red
|
||||||
};
|
};
|
||||||
|
|
||||||
let old_value = data.colors.borrow_mut().insert(key, color);
|
let mut colors = data.colors.borrow_mut();
|
||||||
debug_assert!(old_value.is_none(),
|
debug_assert!(colors.get(prev_index).is_none(),
|
||||||
"DepGraph::with_task() - Duplicate DepNodeColor \
|
"DepGraph::with_task() - Duplicate DepNodeColor \
|
||||||
insertion for {:?}", key);
|
insertion for {:?}", key);
|
||||||
|
|
||||||
|
colors.insert(prev_index, color);
|
||||||
}
|
}
|
||||||
|
|
||||||
(result, dep_node_index)
|
(result, dep_node_index)
|
||||||
|
@ -281,9 +283,11 @@ impl DepGraph {
|
||||||
let mut fingerprints = self.fingerprints.borrow_mut();
|
let mut fingerprints = self.fingerprints.borrow_mut();
|
||||||
let dep_node_index = DepNodeIndex::new(fingerprints.len());
|
let dep_node_index = DepNodeIndex::new(fingerprints.len());
|
||||||
fingerprints.push(fingerprint);
|
fingerprints.push(fingerprint);
|
||||||
|
|
||||||
debug_assert!(fingerprints[dep_node_index] == fingerprint,
|
debug_assert!(fingerprints[dep_node_index] == fingerprint,
|
||||||
"DepGraph::with_task() - Assigned fingerprint to \
|
"DepGraph::with_task() - Assigned fingerprint to \
|
||||||
unexpected index for {:?}", key);
|
unexpected index for {:?}", key);
|
||||||
|
|
||||||
(result, dep_node_index)
|
(result, dep_node_index)
|
||||||
} else {
|
} else {
|
||||||
(task(cx, arg), DepNodeIndex::INVALID)
|
(task(cx, arg), DepNodeIndex::INVALID)
|
||||||
|
@ -356,6 +360,15 @@ impl DepGraph {
|
||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn dep_node_exists(&self, dep_node: &DepNode) -> bool {
|
||||||
|
if let Some(ref data) = self.data {
|
||||||
|
data.current.borrow_mut().node_to_node_index.contains_key(dep_node)
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint {
|
pub fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint {
|
||||||
match self.fingerprints.borrow().get(dep_node_index) {
|
match self.fingerprints.borrow().get(dep_node_index) {
|
||||||
|
@ -495,7 +508,17 @@ impl DepGraph {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn node_color(&self, dep_node: &DepNode) -> Option<DepNodeColor> {
|
pub fn node_color(&self, dep_node: &DepNode) -> Option<DepNodeColor> {
|
||||||
self.data.as_ref().and_then(|data| data.colors.borrow().get(dep_node).cloned())
|
if let Some(ref data) = self.data {
|
||||||
|
if let Some(prev_index) = data.previous.node_to_index_opt(dep_node) {
|
||||||
|
return data.colors.borrow().get(prev_index)
|
||||||
|
} else {
|
||||||
|
// This is a node that did not exist in the previous compilation
|
||||||
|
// session, so we consider it to be red.
|
||||||
|
return Some(DepNodeColor::Red)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn try_mark_green<'tcx>(&self,
|
pub fn try_mark_green<'tcx>(&self,
|
||||||
|
@ -505,7 +528,6 @@ impl DepGraph {
|
||||||
debug!("try_mark_green({:?}) - BEGIN", dep_node);
|
debug!("try_mark_green({:?}) - BEGIN", dep_node);
|
||||||
let data = self.data.as_ref().unwrap();
|
let data = self.data.as_ref().unwrap();
|
||||||
|
|
||||||
debug_assert!(!data.colors.borrow().contains_key(dep_node));
|
|
||||||
debug_assert!(!data.current.borrow().node_to_node_index.contains_key(dep_node));
|
debug_assert!(!data.current.borrow().node_to_node_index.contains_key(dep_node));
|
||||||
|
|
||||||
if dep_node.kind.is_input() {
|
if dep_node.kind.is_input() {
|
||||||
|
@ -535,19 +557,22 @@ impl DepGraph {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
debug_assert!(data.colors.borrow().get(prev_dep_node_index).is_none());
|
||||||
|
|
||||||
let mut current_deps = Vec::new();
|
let mut current_deps = Vec::new();
|
||||||
|
|
||||||
for &dep_dep_node_index in prev_deps {
|
for &dep_dep_node_index in prev_deps {
|
||||||
let dep_dep_node = &data.previous.index_to_node(dep_dep_node_index);
|
let dep_dep_node_color = data.colors.borrow().get(dep_dep_node_index);
|
||||||
|
|
||||||
let dep_dep_node_color = data.colors.borrow().get(dep_dep_node).cloned();
|
|
||||||
match dep_dep_node_color {
|
match dep_dep_node_color {
|
||||||
Some(DepNodeColor::Green(node_index)) => {
|
Some(DepNodeColor::Green(node_index)) => {
|
||||||
// This dependency has been marked as green before, we are
|
// This dependency has been marked as green before, we are
|
||||||
// still fine and can continue with checking the other
|
// still fine and can continue with checking the other
|
||||||
// dependencies.
|
// dependencies.
|
||||||
debug!("try_mark_green({:?}) --- found dependency {:?} to \
|
debug!("try_mark_green({:?}) --- found dependency {:?} to \
|
||||||
be immediately green", dep_node, dep_dep_node);
|
be immediately green",
|
||||||
|
dep_node,
|
||||||
|
data.previous.index_to_node(dep_dep_node_index));
|
||||||
current_deps.push(node_index);
|
current_deps.push(node_index);
|
||||||
}
|
}
|
||||||
Some(DepNodeColor::Red) => {
|
Some(DepNodeColor::Red) => {
|
||||||
|
@ -556,10 +581,14 @@ impl DepGraph {
|
||||||
// mark the DepNode as green and also don't need to bother
|
// mark the DepNode as green and also don't need to bother
|
||||||
// with checking any of the other dependencies.
|
// with checking any of the other dependencies.
|
||||||
debug!("try_mark_green({:?}) - END - dependency {:?} was \
|
debug!("try_mark_green({:?}) - END - dependency {:?} was \
|
||||||
immediately red", dep_node, dep_dep_node);
|
immediately red",
|
||||||
|
dep_node,
|
||||||
|
data.previous.index_to_node(dep_dep_node_index));
|
||||||
return None
|
return None
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
|
let dep_dep_node = &data.previous.index_to_node(dep_dep_node_index);
|
||||||
|
|
||||||
// We don't know the state of this dependency. If it isn't
|
// We don't know the state of this dependency. If it isn't
|
||||||
// an input node, let's try to mark it green recursively.
|
// an input node, let's try to mark it green recursively.
|
||||||
if !dep_dep_node.kind.is_input() {
|
if !dep_dep_node.kind.is_input() {
|
||||||
|
@ -601,10 +630,8 @@ impl DepGraph {
|
||||||
debug!("try_mark_green({:?}) --- trying to force \
|
debug!("try_mark_green({:?}) --- trying to force \
|
||||||
dependency {:?}", dep_node, dep_dep_node);
|
dependency {:?}", dep_node, dep_dep_node);
|
||||||
if ::ty::maps::force_from_dep_node(tcx, dep_dep_node) {
|
if ::ty::maps::force_from_dep_node(tcx, dep_dep_node) {
|
||||||
let dep_dep_node_color = data.colors
|
let dep_dep_node_color = data.colors.borrow().get(dep_dep_node_index);
|
||||||
.borrow()
|
|
||||||
.get(dep_dep_node)
|
|
||||||
.cloned();
|
|
||||||
match dep_dep_node_color {
|
match dep_dep_node_color {
|
||||||
Some(DepNodeColor::Green(node_index)) => {
|
Some(DepNodeColor::Green(node_index)) => {
|
||||||
debug!("try_mark_green({:?}) --- managed to \
|
debug!("try_mark_green({:?}) --- managed to \
|
||||||
|
@ -681,26 +708,21 @@ impl DepGraph {
|
||||||
}
|
}
|
||||||
|
|
||||||
// ... and finally storing a "Green" entry in the color map.
|
// ... and finally storing a "Green" entry in the color map.
|
||||||
let old_color = data.colors
|
let mut colors = data.colors.borrow_mut();
|
||||||
.borrow_mut()
|
debug_assert!(colors.get(prev_dep_node_index).is_none(),
|
||||||
.insert(*dep_node, DepNodeColor::Green(dep_node_index));
|
|
||||||
debug_assert!(old_color.is_none(),
|
|
||||||
"DepGraph::try_mark_green() - Duplicate DepNodeColor \
|
"DepGraph::try_mark_green() - Duplicate DepNodeColor \
|
||||||
insertion for {:?}", dep_node);
|
insertion for {:?}", dep_node);
|
||||||
|
|
||||||
|
colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index));
|
||||||
|
|
||||||
debug!("try_mark_green({:?}) - END - successfully marked as green", dep_node);
|
debug!("try_mark_green({:?}) - END - successfully marked as green", dep_node);
|
||||||
Some(dep_node_index)
|
Some(dep_node_index)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Used in various assertions
|
// Returns true if the given node has been marked as green during the
|
||||||
pub fn is_green(&self, dep_node_index: DepNodeIndex) -> bool {
|
// current compilation session. Used in various assertions
|
||||||
let dep_node = self.data.as_ref().unwrap().current.borrow().nodes[dep_node_index];
|
pub fn is_green(&self, dep_node: &DepNode) -> bool {
|
||||||
self.data.as_ref().unwrap().colors.borrow().get(&dep_node).map(|&color| {
|
self.node_color(dep_node).map(|c| c.is_green()).unwrap_or(false)
|
||||||
match color {
|
|
||||||
DepNodeColor::Red => false,
|
|
||||||
DepNodeColor::Green(_) => true,
|
|
||||||
}
|
|
||||||
}).unwrap_or(false)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// This method loads all on-disk cacheable query results into memory, so
|
// This method loads all on-disk cacheable query results into memory, so
|
||||||
|
@ -714,20 +736,25 @@ impl DepGraph {
|
||||||
pub fn exec_cache_promotions<'a, 'tcx>(&self, tcx: TyCtxt<'a, 'tcx, 'tcx>) {
|
pub fn exec_cache_promotions<'a, 'tcx>(&self, tcx: TyCtxt<'a, 'tcx, 'tcx>) {
|
||||||
let green_nodes: Vec<DepNode> = {
|
let green_nodes: Vec<DepNode> = {
|
||||||
let data = self.data.as_ref().unwrap();
|
let data = self.data.as_ref().unwrap();
|
||||||
data.colors.borrow().iter().filter_map(|(dep_node, color)| match color {
|
let colors = data.colors.borrow();
|
||||||
DepNodeColor::Green(_) => {
|
colors.values.indices().filter_map(|prev_index| {
|
||||||
if dep_node.cache_on_disk(tcx) {
|
match colors.get(prev_index) {
|
||||||
Some(*dep_node)
|
Some(DepNodeColor::Green(_)) => {
|
||||||
} else {
|
let dep_node = data.previous.index_to_node(prev_index);
|
||||||
|
if dep_node.cache_on_disk(tcx) {
|
||||||
|
Some(dep_node)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None |
|
||||||
|
Some(DepNodeColor::Red) => {
|
||||||
|
// We can skip red nodes because a node can only be marked
|
||||||
|
// as red if the query result was recomputed and thus is
|
||||||
|
// already in memory.
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
DepNodeColor::Red => {
|
|
||||||
// We can skip red nodes because a node can only be marked
|
|
||||||
// as red if the query result was recomputed and thus is
|
|
||||||
// already in memory.
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}).collect()
|
}).collect()
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -1052,3 +1079,36 @@ enum OpenTask {
|
||||||
node: DepNode,
|
node: DepNode,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A data structure that stores Option<DepNodeColor> values as a contiguous
|
||||||
|
// array, using one u32 per entry.
|
||||||
|
struct DepNodeColorMap {
|
||||||
|
values: IndexVec<SerializedDepNodeIndex, u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
const COMPRESSED_NONE: u32 = 0;
|
||||||
|
const COMPRESSED_RED: u32 = 1;
|
||||||
|
const COMPRESSED_FIRST_GREEN: u32 = 2;
|
||||||
|
|
||||||
|
impl DepNodeColorMap {
|
||||||
|
fn new(size: usize) -> DepNodeColorMap {
|
||||||
|
DepNodeColorMap {
|
||||||
|
values: IndexVec::from_elem_n(COMPRESSED_NONE, size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get(&self, index: SerializedDepNodeIndex) -> Option<DepNodeColor> {
|
||||||
|
match self.values[index] {
|
||||||
|
COMPRESSED_NONE => None,
|
||||||
|
COMPRESSED_RED => Some(DepNodeColor::Red),
|
||||||
|
value => Some(DepNodeColor::Green(DepNodeIndex(value - COMPRESSED_FIRST_GREEN)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert(&mut self, index: SerializedDepNodeIndex, color: DepNodeColor) {
|
||||||
|
self.values[index] = match color {
|
||||||
|
DepNodeColor::Red => COMPRESSED_RED,
|
||||||
|
DepNodeColor::Green(index) => index.0 + COMPRESSED_FIRST_GREEN,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -49,6 +49,11 @@ impl PreviousDepGraph {
|
||||||
self.index[dep_node]
|
self.index[dep_node]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option<SerializedDepNodeIndex> {
|
||||||
|
self.index.get(dep_node).cloned()
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn fingerprint_of(&self, dep_node: &DepNode) -> Option<Fingerprint> {
|
pub fn fingerprint_of(&self, dep_node: &DepNode) -> Option<Fingerprint> {
|
||||||
self.index
|
self.index
|
||||||
|
|
|
@ -147,7 +147,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
|
||||||
}
|
}
|
||||||
match self.dep_graph.try_mark_green(self.global_tcx(), &dep_node) {
|
match self.dep_graph.try_mark_green(self.global_tcx(), &dep_node) {
|
||||||
Some(dep_node_index) => {
|
Some(dep_node_index) => {
|
||||||
debug_assert!(self.dep_graph.is_green(dep_node_index));
|
debug_assert!(self.dep_graph.is_green(&dep_node));
|
||||||
self.dep_graph.read_index(dep_node_index);
|
self.dep_graph.read_index(dep_node_index);
|
||||||
Some(dep_node_index)
|
Some(dep_node_index)
|
||||||
}
|
}
|
||||||
|
@ -390,7 +390,7 @@ macro_rules! define_maps {
|
||||||
dep_node: &DepNode)
|
dep_node: &DepNode)
|
||||||
-> Result<$V, CycleError<'a, $tcx>>
|
-> Result<$V, CycleError<'a, $tcx>>
|
||||||
{
|
{
|
||||||
debug_assert!(tcx.dep_graph.is_green(dep_node_index));
|
debug_assert!(tcx.dep_graph.is_green(dep_node));
|
||||||
|
|
||||||
// First we try to load the result from the on-disk cache
|
// First we try to load the result from the on-disk cache
|
||||||
let result = if Self::cache_on_disk(key) &&
|
let result = if Self::cache_on_disk(key) &&
|
||||||
|
@ -478,7 +478,7 @@ macro_rules! define_maps {
|
||||||
span: Span,
|
span: Span,
|
||||||
dep_node: DepNode)
|
dep_node: DepNode)
|
||||||
-> Result<($V, DepNodeIndex), CycleError<'a, $tcx>> {
|
-> Result<($V, DepNodeIndex), CycleError<'a, $tcx>> {
|
||||||
debug_assert!(tcx.dep_graph.node_color(&dep_node).is_none());
|
debug_assert!(!tcx.dep_graph.dep_node_exists(&dep_node));
|
||||||
|
|
||||||
profq_msg!(tcx, ProfileQueriesMsg::ProviderBegin);
|
profq_msg!(tcx, ProfileQueriesMsg::ProviderBegin);
|
||||||
let res = tcx.cycle_check(span, Query::$name(key), || {
|
let res = tcx.cycle_check(span, Query::$name(key), || {
|
||||||
|
|
Loading…
Reference in a new issue