cache-align the shards to improve throughput (#303)

* cache-align the shards to improve throughput

* fmt

* fix ci

* fix features

.github/workflows/ci.yml

@@ -49,7 +49,7 @@ jobs:
           # --
           # install cross
           echo "installing cross"
-          cargo install cross
+          cargo install cross --locked --version 0.2.5
           # --
       - name: test
         run: cross test --target ${{ matrix.target }}

Cargo.lock

@@ -75,6 +75,7 @@ version = "5.5.3"
 dependencies = [
  "arbitrary",
  "cfg-if",
+ "crossbeam-utils",
  "hashbrown",
  "lock_api",
  "once_cell",

Cargo.toml

@@ -26,6 +26,7 @@ cfg-if = "1.0.0"
 rayon = { version = "1.7.0", optional = true }
 once_cell = "1.18.0"
 arbitrary = { version = "1.3.0", optional = true }
+crossbeam-utils = "0.8"
 
 [package.metadata.docs.rs]
 features = ["rayon", "raw-api", "serde"]

src/lib.rs

@@ -34,6 +34,7 @@ use core::fmt;
 use core::hash::{BuildHasher, Hash, Hasher};
 use core::iter::FromIterator;
 use core::ops::{BitAnd, BitOr, Shl, Shr, Sub};
+use crossbeam_utils::CachePadded;
 use iter::{Iter, IterMut, OwningIter};
 use mapref::entry::{Entry, OccupiedEntry, VacantEntry};
 use mapref::multiple::RefMulti;
@@ -87,7 +88,7 @@ fn ncb(shard_amount: usize) -> usize {
 /// This means that it is safe to ignore it across multiple threads.
 pub struct DashMap<K, V, S = RandomState> {
     shift: usize,
-    shards: Box<[RwLock<HashMap<K, V, S>>]>,
+    shards: Box<[CachePadded<RwLock<HashMap<K, V, S>>>]>,
     hasher: S,
 }
 
@@ -98,7 +99,7 @@ impl<K: Eq + Hash + Clone, V: Clone, S: Clone> Clone for DashMap<K, V, S> {
         for shard in self.shards.iter() {
             let shard = shard.read();
 
-            inner_shards.push(RwLock::new((*shard).clone()));
+            inner_shards.push(CachePadded::new(RwLock::new((*shard).clone())));
         }
 
         Self {
@@ -282,7 +283,12 @@ impl<'a, K: 'a + Eq + Hash, V: 'a, S: BuildHasher + Clone> DashMap<K, V, S> {
         let cps = capacity / shard_amount;
 
         let shards = (0..shard_amount)
-            .map(|_| RwLock::new(HashMap::with_capacity_and_hasher(cps, hasher.clone())))
+            .map(|_| {
+                CachePadded::new(RwLock::new(HashMap::with_capacity_and_hasher(
+                    cps,
+                    hasher.clone(),
+                )))
+            })
             .collect();
 
         Self {
@@ -317,7 +323,7 @@ impl<'a, K: 'a + Eq + Hash, V: 'a, S: BuildHasher + Clone> DashMap<K, V, S> {
             /// let map = DashMap::<(), ()>::new();
             /// println!("Amount of shards: {}", map.shards().len());
             /// ```
-            pub fn shards(&self) -> &[RwLock<HashMap<K, V, S>>] {
+            pub fn shards(&self) -> &[CachePadded<RwLock<HashMap<K, V, S>>>] {
                 &self.shards
             }
 
@@ -337,7 +343,7 @@ impl<'a, K: 'a + Eq + Hash, V: 'a, S: BuildHasher + Clone> DashMap<K, V, S> {
             /// map.shards_mut()[shard_ind].get_mut().insert(42, SharedValue::new("forty two"));
             /// assert_eq!(*map.get(&42).unwrap(), "forty two");
             /// ```
-            pub fn shards_mut(&mut self) -> &mut [RwLock<HashMap<K, V, S>>] {
+            pub fn shards_mut(&mut self) -> &mut [CachePadded<RwLock<HashMap<K, V, S>>>] {
                 &mut self.shards
             }
 
@@ -347,22 +353,22 @@ impl<'a, K: 'a + Eq + Hash, V: 'a, S: BuildHasher + Clone> DashMap<K, V, S> {
             /// Requires the `raw-api` feature to be enabled.
             ///
             /// See [`DashMap::shards()`] and [`DashMap::shards_mut()`] for more information.
-            pub fn into_shards(self) -> Box<[RwLock<HashMap<K, V, S>>]> {
+            pub fn into_shards(self) -> Box<[CachePadded<RwLock<HashMap<K, V, S>>>]> {
                 self.shards
             }
         } else {
             #[allow(dead_code)]
-            pub(crate) fn shards(&self) -> &[RwLock<HashMap<K, V, S>>] {
+            pub(crate) fn shards(&self) -> &[CachePadded<RwLock<HashMap<K, V, S>>>] {
                 &self.shards
             }
 
             #[allow(dead_code)]
-            pub(crate) fn shards_mut(&mut self) -> &mut [RwLock<HashMap<K, V, S>>] {
+            pub(crate) fn shards_mut(&mut self) -> &mut [CachePadded<RwLock<HashMap<K, V, S>>>] {
                 &mut self.shards
             }
 
             #[allow(dead_code)]
-            pub(crate) fn into_shards(self) -> Box<[RwLock<HashMap<K, V, S>>]> {
+            pub(crate) fn into_shards(self) -> Box<[CachePadded<RwLock<HashMap<K, V, S>>>]> {
                 self.shards
             }
         }

src/rayon/map.rs

@@ -3,6 +3,7 @@ use crate::mapref::multiple::{RefMulti, RefMutMulti};
 use crate::util;
 use crate::{DashMap, HashMap};
 use core::hash::{BuildHasher, Hash};
+use crossbeam_utils::CachePadded;
 use rayon::iter::plumbing::UnindexedConsumer;
 use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelExtend, ParallelIterator};
 use std::collections::hash_map::RandomState;
@@ -80,7 +81,7 @@ where
 }
 
 pub struct OwningIter<K, V, S = RandomState> {
-    pub(super) shards: Box<[RwLock<HashMap<K, V, S>>]>,
+    pub(super) shards: Box<[CachePadded<RwLock<HashMap<K, V, S>>>]>,
 }
 
 impl<K, V, S> ParallelIterator for OwningIter<K, V, S>
@@ -99,6 +100,7 @@ where
             .into_par_iter()
             .flat_map_iter(|shard| {
                 shard
+                    .into_inner()
                     .into_inner()
                     .into_iter()
                     .map(|(k, v)| (k, v.into_inner()))
@@ -125,7 +127,7 @@ where
 }
 
 pub struct Iter<'a, K, V, S = RandomState> {
-    pub(super) shards: &'a [RwLock<HashMap<K, V, S>>],
+    pub(super) shards: &'a [CachePadded<RwLock<HashMap<K, V, S>>>],
 }
 
 impl<'a, K, V, S> ParallelIterator for Iter<'a, K, V, S>
@@ -188,7 +190,7 @@ where
 }
 
 pub struct IterMut<'a, K, V, S = RandomState> {
-    shards: &'a [RwLock<HashMap<K, V, S>>],
+    shards: &'a [CachePadded<RwLock<HashMap<K, V, S>>>],
 }
 
 impl<'a, K, V, S> ParallelIterator for IterMut<'a, K, V, S>

src/read_only.rs

@@ -5,6 +5,7 @@ use cfg_if::cfg_if;
 use core::borrow::Borrow;
 use core::fmt;
 use core::hash::{BuildHasher, Hash};
+use crossbeam_utils::CachePadded;
 use std::collections::hash_map::RandomState;
 
 /// A read-only view into a `DashMap`. Allows to obtain raw references to the stored values.
@@ -139,12 +140,12 @@ impl<'a, K: 'a + Eq + Hash, V: 'a, S: BuildHasher + Clone> ReadOnlyView<K, V, S>
             /// let map = DashMap::<(), ()>::new().into_read_only();
             /// println!("Amount of shards: {}", map.shards().len());
             /// ```
-            pub fn shards(&self) -> &[RwLock<HashMap<K, V, S>>] {
+            pub fn shards(&self) -> &[CachePadded<RwLock<HashMap<K, V, S>>>] {
                 &self.map.shards
             }
         } else {
             #[allow(dead_code)]
-            pub(crate) fn shards(&self) -> &[RwLock<HashMap<K, V, S>>] {
+            pub(crate) fn shards(&self) -> &[CachePadded<RwLock<HashMap<K, V, S>>>] {
                 &self.map.shards
             }
         }

src/set.rs

@@ -10,6 +10,8 @@ use core::borrow::Borrow;
 use core::fmt;
 use core::hash::{BuildHasher, Hash};
 use core::iter::FromIterator;
+#[cfg(feature = "raw-api")]
+use crossbeam_utils::CachePadded;
 use std::collections::hash_map::RandomState;
 
 /// DashSet is a thin wrapper around [`DashMap`] using `()` as the value type. It uses
@@ -136,7 +138,7 @@ impl<'a, K: 'a + Eq + Hash, S: BuildHasher + Clone> DashSet<K, S> {
             /// let set = DashSet::<()>::new();
             /// println!("Amount of shards: {}", set.shards().len());
             /// ```
-            pub fn shards(&self) -> &[RwLock<HashMap<K, (), S>>] {
+            pub fn shards(&self) -> &[CachePadded<RwLock<HashMap<K, (), S>>>] {
                 self.inner.shards()
             }
         }