kernel_api/sync/
rwlock.rs

1use core::fmt::Formatter;
2use core::mem;
3use core::panic::Location;
4use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering};
5use log::warn;
6
7/// A reader-writer lock
8pub type RwSpinlock<T: ?Sized> = lock_api::RwLock<RwCount, T>;
9
10/// RAII structure used to release the shared read access of a lock when dropped.
11pub type RwReadGuard<'a, T: ?Sized> = lock_api::RwLockReadGuard<'a, RwCount, T>;
12
13/// RAII structure used to release upgradable read access of a lock when dropped.
14pub type RwUpgradableReadGuard<'a, T: ?Sized> = lock_api::RwLockUpgradableReadGuard<'a, RwCount, T>;
15
16/// RAII structure used to release the exclusive write access of a lock when dropped.
17pub type RwWriteGuard<'a, T: ?Sized> = lock_api::RwLockWriteGuard<'a, RwCount, T>;
18
19#[doc(hidden)]
20pub struct RwCount(AtomicUsize, AtomicPtr<Location<'static>>);
21
22// FIXME: Deadlocks due to interrupts
23impl RwCount {
24    const WRITE_BIT_MASK: usize = 1<<(mem::size_of::<usize>() * 8 - 1);
25    const UPGRADEABLE_BIT_MASK: usize = 1<<(mem::size_of::<usize>() * 8 - 2);
26    const READ_COUNT_MASK: usize = !(Self::WRITE_BIT_MASK | Self::UPGRADEABLE_BIT_MASK);
27}
28
29impl core::fmt::Debug for RwCount {
30    fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result {
31        let mut d = f.debug_struct("RwCount");
32        let val = self.0.load(Ordering::Relaxed);
33        let write = val & Self::WRITE_BIT_MASK != 0;
34        let read = val & Self::READ_COUNT_MASK;
35        let upgradeable_reader = val & Self::UPGRADEABLE_BIT_MASK != 0;
36        d.field("write", &write);
37        d.field("read", &(read + if upgradeable_reader { 1 } else { 0 }));
38        d.finish()
39    }
40}
41
42impl RwCount {
43    fn lock_location(&self) -> Option<&'static Location<'static>> {
44        let location = self.1.load(Ordering::Relaxed);
45        unsafe { location.as_ref::<'static>() }
46    }
47}
48
49unsafe impl lock_api::RawRwLock for RwCount {
50    const INIT: Self = Self(AtomicUsize::new(0), AtomicPtr::new(core::ptr::null_mut()));
51    type GuardMarker = lock_api::GuardSend; // Doesn't (yet) touch interrupts so safe to send to other core
52
53    #[track_caller]
54    fn lock_shared(&self) {
55        while !self.try_lock_shared() {
56            core::hint::spin_loop();
57        }
58    }
59
60    #[track_caller]
61    fn try_lock_shared(&self) -> bool {
62        let mut old_value = self.0.load(Ordering::Relaxed);
63
64        loop {
65            let old_normal_count = old_value & Self::READ_COUNT_MASK;
66
67            if old_normal_count == Self::READ_COUNT_MASK { panic!("Reader count overflowed") }
68            if (old_value & Self::WRITE_BIT_MASK) != 0 { return false; }
69
70            let new_value = (old_normal_count + 1) | (old_value & Self::UPGRADEABLE_BIT_MASK);
71
72            match self.0.compare_exchange_weak(old_value, new_value, Ordering::Acquire, Ordering::Relaxed) {
73                Ok(_) => {
74                    self.1.store(Location::caller() as *const _ as *mut _, Ordering::Relaxed);
75                    return true
76                },
77                Err(new_old_value) => {
78                    warn!("locked at {:?}", self.lock_location());
79                    old_value = new_old_value
80                }
81            }
82        }
83    }
84
85    unsafe fn unlock_shared(&self) {
86        let mut old_value = self.0.load(Ordering::Relaxed);
87        loop {
88            let old_normal_count = old_value & !Self::UPGRADEABLE_BIT_MASK;
89
90            if cfg!(debug_assertions) && (old_value & Self::WRITE_BIT_MASK != 0) {
91                panic!("BUG: RwLock reader dropped while writer was active")
92            }
93            let new_value = (old_normal_count - 1) | (old_value & Self::UPGRADEABLE_BIT_MASK);
94            match self.0.compare_exchange_weak(old_value, new_value, Ordering::Release, Ordering::Relaxed) {
95                Ok(_) => return,
96                Err(new_old_value) => old_value = new_old_value
97            }
98        }
99    }
100
101    #[track_caller]
102    fn lock_exclusive(&self) {
103        while !self.try_lock_exclusive() {
104            core::hint::spin_loop();
105        }
106    }
107
108    #[track_caller]
109    fn try_lock_exclusive(&self) -> bool {
110        let res = self.0.compare_exchange_weak(0, Self::WRITE_BIT_MASK, Ordering::Acquire, Ordering::Relaxed)
111            .is_ok();
112        if !res {
113            warn!("locked at {:?}", self.lock_location());
114        } else {
115            self.1.store(Location::caller() as *const _ as *mut _, Ordering::Relaxed);
116        }
117        res
118    }
119
120    unsafe fn unlock_exclusive(&self) {
121        if cfg!(debug_assertions) {
122            self.0.compare_exchange(Self::WRITE_BIT_MASK, 0, Ordering::Release, Ordering::Relaxed)
123                .expect("BUG: RwLock writer dropped while readers were active");
124        } else {
125            self.0.store(0, Ordering::Release);
126        }
127    }
128}
129
130unsafe impl lock_api::RawRwLockDowngrade for RwCount {
131    unsafe fn downgrade(&self) {
132        if cfg!(debug_assertions) {
133            self.0.compare_exchange(Self::WRITE_BIT_MASK, 1, Ordering::SeqCst, Ordering::Relaxed)
134                .expect("BUG: RwLock writer downgraded while readers were active");
135        } else {
136            // No existing readers should exist therefore can unconditionally set read count to 1
137            self.0.store(1, Ordering::SeqCst); // FIXME: what order to use here
138        }
139    }
140}
141
142unsafe impl lock_api::RawRwLockUpgrade for RwCount {
143    fn lock_upgradable(&self) {
144        while !self.try_lock_upgradable() {
145            core::hint::spin_loop();
146        }
147    }
148
149    fn try_lock_upgradable(&self) -> bool {
150        let mut old_value = self.0.load(Ordering::Relaxed);
151
152        loop {
153            if (old_value & Self::WRITE_BIT_MASK) != 0 { return false; }
154            if (old_value & Self::UPGRADEABLE_BIT_MASK) != 0 { return false; }
155
156            let new_value = old_value | Self::UPGRADEABLE_BIT_MASK;
157
158            match self.0.compare_exchange_weak(old_value, new_value, Ordering::Acquire, Ordering::Relaxed) {
159                Ok(_) => return true,
160                Err(new_old_value) => old_value = new_old_value
161            }
162        }
163    }
164
165    unsafe fn unlock_upgradable(&self) {
166        let mut old_value = self.0.load(Ordering::Relaxed);
167        loop {
168            if cfg!(debug_assertions) && (old_value & Self::WRITE_BIT_MASK != 0) {
169                panic!("BUG: RwLock upgradable reader dropped while writer was active")
170            }
171            let new_value = old_value & !Self::UPGRADEABLE_BIT_MASK;
172            match self.0.compare_exchange_weak(old_value, new_value, Ordering::Release, Ordering::Relaxed) {
173                Ok(_) => return,
174                Err(new_old_value) => old_value = new_old_value
175            }
176        }
177    }
178
179    unsafe fn upgrade(&self) {
180        while !self.try_upgrade() {
181            core::hint::spin_loop();
182        }
183    }
184
185    unsafe fn try_upgrade(&self) -> bool {
186        self.0.compare_exchange_weak(Self::UPGRADEABLE_BIT_MASK, Self::WRITE_BIT_MASK, Ordering::Relaxed, Ordering::Relaxed)
187            .is_ok()
188    }
189}
190
191unsafe impl lock_api::RawRwLockUpgradeDowngrade for RwCount {
192    unsafe fn downgrade_upgradable(&self) {
193        let mut old_value = self.0.load(Ordering::Relaxed);
194        loop {
195            if cfg!(debug_assertions) && (old_value & Self::WRITE_BIT_MASK != 0) {
196                panic!("BUG: RwLock upgradable reader downgraded while writer was active")
197            }
198
199            let old_normal_count = old_value & Self::READ_COUNT_MASK;
200            if old_normal_count == Self::READ_COUNT_MASK { panic!("Reader count overflowed") }
201
202            let new_value = old_normal_count + 1;
203            match self.0.compare_exchange_weak(old_value, new_value, Ordering::Relaxed, Ordering::Relaxed) {
204                Ok(_) => return,
205                Err(new_old_value) => old_value = new_old_value
206            }
207        }
208    }
209
210    unsafe fn downgrade_to_upgradable(&self) {
211        if cfg!(debug_assertions) {
212            self.0.compare_exchange(Self::WRITE_BIT_MASK, Self::UPGRADEABLE_BIT_MASK, Ordering::SeqCst, Ordering::Relaxed)
213                .expect("BUG: RwLock writer downgraded while readers were active");
214        } else {
215            // No existing readers should exist therefore can unconditionally set upgradable bit
216            self.0.store(Self::UPGRADEABLE_BIT_MASK, Ordering::SeqCst); // FIXME: what ordering to use here?
217        }
218    }
219}
kernel_api/sync/rwlock.rs

kernel_api/sync/
rwlock.rs
