/build/source/nativelink-util/src/adaptive_permits.rs
Line | Count | Source |
1 | | // Copyright 2026 Trace Machina, Inc. All rights reserved. |
2 | | // |
3 | | // Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // See LICENSE file for details |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | use core::sync::atomic::{AtomicUsize, Ordering}; |
16 | | use core::time::Duration; |
17 | | use std::sync::Arc; |
18 | | |
19 | | use parking_lot::Mutex; |
20 | | use tokio::sync::{OwnedSemaphorePermit, Semaphore}; |
21 | | use tokio::time::Instant; |
22 | | use tracing::{debug, info}; |
23 | | |
24 | | use crate::metrics::{record_object_store_read_limit, record_object_store_read_wait}; |
25 | | |
26 | | /// A throttle within this long of the last cut is the same event seen by |
27 | | /// another request in flight, not a second signal. |
28 | | const CUT_WINDOW: Duration = Duration::from_millis(200); |
29 | | |
30 | | /// A pool of permits in front of a shared resource whose capacity is not |
31 | | /// known, sized the way TCP sizes its window: a rejection halves the limit, |
32 | | /// a run of successes raises it by one. Requests wait for a permit inside |
33 | | /// the process instead of being sent and refused, and many processes |
34 | | /// behind one limit converge on a share of it. |
35 | | /// |
36 | | /// Built for reads against a cloud object store that answers 503 or 429 |
37 | | /// when an account is over its limit; the store calls [`record_throttle`] |
38 | | /// on such a response and [`record_success`] on a completed request. |
39 | | /// |
40 | | /// [`record_throttle`]: Self::record_throttle |
41 | | /// [`record_success`]: Self::record_success |
42 | | #[derive(Debug)] |
43 | | pub struct AdaptivePermits { |
44 | | semaphore: Arc<Semaphore>, |
45 | | /// The current limit, between `min` and `max`. |
46 | | limit: AtomicUsize, |
47 | | min: usize, |
48 | | max: usize, |
49 | | /// Permits a cut could not take back because they were out; each is |
50 | | /// forgotten as it is released instead of returned. |
51 | | owed: AtomicUsize, |
52 | | /// Successes since the limit last moved. |
53 | | streak: AtomicUsize, |
54 | | /// Successes in a row needed to add a permit. Zero fixes the limit. |
55 | | grow_after: usize, |
56 | | last_cut: Mutex<Option<Instant>>, |
57 | | /// Who the pool belongs to, the `store.object.provider` label. |
58 | | name: &'static str, |
59 | | } |
60 | | |
61 | | /// One permit from an [`AdaptivePermits`] pool, held for the length of a |
62 | | /// request. |
63 | | #[derive(Debug)] |
64 | | pub struct AdaptivePermit { |
65 | | pool: Arc<AdaptivePermits>, |
66 | | inner: Option<OwnedSemaphorePermit>, |
67 | | } |
68 | | |
69 | | impl AdaptivePermits { |
70 | | /// A pool that starts at `initial` permits and moves between `min` and |
71 | | /// `max` as [`record_throttle`] and [`record_success`] are called; with |
72 | | /// `grow_after` zero the limit never grows and only falls. `initial` is |
73 | | /// clamped into `[min, max]`. |
74 | | /// |
75 | | /// [`record_throttle`]: Self::record_throttle |
76 | | /// [`record_success`]: Self::record_success |
77 | 19 | pub fn new( |
78 | 19 | name: &'static str, |
79 | 19 | initial: usize, |
80 | 19 | min: usize, |
81 | 19 | max: usize, |
82 | 19 | grow_after: usize, |
83 | 19 | ) -> Arc<Self> { |
84 | 19 | let min = min.max(1); |
85 | 19 | let max = max.max(min); |
86 | 19 | let initial = initial.clamp(min, max); |
87 | 19 | record_object_store_read_limit(name, initial); |
88 | 19 | Arc::new(Self { |
89 | 19 | semaphore: Arc::new(Semaphore::new(initial)), |
90 | 19 | limit: AtomicUsize::new(initial), |
91 | 19 | min, |
92 | 19 | max, |
93 | 19 | owed: AtomicUsize::new(0), |
94 | 19 | streak: AtomicUsize::new(0), |
95 | 19 | grow_after, |
96 | 19 | last_cut: Mutex::new(None), |
97 | 19 | name, |
98 | 19 | }) |
99 | 19 | } |
100 | | |
101 | | /// Waits for a permit. The permit is returned to the pool when dropped, |
102 | | /// unless the limit was cut while it was out. |
103 | 44 | pub async fn acquire(self: &Arc<Self>) -> AdaptivePermit { |
104 | 44 | if self.semaphore.available_permits() == 0 { |
105 | 7 | record_object_store_read_wait(self.name); |
106 | 37 | } |
107 | 44 | let inner = Arc::clone(&self.semaphore) |
108 | 44 | .acquire_owned() |
109 | 44 | .await |
110 | 44 | .expect("AdaptivePermits semaphore is never closed"); |
111 | 44 | AdaptivePermit { |
112 | 44 | pool: Arc::clone(self), |
113 | 44 | inner: Some(inner), |
114 | 44 | } |
115 | 44 | } |
116 | | |
117 | | /// Settles one owed permit if any is owed. True when it did. |
118 | 49 | fn take_owed(&self) -> bool { |
119 | 49 | let mut owed = self.owed.load(Ordering::Acquire); |
120 | | loop { |
121 | 49 | if owed == 0 { |
122 | 43 | return false; |
123 | 6 | } |
124 | 6 | match self.owed.compare_exchange_weak( |
125 | 6 | owed, |
126 | 6 | owed - 1, |
127 | 6 | Ordering::AcqRel, |
128 | 6 | Ordering::Acquire, |
129 | 6 | ) { |
130 | 6 | Ok(_) => return true, |
131 | 0 | Err(current) => owed = current, |
132 | | } |
133 | | } |
134 | 49 | } |
135 | | |
136 | | /// The limit as it stands. |
137 | 22 | pub fn limit(&self) -> usize { |
138 | 22 | self.limit.load(Ordering::Acquire) |
139 | 22 | } |
140 | | |
141 | | /// Permits that can be taken right now. |
142 | 9 | pub fn available(&self) -> usize { |
143 | 9 | self.semaphore.available_permits() |
144 | 9 | } |
145 | | |
146 | | /// A request went through. After `grow_after` of these in a row the |
147 | | /// limit rises by one, up to `max`. |
148 | 264 | pub fn record_success(&self) { |
149 | 264 | if self.grow_after == 0 { |
150 | 209 | return; |
151 | 55 | } |
152 | 55 | let streak = self.streak.fetch_add(1, Ordering::AcqRel) + 1; |
153 | 55 | if streak < self.grow_after { |
154 | 41 | return; |
155 | 14 | } |
156 | 14 | self.streak.store(0, Ordering::Release); |
157 | 14 | let _guard = self.last_cut.lock(); |
158 | 14 | let limit = self.limit.load(Ordering::Acquire); |
159 | 14 | if limit >= self.max { |
160 | 9 | return; |
161 | 5 | } |
162 | | // A permit still owed from a cut is cancelled instead of a new one |
163 | | // being added, so the pool never holds more than `limit`. |
164 | 5 | if !self.take_owed() { |
165 | 4 | self.semaphore.add_permits(1); |
166 | 4 | }1 |
167 | 5 | self.limit.store(limit + 1, Ordering::Release); |
168 | 5 | record_object_store_read_limit(self.name, limit + 1); |
169 | 5 | debug!(pool = self.name, limit = limit + 1, "Read limit raised"); |
170 | 264 | } |
171 | | |
172 | | /// The resource refused a request for being busy. The limit halves, not |
173 | | /// below `min`; permits that are out are forgotten as they come back. |
174 | | /// A second throttle within `CUT_WINDOW` of the first is the same event |
175 | | /// and does not cut again. |
176 | 21 | pub fn record_throttle(&self) { |
177 | 21 | let mut last_cut = self.last_cut.lock(); |
178 | 21 | let now = Instant::now(); |
179 | 21 | if last_cut.is_some_and(|at| now13 .duration_since(at) < CUT_WINDOW13 ) { |
180 | 2 | return; |
181 | 19 | } |
182 | 19 | *last_cut = Some(now); |
183 | 19 | self.streak.store(0, Ordering::Release); |
184 | 19 | let limit = self.limit.load(Ordering::Acquire); |
185 | 19 | let target = (limit / 2).max(self.min); |
186 | 19 | if target >= limit { |
187 | 8 | return; |
188 | 11 | } |
189 | 11 | let cut = limit - target; |
190 | | // Take back what is idle now; the rest is owed and leaves with the |
191 | | // permits that hold it. `acquire` does not take this lock, so the |
192 | | // count forgotten is what counts, not what looked idle a moment ago. |
193 | 11 | let taken = self.semaphore.forget_permits(cut); |
194 | 11 | self.owed.fetch_add(cut - taken, Ordering::AcqRel); |
195 | 11 | self.limit.store(target, Ordering::Release); |
196 | 11 | record_object_store_read_limit(self.name, target); |
197 | 11 | info!( |
198 | | pool = self.name, |
199 | | from = limit, |
200 | | to = target, |
201 | | "Read limit halved after a throttled response" |
202 | | ); |
203 | 21 | } |
204 | | } |
205 | | |
206 | | impl Drop for AdaptivePermit { |
207 | 44 | fn drop(&mut self) { |
208 | 44 | let Some(permit) = self.inner.take() else { |
209 | 0 | return; |
210 | | }; |
211 | 44 | if self.pool.take_owed() { |
212 | 5 | permit.forget(); |
213 | 39 | } else { |
214 | 39 | drop(permit); |
215 | 39 | } |
216 | 44 | } |
217 | | } |