Coverage Report

Created: 2026-11-13 23:29

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-util/src/adaptive_permits.rs
Line
Count
Source
1
// Copyright 2026 Trace Machina, Inc. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::sync::atomic::{AtomicUsize, Ordering};
16
use core::time::Duration;
17
use std::sync::Arc;
18
19
use parking_lot::Mutex;
20
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
21
use tokio::time::Instant;
22
use tracing::{debug, info};
23
24
use crate::metrics::{record_object_store_read_limit, record_object_store_read_wait};
25
26
/// A throttle within this long of the last cut is the same event seen by
27
/// another request in flight, not a second signal.
28
const CUT_WINDOW: Duration = Duration::from_millis(200);
29
30
/// A pool of permits in front of a shared resource whose capacity is not
31
/// known, sized the way TCP sizes its window: a rejection halves the limit,
32
/// a run of successes raises it by one. Requests wait for a permit inside
33
/// the process instead of being sent and refused, and many processes
34
/// behind one limit converge on a share of it.
35
///
36
/// Built for reads against a cloud object store that answers 503 or 429
37
/// when an account is over its limit; the store calls [`record_throttle`]
38
/// on such a response and [`record_success`] on a completed request.
39
///
40
/// [`record_throttle`]: Self::record_throttle
41
/// [`record_success`]: Self::record_success
42
#[derive(Debug)]
43
pub struct AdaptivePermits {
44
    semaphore: Arc<Semaphore>,
45
    /// The current limit, between `min` and `max`.
46
    limit: AtomicUsize,
47
    min: usize,
48
    max: usize,
49
    /// Permits a cut could not take back because they were out; each is
50
    /// forgotten as it is released instead of returned.
51
    owed: AtomicUsize,
52
    /// Successes since the limit last moved.
53
    streak: AtomicUsize,
54
    /// Successes in a row needed to add a permit. Zero fixes the limit.
55
    grow_after: usize,
56
    last_cut: Mutex<Option<Instant>>,
57
    /// Who the pool belongs to, the `store.object.provider` label.
58
    name: &'static str,
59
}
60
61
/// One permit from an [`AdaptivePermits`] pool, held for the length of a
62
/// request.
63
#[derive(Debug)]
64
pub struct AdaptivePermit {
65
    pool: Arc<AdaptivePermits>,
66
    inner: Option<OwnedSemaphorePermit>,
67
}
68
69
impl AdaptivePermits {
70
    /// A pool that starts at `initial` permits and moves between `min` and
71
    /// `max` as [`record_throttle`] and [`record_success`] are called; with
72
    /// `grow_after` zero the limit never grows and only falls. `initial` is
73
    /// clamped into `[min, max]`.
74
    ///
75
    /// [`record_throttle`]: Self::record_throttle
76
    /// [`record_success`]: Self::record_success
77
19
    pub fn new(
78
19
        name: &'static str,
79
19
        initial: usize,
80
19
        min: usize,
81
19
        max: usize,
82
19
        grow_after: usize,
83
19
    ) -> Arc<Self> {
84
19
        let min = min.max(1);
85
19
        let max = max.max(min);
86
19
        let initial = initial.clamp(min, max);
87
19
        record_object_store_read_limit(name, initial);
88
19
        Arc::new(Self {
89
19
            semaphore: Arc::new(Semaphore::new(initial)),
90
19
            limit: AtomicUsize::new(initial),
91
19
            min,
92
19
            max,
93
19
            owed: AtomicUsize::new(0),
94
19
            streak: AtomicUsize::new(0),
95
19
            grow_after,
96
19
            last_cut: Mutex::new(None),
97
19
            name,
98
19
        })
99
19
    }
100
101
    /// Waits for a permit. The permit is returned to the pool when dropped,
102
    /// unless the limit was cut while it was out.
103
44
    pub async fn acquire(self: &Arc<Self>) -> AdaptivePermit {
104
44
        if self.semaphore.available_permits() == 0 {
105
7
            record_object_store_read_wait(self.name);
106
37
        }
107
44
        let inner = Arc::clone(&self.semaphore)
108
44
            .acquire_owned()
109
44
            .await
110
44
            .expect("AdaptivePermits semaphore is never closed");
111
44
        AdaptivePermit {
112
44
            pool: Arc::clone(self),
113
44
            inner: Some(inner),
114
44
        }
115
44
    }
116
117
    /// Settles one owed permit if any is owed. True when it did.
118
49
    fn take_owed(&self) -> bool {
119
49
        let mut owed = self.owed.load(Ordering::Acquire);
120
        loop {
121
49
            if owed == 0 {
122
43
                return false;
123
6
            }
124
6
            match self.owed.compare_exchange_weak(
125
6
                owed,
126
6
                owed - 1,
127
6
                Ordering::AcqRel,
128
6
                Ordering::Acquire,
129
6
            ) {
130
6
                Ok(_) => return true,
131
0
                Err(current) => owed = current,
132
            }
133
        }
134
49
    }
135
136
    /// The limit as it stands.
137
22
    pub fn limit(&self) -> usize {
138
22
        self.limit.load(Ordering::Acquire)
139
22
    }
140
141
    /// Permits that can be taken right now.
142
9
    pub fn available(&self) -> usize {
143
9
        self.semaphore.available_permits()
144
9
    }
145
146
    /// A request went through. After `grow_after` of these in a row the
147
    /// limit rises by one, up to `max`.
148
264
    pub fn record_success(&self) {
149
264
        if self.grow_after == 0 {
150
209
            return;
151
55
        }
152
55
        let streak = self.streak.fetch_add(1, Ordering::AcqRel) + 1;
153
55
        if streak < self.grow_after {
154
41
            return;
155
14
        }
156
14
        self.streak.store(0, Ordering::Release);
157
14
        let _guard = self.last_cut.lock();
158
14
        let limit = self.limit.load(Ordering::Acquire);
159
14
        if limit >= self.max {
160
9
            return;
161
5
        }
162
        // A permit still owed from a cut is cancelled instead of a new one
163
        // being added, so the pool never holds more than `limit`.
164
5
        if !self.take_owed() {
165
4
            self.semaphore.add_permits(1);
166
4
        
}1
167
5
        self.limit.store(limit + 1, Ordering::Release);
168
5
        record_object_store_read_limit(self.name, limit + 1);
169
5
        debug!(pool = self.name, limit = limit + 1, "Read limit raised");
170
264
    }
171
172
    /// The resource refused a request for being busy. The limit halves, not
173
    /// below `min`; permits that are out are forgotten as they come back.
174
    /// A second throttle within `CUT_WINDOW` of the first is the same event
175
    /// and does not cut again.
176
21
    pub fn record_throttle(&self) {
177
21
        let mut last_cut = self.last_cut.lock();
178
21
        let now = Instant::now();
179
21
        if last_cut.is_some_and(|at| 
now13
.duration_since(at) <
CUT_WINDOW13
) {
180
2
            return;
181
19
        }
182
19
        *last_cut = Some(now);
183
19
        self.streak.store(0, Ordering::Release);
184
19
        let limit = self.limit.load(Ordering::Acquire);
185
19
        let target = (limit / 2).max(self.min);
186
19
        if target >= limit {
187
8
            return;
188
11
        }
189
11
        let cut = limit - target;
190
        // Take back what is idle now; the rest is owed and leaves with the
191
        // permits that hold it. `acquire` does not take this lock, so the
192
        // count forgotten is what counts, not what looked idle a moment ago.
193
11
        let taken = self.semaphore.forget_permits(cut);
194
11
        self.owed.fetch_add(cut - taken, Ordering::AcqRel);
195
11
        self.limit.store(target, Ordering::Release);
196
11
        record_object_store_read_limit(self.name, target);
197
11
        info!(
198
            pool = self.name,
199
            from = limit,
200
            to = target,
201
            "Read limit halved after a throttled response"
202
        );
203
21
    }
204
}
205
206
impl Drop for AdaptivePermit {
207
44
    fn drop(&mut self) {
208
44
        let Some(permit) = self.inner.take() else {
209
0
            return;
210
        };
211
44
        if self.pool.take_owed() {
212
5
            permit.forget();
213
39
        } else {
214
39
            drop(permit);
215
39
        }
216
44
    }
217
}