Coverage Report

Created: 2026-09-18 20:40

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-util/src/retry.rs
Line
Count
Source
1
// Copyright 2024 The NativeLink Authors. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::pin::Pin;
16
use core::time::Duration;
17
use std::sync::Arc;
18
19
use futures::future::Future;
20
use futures::stream::StreamExt;
21
use nativelink_config::stores::{ErrorCode, Retry};
22
use nativelink_error::{Code, Error, make_err};
23
use tracing::{error, info};
24
25
struct ExponentialBackoff {
26
    current: Duration,
27
}
28
29
impl ExponentialBackoff {
30
171
    const fn new(base: Duration) -> Self {
31
171
        Self { current: base }
32
171
    }
33
}
34
35
impl Iterator for ExponentialBackoff {
36
    type Item = Duration;
37
38
20
    fn next(&mut self) -> Option<Duration> {
39
20
        self.current *= 2;
40
20
        Some(self.current)
41
20
    }
42
}
43
44
type SleepFn = Arc<dyn Fn(Duration) -> Pin<Box<dyn Future<Output = ()> + Send>> + Sync + Send>;
45
pub(crate) type JitterFn = Arc<dyn Fn(Duration) -> Duration + Send + Sync>;
46
47
#[derive(PartialEq, Eq, Debug)]
48
pub enum RetryResult<T> {
49
    Ok(T),
50
    Retry(Error),
51
    Err(Error),
52
}
53
54
/// Class used to retry a job with a sleep function in between each retry.
55
#[derive(Clone)]
56
pub struct Retrier {
57
    sleep_fn: SleepFn,
58
    jitter_fn: JitterFn,
59
    config: Retry,
60
}
61
62
impl core::fmt::Debug for Retrier {
63
0
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
64
0
        f.debug_struct("Retrier")
65
0
            .field("config", &self.config)
66
0
            .finish_non_exhaustive()
67
0
    }
68
}
69
70
3
const fn to_error_code(code: Code) -> ErrorCode {
71
3
    match code {
72
0
        Code::Cancelled => ErrorCode::Cancelled,
73
0
        Code::InvalidArgument => ErrorCode::InvalidArgument,
74
0
        Code::DeadlineExceeded => ErrorCode::DeadlineExceeded,
75
3
        Code::NotFound => ErrorCode::NotFound,
76
0
        Code::AlreadyExists => ErrorCode::AlreadyExists,
77
0
        Code::PermissionDenied => ErrorCode::PermissionDenied,
78
0
        Code::ResourceExhausted => ErrorCode::ResourceExhausted,
79
0
        Code::FailedPrecondition => ErrorCode::FailedPrecondition,
80
0
        Code::Aborted => ErrorCode::Aborted,
81
0
        Code::OutOfRange => ErrorCode::OutOfRange,
82
0
        Code::Unimplemented => ErrorCode::Unimplemented,
83
0
        Code::Internal => ErrorCode::Internal,
84
0
        Code::Unavailable => ErrorCode::Unavailable,
85
0
        Code::DataLoss => ErrorCode::DataLoss,
86
0
        Code::Unauthenticated => ErrorCode::Unauthenticated,
87
0
        _ => ErrorCode::Unknown,
88
    }
89
3
}
90
91
impl Retrier {
92
150
    pub fn new(sleep_fn: SleepFn, jitter_fn: JitterFn, config: Retry) -> Self {
93
150
        Self {
94
150
            sleep_fn,
95
150
            jitter_fn,
96
150
            config,
97
150
        }
98
150
    }
99
100
    /// This should only return true if the error code should be interpreted as
101
    /// temporary.
102
34
    fn should_retry(&self, code: Code) -> bool {
103
34
        if let Some(
retry_codes3
) = &self.config.retry_on_errors {
104
3
            retry_codes.contains(&to_error_code(code))
105
        } else {
106
            // Match is intentionally exhaustive: if a new `Code` variant is added,
107
            // the compiler forces a decision here. New codes default to permanent
108
            // (no retry) rather than silently being retried.
109
31
            match code {
110
                Code::Cancelled
111
                | Code::Unknown
112
                | Code::DeadlineExceeded
113
                | Code::ResourceExhausted
114
                | Code::Aborted
115
                | Code::Internal
116
                | Code::Unavailable
117
28
                | Code::DataLoss => true,
118
                Code::Ok
119
                | Code::InvalidArgument
120
                | Code::NotFound
121
                | Code::AlreadyExists
122
                | Code::PermissionDenied
123
                | Code::FailedPrecondition
124
                | Code::OutOfRange
125
                | Code::Unimplemented
126
3
                | Code::Unauthenticated => false,
127
            }
128
        }
129
34
    }
130
131
171
    fn get_retry_config(&self) -> impl Iterator<Item = Duration> + '_ {
132
171
        ExponentialBackoff::new(Duration::from_secs_f32(self.config.delay))
133
171
            .map(|d| 
(self.jitter_fn)20
(
d20
))
134
171
            .take(self.config.max_retries) // Remember this is number of retries, so will run max_retries + 1.
135
171
    }
136
137
    #[expect(
138
        clippy::manual_async_fn,
139
        reason = "making an `async fn` results in a potential compiler bug in seemingly unrelated \
140
            code"
141
    )]
142
171
    pub fn retry<'a, T: Send>(
143
171
        &'a self,
144
171
        operation: impl futures::stream::Stream<Item = RetryResult<T>> + Send + 'a,
145
171
    ) -> impl Future<Output = Result<T, Error>> + Send + 'a {
146
171
        async move {
147
171
            let mut iter = self.get_retry_config();
148
171
            tokio::pin!(operation);
149
171
            let mut attempt = 0;
150
            loop {
151
191
                attempt += 1;
152
191
                match operation.next().await {
153
                    None => {
154
0
                        return Err(make_err!(
155
0
                            Code::Internal,
156
0
                            "Retry stream ended abruptly on attempt {attempt}",
157
0
                        ));
158
                    }
159
153
                    Some(RetryResult::Ok(value)) => return Ok(value),
160
4
                    Some(RetryResult::Err(e)) => {
161
4
                        return Err(e.append(format!("On attempt {attempt}")));
162
                    }
163
34
                    Some(RetryResult::Retry(err)) => {
164
34
                        if !self.should_retry(err.code) {
165
3
                            if err.code == Code::NotFound {
166
3
                                info!(?err, "Not found, not retrying");
167
                            } else {
168
0
                                error!(?attempt, ?err, "Not retrying permanent error");
169
                            }
170
3
                            return Err(err);
171
31
                        }
172
20
                        (self.sleep_fn)(
173
31
                            iter.next()
174
31
                                .ok_or_else(|| 
err11
.
append11
(
format!11
("On attempt {attempt}")))
?11
,
175
                        )
176
20
                        .await;
177
                    }
178
                }
179
            }
180
171
        }
181
171
    }
182
}