/build/source/nativelink-util/src/retry.rs
Line | Count | Source |
1 | | // Copyright 2024 The NativeLink Authors. All rights reserved. |
2 | | // |
3 | | // Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // See LICENSE file for details |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | use core::pin::Pin; |
16 | | use core::time::Duration; |
17 | | use std::sync::Arc; |
18 | | |
19 | | use futures::future::Future; |
20 | | use futures::stream::StreamExt; |
21 | | use nativelink_config::stores::{ErrorCode, Retry}; |
22 | | use nativelink_error::{Code, Error, make_err}; |
23 | | use tracing::{error, info}; |
24 | | |
25 | | struct ExponentialBackoff { |
26 | | current: Duration, |
27 | | } |
28 | | |
29 | | impl ExponentialBackoff { |
30 | 171 | const fn new(base: Duration) -> Self { |
31 | 171 | Self { current: base } |
32 | 171 | } |
33 | | } |
34 | | |
35 | | impl Iterator for ExponentialBackoff { |
36 | | type Item = Duration; |
37 | | |
38 | 20 | fn next(&mut self) -> Option<Duration> { |
39 | 20 | self.current *= 2; |
40 | 20 | Some(self.current) |
41 | 20 | } |
42 | | } |
43 | | |
44 | | type SleepFn = Arc<dyn Fn(Duration) -> Pin<Box<dyn Future<Output = ()> + Send>> + Sync + Send>; |
45 | | pub(crate) type JitterFn = Arc<dyn Fn(Duration) -> Duration + Send + Sync>; |
46 | | |
47 | | #[derive(PartialEq, Eq, Debug)] |
48 | | pub enum RetryResult<T> { |
49 | | Ok(T), |
50 | | Retry(Error), |
51 | | Err(Error), |
52 | | } |
53 | | |
54 | | /// Class used to retry a job with a sleep function in between each retry. |
55 | | #[derive(Clone)] |
56 | | pub struct Retrier { |
57 | | sleep_fn: SleepFn, |
58 | | jitter_fn: JitterFn, |
59 | | config: Retry, |
60 | | } |
61 | | |
62 | | impl core::fmt::Debug for Retrier { |
63 | 0 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { |
64 | 0 | f.debug_struct("Retrier") |
65 | 0 | .field("config", &self.config) |
66 | 0 | .finish_non_exhaustive() |
67 | 0 | } |
68 | | } |
69 | | |
70 | 3 | const fn to_error_code(code: Code) -> ErrorCode { |
71 | 3 | match code { |
72 | 0 | Code::Cancelled => ErrorCode::Cancelled, |
73 | 0 | Code::InvalidArgument => ErrorCode::InvalidArgument, |
74 | 0 | Code::DeadlineExceeded => ErrorCode::DeadlineExceeded, |
75 | 3 | Code::NotFound => ErrorCode::NotFound, |
76 | 0 | Code::AlreadyExists => ErrorCode::AlreadyExists, |
77 | 0 | Code::PermissionDenied => ErrorCode::PermissionDenied, |
78 | 0 | Code::ResourceExhausted => ErrorCode::ResourceExhausted, |
79 | 0 | Code::FailedPrecondition => ErrorCode::FailedPrecondition, |
80 | 0 | Code::Aborted => ErrorCode::Aborted, |
81 | 0 | Code::OutOfRange => ErrorCode::OutOfRange, |
82 | 0 | Code::Unimplemented => ErrorCode::Unimplemented, |
83 | 0 | Code::Internal => ErrorCode::Internal, |
84 | 0 | Code::Unavailable => ErrorCode::Unavailable, |
85 | 0 | Code::DataLoss => ErrorCode::DataLoss, |
86 | 0 | Code::Unauthenticated => ErrorCode::Unauthenticated, |
87 | 0 | _ => ErrorCode::Unknown, |
88 | | } |
89 | 3 | } |
90 | | |
91 | | impl Retrier { |
92 | 150 | pub fn new(sleep_fn: SleepFn, jitter_fn: JitterFn, config: Retry) -> Self { |
93 | 150 | Self { |
94 | 150 | sleep_fn, |
95 | 150 | jitter_fn, |
96 | 150 | config, |
97 | 150 | } |
98 | 150 | } |
99 | | |
100 | | /// This should only return true if the error code should be interpreted as |
101 | | /// temporary. |
102 | 34 | fn should_retry(&self, code: Code) -> bool { |
103 | 34 | if let Some(retry_codes3 ) = &self.config.retry_on_errors { |
104 | 3 | retry_codes.contains(&to_error_code(code)) |
105 | | } else { |
106 | | // Match is intentionally exhaustive: if a new `Code` variant is added, |
107 | | // the compiler forces a decision here. New codes default to permanent |
108 | | // (no retry) rather than silently being retried. |
109 | 31 | match code { |
110 | | Code::Cancelled |
111 | | | Code::Unknown |
112 | | | Code::DeadlineExceeded |
113 | | | Code::ResourceExhausted |
114 | | | Code::Aborted |
115 | | | Code::Internal |
116 | | | Code::Unavailable |
117 | 28 | | Code::DataLoss => true, |
118 | | Code::Ok |
119 | | | Code::InvalidArgument |
120 | | | Code::NotFound |
121 | | | Code::AlreadyExists |
122 | | | Code::PermissionDenied |
123 | | | Code::FailedPrecondition |
124 | | | Code::OutOfRange |
125 | | | Code::Unimplemented |
126 | 3 | | Code::Unauthenticated => false, |
127 | | } |
128 | | } |
129 | 34 | } |
130 | | |
131 | 171 | fn get_retry_config(&self) -> impl Iterator<Item = Duration> + '_ { |
132 | 171 | ExponentialBackoff::new(Duration::from_secs_f32(self.config.delay)) |
133 | 171 | .map(|d| (self.jitter_fn)20 (d20 )) |
134 | 171 | .take(self.config.max_retries) // Remember this is number of retries, so will run max_retries + 1. |
135 | 171 | } |
136 | | |
137 | | #[expect( |
138 | | clippy::manual_async_fn, |
139 | | reason = "making an `async fn` results in a potential compiler bug in seemingly unrelated \ |
140 | | code" |
141 | | )] |
142 | 171 | pub fn retry<'a, T: Send>( |
143 | 171 | &'a self, |
144 | 171 | operation: impl futures::stream::Stream<Item = RetryResult<T>> + Send + 'a, |
145 | 171 | ) -> impl Future<Output = Result<T, Error>> + Send + 'a { |
146 | 171 | async move { |
147 | 171 | let mut iter = self.get_retry_config(); |
148 | 171 | tokio::pin!(operation); |
149 | 171 | let mut attempt = 0; |
150 | | loop { |
151 | 191 | attempt += 1; |
152 | 191 | match operation.next().await { |
153 | | None => { |
154 | 0 | return Err(make_err!( |
155 | 0 | Code::Internal, |
156 | 0 | "Retry stream ended abruptly on attempt {attempt}", |
157 | 0 | )); |
158 | | } |
159 | 153 | Some(RetryResult::Ok(value)) => return Ok(value), |
160 | 4 | Some(RetryResult::Err(e)) => { |
161 | 4 | return Err(e.append(format!("On attempt {attempt}"))); |
162 | | } |
163 | 34 | Some(RetryResult::Retry(err)) => { |
164 | 34 | if !self.should_retry(err.code) { |
165 | 3 | if err.code == Code::NotFound { |
166 | 3 | info!(?err, "Not found, not retrying"); |
167 | | } else { |
168 | 0 | error!(?attempt, ?err, "Not retrying permanent error"); |
169 | | } |
170 | 3 | return Err(err); |
171 | 31 | } |
172 | 20 | (self.sleep_fn)( |
173 | 31 | iter.next() |
174 | 31 | .ok_or_else(|| err11 .append11 (format!11 ("On attempt {attempt}")))?11 , |
175 | | ) |
176 | 20 | .await; |
177 | | } |
178 | | } |
179 | | } |
180 | 171 | } |
181 | 171 | } |
182 | | } |