/build/source/nativelink-error/src/lib.rs
Line | Count | Source |
1 | | // Copyright 2024 The NativeLink Authors. All rights reserved. |
2 | | // |
3 | | // Licensed under the Apache License, Version 2.0 (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // http://www.apache.org/licenses/LICENSE-2.0 |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | use core::convert::Into; |
16 | | |
17 | | use nativelink_metric::{ |
18 | | MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent, |
19 | | }; |
20 | | use prost_types::TimestampError; |
21 | | use serde::{Deserialize, Serialize}; |
22 | | // Reexport of tonic's error codes which we use as "nativelink_error::Code". |
23 | | pub use tonic::Code; |
24 | | |
25 | | #[macro_export] |
26 | | macro_rules! make_err { |
27 | | ($code:expr, $($arg:tt)+) => {{ |
28 | | $crate::Error::new( |
29 | | $code, |
30 | | format!("{}", format_args!($($arg)+)), |
31 | | ) |
32 | | }}; |
33 | | } |
34 | | |
35 | | #[macro_export] |
36 | | macro_rules! make_input_err { |
37 | | ($($arg:tt)+) => {{ |
38 | | $crate::make_err!($crate::Code::InvalidArgument, $($arg)+) |
39 | | }}; |
40 | | } |
41 | | |
42 | | #[macro_export] |
43 | | macro_rules! error_if { |
44 | | ($cond:expr, $($arg:tt)+) => {{ |
45 | | if $cond { |
46 | | Err($crate::make_err!($crate::Code::InvalidArgument, $($arg)+))?; |
47 | | } |
48 | | }}; |
49 | | } |
50 | | |
51 | | #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] |
52 | | pub struct Error { |
53 | | #[serde(with = "CodeDef")] |
54 | | pub code: Code, |
55 | | pub messages: Vec<String>, |
56 | | } |
57 | | |
58 | | impl MetricsComponent for Error { |
59 | 0 | fn publish( |
60 | 0 | &self, |
61 | 0 | kind: MetricKind, |
62 | 0 | field_metadata: MetricFieldData, |
63 | 0 | ) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> { |
64 | 0 | self.to_string().publish(kind, field_metadata) |
65 | 0 | } |
66 | | } |
67 | | |
68 | | impl Error { |
69 | | #[must_use] |
70 | 166 | pub const fn new_with_messages(code: Code, messages: Vec<String>) -> Self { |
71 | 166 | Self { code, messages } |
72 | 166 | } |
73 | | |
74 | | #[must_use] |
75 | 165 | pub fn new(code: Code, msg: String) -> Self { |
76 | 165 | if msg.is_empty() { Branch (76:12): [True: 0, False: 165]
Branch (76:12): [Folded - Ignored]
|
77 | 0 | Self::new_with_messages(code, vec![]) |
78 | | } else { |
79 | 165 | Self::new_with_messages(code, vec![msg]) |
80 | | } |
81 | 165 | } |
82 | | |
83 | | #[inline] |
84 | | #[must_use] |
85 | 9 | pub fn append<S: Into<String>>(mut self, msg: S) -> Self { |
86 | 9 | self.messages.push(msg.into()); |
87 | 9 | self |
88 | 9 | } |
89 | | |
90 | | #[must_use] |
91 | 3 | pub fn merge<E: Into<Self>>(mut self, other: E) -> Self { |
92 | 3 | let mut other: Self = other.into(); |
93 | | // This will help with knowing which messages are tied to different errors. |
94 | 3 | self.messages.push("---".to_string()); |
95 | 3 | self.messages.append(&mut other.messages); |
96 | 3 | self |
97 | 3 | } |
98 | | |
99 | | #[must_use] |
100 | 17 | pub fn merge_option<T: Into<Self>, U: Into<Self>>( |
101 | 17 | this: Option<T>, |
102 | 17 | other: Option<U>, |
103 | 17 | ) -> Option<Self> { |
104 | 17 | if let Some(this4 ) = this { Branch (104:16): [Folded - Ignored]
Branch (104:16): [Folded - Ignored]
Branch (104:16): [True: 4, False: 13]
|
105 | 4 | if let Some(other0 ) = other { Branch (105:20): [Folded - Ignored]
Branch (105:20): [Folded - Ignored]
Branch (105:20): [True: 0, False: 4]
|
106 | 0 | return Some(this.into().merge(other)); |
107 | 4 | } |
108 | 4 | return Some(this.into()); |
109 | 13 | } |
110 | 13 | other.map(Into::into) |
111 | 17 | } |
112 | | |
113 | | #[must_use] |
114 | 1 | pub fn to_std_err(self) -> std::io::Error { |
115 | 1 | std::io::Error::new(self.code.into_error_kind(), self.messages.join(" : ")) |
116 | 1 | } |
117 | | |
118 | | #[must_use] |
119 | 6 | pub fn message_string(&self) -> String { |
120 | 6 | self.messages.join(" : ") |
121 | 6 | } |
122 | | } |
123 | | |
124 | | impl core::error::Error for Error {} |
125 | | |
126 | | impl From<Error> for nativelink_proto::google::rpc::Status { |
127 | 6 | fn from(val: Error) -> Self { |
128 | 6 | Self { |
129 | 6 | code: val.code as i32, |
130 | 6 | message: val.message_string(), |
131 | 6 | details: vec![], |
132 | 6 | } |
133 | 6 | } |
134 | | } |
135 | | |
136 | | impl From<nativelink_proto::google::rpc::Status> for Error { |
137 | 2 | fn from(val: nativelink_proto::google::rpc::Status) -> Self { |
138 | 2 | Self { |
139 | 2 | code: val.code.into(), |
140 | 2 | messages: vec![val.message], |
141 | 2 | } |
142 | 2 | } |
143 | | } |
144 | | |
145 | | impl core::fmt::Display for Error { |
146 | 21 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { |
147 | | // A manual impl to reduce the noise of frequently empty fields. |
148 | 21 | let mut builder = f.debug_struct("Error"); |
149 | | |
150 | 21 | builder.field("code", &self.code); |
151 | | |
152 | 21 | if !self.messages.is_empty() { Branch (152:12): [True: 21, False: 0]
Branch (152:12): [Folded - Ignored]
|
153 | 21 | builder.field("messages", &self.messages); |
154 | 21 | }0 |
155 | | |
156 | 21 | builder.finish() |
157 | 21 | } |
158 | | } |
159 | | |
160 | | impl From<prost::DecodeError> for Error { |
161 | 0 | fn from(err: prost::DecodeError) -> Self { |
162 | 0 | make_err!(Code::Internal, "{}", err.to_string()) |
163 | 0 | } |
164 | | } |
165 | | |
166 | | impl From<prost::EncodeError> for Error { |
167 | 0 | fn from(err: prost::EncodeError) -> Self { |
168 | 0 | make_err!(Code::Internal, "{}", err.to_string()) |
169 | 0 | } |
170 | | } |
171 | | |
172 | | impl From<prost::UnknownEnumValue> for Error { |
173 | 0 | fn from(err: prost::UnknownEnumValue) -> Self { |
174 | 0 | make_err!(Code::Internal, "{}", err.to_string()) |
175 | 0 | } |
176 | | } |
177 | | |
178 | | impl From<core::num::TryFromIntError> for Error { |
179 | 0 | fn from(err: core::num::TryFromIntError) -> Self { |
180 | 0 | make_err!(Code::InvalidArgument, "{}", err.to_string()) |
181 | 0 | } |
182 | | } |
183 | | |
184 | | impl From<tokio::task::JoinError> for Error { |
185 | 0 | fn from(err: tokio::task::JoinError) -> Self { |
186 | 0 | make_err!(Code::Internal, "{}", err.to_string()) |
187 | 0 | } |
188 | | } |
189 | | |
190 | | impl From<serde_json5::Error> for Error { |
191 | 0 | fn from(err: serde_json5::Error) -> Self { |
192 | 0 | match err { |
193 | 0 | serde_json5::Error::Message { msg, location } => { |
194 | 0 | if let Some(has_location) = location { Branch (194:24): [True: 0, False: 0]
Branch (194:24): [Folded - Ignored]
|
195 | 0 | make_err!( |
196 | 0 | Code::Internal, |
197 | | "line {}, column {} - {}", |
198 | | has_location.line, |
199 | | has_location.column, |
200 | | msg |
201 | | ) |
202 | | } else { |
203 | 0 | make_err!(Code::Internal, "{}", msg) |
204 | | } |
205 | | } |
206 | | } |
207 | 0 | } |
208 | | } |
209 | | |
210 | | impl From<core::num::ParseIntError> for Error { |
211 | 0 | fn from(err: core::num::ParseIntError) -> Self { |
212 | 0 | make_err!(Code::InvalidArgument, "{}", err.to_string()) |
213 | 0 | } |
214 | | } |
215 | | |
216 | | impl From<core::convert::Infallible> for Error { |
217 | 0 | fn from(_err: core::convert::Infallible) -> Self { |
218 | | // Infallible is an error type that can never happen. |
219 | 0 | unreachable!(); |
220 | | } |
221 | | } |
222 | | |
223 | | impl From<TimestampError> for Error { |
224 | 0 | fn from(err: TimestampError) -> Self { |
225 | 0 | make_err!(Code::InvalidArgument, "{}", err) |
226 | 0 | } |
227 | | } |
228 | | |
229 | | impl From<std::io::Error> for Error { |
230 | 12 | fn from(err: std::io::Error) -> Self { |
231 | 12 | Self { |
232 | 12 | code: err.kind().into_code(), |
233 | 12 | messages: vec![err.to_string()], |
234 | 12 | } |
235 | 12 | } |
236 | | } |
237 | | |
238 | | impl From<fred::error::Error> for Error { |
239 | 1 | fn from(error: fred::error::Error) -> Self { |
240 | | use fred::error::ErrorKind::{ |
241 | | Auth, Backpressure, Canceled, Cluster, Config, IO, InvalidArgument, InvalidCommand, |
242 | | NotFound, Parse, Protocol, Routing, Sentinel, Timeout, Tls, Unknown, Url, |
243 | | }; |
244 | | |
245 | | // Conversions here are based on https://grpc.github.io/grpc/core/md_doc_statuscodes.html. |
246 | 1 | let code = match error.kind() { |
247 | 0 | Config | InvalidCommand | InvalidArgument | Url => Code::InvalidArgument, |
248 | 0 | IO | Protocol | Tls | Cluster | Parse | Sentinel | Routing => Code::Internal, |
249 | 0 | Auth => Code::PermissionDenied, |
250 | 0 | Canceled => Code::Aborted, |
251 | 0 | Unknown => Code::Unknown, |
252 | 1 | Timeout => Code::DeadlineExceeded, |
253 | 0 | NotFound => Code::NotFound, |
254 | 0 | Backpressure => Code::Unavailable, |
255 | | }; |
256 | | |
257 | 1 | make_err!(code, "{error}") |
258 | 1 | } |
259 | | } |
260 | | |
261 | | impl From<tonic::Status> for Error { |
262 | 2 | fn from(status: tonic::Status) -> Self { |
263 | 2 | make_err!(status.code(), "{}", status.to_string()) |
264 | 2 | } |
265 | | } |
266 | | |
267 | | impl From<Error> for tonic::Status { |
268 | 13 | fn from(val: Error) -> Self { |
269 | 13 | Self::new(val.code, val.messages.join(" : ")) |
270 | 13 | } |
271 | | } |
272 | | |
273 | | pub trait ResultExt<T> { |
274 | | /// # Errors |
275 | | /// |
276 | | /// Will return `Err` if we can't convert the error. |
277 | | fn err_tip_with_code<F, S>(self, tip_fn: F) -> Result<T, Error> |
278 | | where |
279 | | Self: Sized, |
280 | | S: ToString, |
281 | | F: (FnOnce(&Error) -> (Code, S)) + Sized; |
282 | | |
283 | | /// # Errors |
284 | | /// |
285 | | /// Will return `Err` if we can't convert the error. |
286 | | #[inline] |
287 | 278k | fn err_tip<F, S>(self, tip_fn: F) -> Result<T, Error> |
288 | 278k | where |
289 | 278k | Self: Sized, |
290 | 278k | S: ToString, |
291 | 278k | F: (FnOnce() -> S) + Sized, |
292 | | { |
293 | 278k | self.err_tip_with_code(|e| (e.code114 , tip_fn()114 )) |
294 | 278k | } |
295 | | |
296 | | /// # Errors |
297 | | /// |
298 | | /// Will return `Err` if we can't merge the errors. |
299 | 0 | fn merge<U>(self, _other: Result<U, Error>) -> Result<U, Error> |
300 | 0 | where |
301 | 0 | Self: Sized, |
302 | | { |
303 | 0 | unreachable!(); |
304 | | } |
305 | | } |
306 | | |
307 | | impl<T, E: Into<Error>> ResultExt<T> for Result<T, E> { |
308 | | #[inline] |
309 | 276k | fn err_tip_with_code<F, S>(self, tip_fn: F) -> Result<T, Error> |
310 | 276k | where |
311 | 276k | Self: Sized, |
312 | 276k | S: ToString, |
313 | 276k | F: (FnOnce(&Error) -> (Code, S)) + Sized, |
314 | | { |
315 | 276k | self.map_err(|e| {103 |
316 | 103 | let mut error: Error = e.into(); |
317 | 103 | let (code, message) = tip_fn(&error); |
318 | 103 | error.code = code; |
319 | 103 | error.messages.push(message.to_string()); |
320 | 103 | error |
321 | 103 | }) |
322 | 276k | } |
323 | | |
324 | 5.72k | fn merge<U>(self, other: Result<U, Error>) -> Result<U, Error> |
325 | 5.72k | where |
326 | 5.72k | Self: Sized, |
327 | | { |
328 | 5.72k | if let Err(e17 ) = self { Branch (328:16): [Folded - Ignored]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 9]
Branch (328:16): [True: 4, False: 39]
Branch (328:16): [True: 4, False: 273]
Branch (328:16): [True: 0, False: 1]
Branch (328:16): [True: 0, False: 1]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 1, False: 17]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 1.23k]
Branch (328:16): [True: 0, False: 1]
Branch (328:16): [True: 3, False: 3.95k]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 1]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 10]
Branch (328:16): [True: 0, False: 1]
Branch (328:16): [True: 2, False: 1]
Branch (328:16): [True: 0, False: 2]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 25]
Branch (328:16): [True: 0, False: 4]
Branch (328:16): [True: 1, False: 3]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [Folded - Ignored]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 3]
Branch (328:16): [True: 0, False: 1]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 5]
Branch (328:16): [True: 1, False: 4]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 2]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 49]
Branch (328:16): [True: 0, False: 5]
Branch (328:16): [True: 0, False: 4]
Branch (328:16): [True: 0, False: 1]
Branch (328:16): [True: 0, False: 5]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 1, False: 46]
Branch (328:16): [True: 0, False: 4]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 0]
Branch (328:16): [True: 0, False: 0]
|
329 | 17 | let mut e: Error = e.into(); |
330 | 17 | if let Err(other_err16 ) = other { Branch (330:20): [Folded - Ignored]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 4, False: 0]
Branch (330:20): [True: 4, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 1, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 3, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 2, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 1, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [Folded - Ignored]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 1, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 1]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
Branch (330:20): [True: 0, False: 0]
|
331 | 16 | let mut other_err: Error = other_err; |
332 | 16 | // This will help with knowing which messages are tied to different errors. |
333 | 16 | e.messages.push("---".to_string()); |
334 | 16 | e.messages.append(&mut other_err.messages); |
335 | 16 | }1 |
336 | 17 | return Err(e); |
337 | 5.70k | } |
338 | 5.70k | other |
339 | 5.72k | } |
340 | | } |
341 | | |
342 | | impl<T> ResultExt<T> for Option<T> { |
343 | | #[inline] |
344 | 5.61k | fn err_tip_with_code<F, S>(self, tip_fn: F) -> Result<T, Error> |
345 | 5.61k | where |
346 | 5.61k | Self: Sized, |
347 | 5.61k | S: ToString, |
348 | 5.61k | F: (FnOnce(&Error) -> (Code, S)) + Sized, |
349 | | { |
350 | 5.61k | self.ok_or_else(|| {18 |
351 | 18 | let mut error = Error { |
352 | 18 | code: Code::Internal, |
353 | 18 | messages: vec![], |
354 | 18 | }; |
355 | 18 | let (code, message) = tip_fn(&error); |
356 | 18 | error.code = code; |
357 | 18 | error.messages.push(message.to_string()); |
358 | 18 | error |
359 | 18 | }) |
360 | 5.61k | } |
361 | | } |
362 | | |
363 | | trait CodeExt { |
364 | | fn into_error_kind(self) -> std::io::ErrorKind; |
365 | | } |
366 | | |
367 | | impl CodeExt for Code { |
368 | 1 | fn into_error_kind(self) -> std::io::ErrorKind { |
369 | 1 | match self { |
370 | 0 | Self::Aborted => std::io::ErrorKind::Interrupted, |
371 | 0 | Self::AlreadyExists => std::io::ErrorKind::AlreadyExists, |
372 | 0 | Self::DeadlineExceeded => std::io::ErrorKind::TimedOut, |
373 | 0 | Self::InvalidArgument => std::io::ErrorKind::InvalidInput, |
374 | 0 | Self::NotFound => std::io::ErrorKind::NotFound, |
375 | 0 | Self::PermissionDenied => std::io::ErrorKind::PermissionDenied, |
376 | 0 | Self::Unavailable => std::io::ErrorKind::ConnectionRefused, |
377 | 1 | _ => std::io::ErrorKind::Other, |
378 | | } |
379 | 1 | } |
380 | | } |
381 | | |
382 | | trait ErrorKindExt { |
383 | | fn into_code(self) -> Code; |
384 | | } |
385 | | |
386 | | impl ErrorKindExt for std::io::ErrorKind { |
387 | 12 | fn into_code(self) -> Code { |
388 | 12 | match self { |
389 | 11 | Self::NotFound => Code::NotFound, |
390 | 0 | Self::PermissionDenied => Code::PermissionDenied, |
391 | | Self::ConnectionRefused | Self::ConnectionReset | Self::ConnectionAborted => { |
392 | 0 | Code::Unavailable |
393 | | } |
394 | 0 | Self::AlreadyExists => Code::AlreadyExists, |
395 | 0 | Self::InvalidInput | Self::InvalidData => Code::InvalidArgument, |
396 | 0 | Self::TimedOut => Code::DeadlineExceeded, |
397 | 0 | Self::Interrupted => Code::Aborted, |
398 | | Self::NotConnected |
399 | | | Self::AddrInUse |
400 | | | Self::AddrNotAvailable |
401 | | | Self::BrokenPipe |
402 | | | Self::WouldBlock |
403 | | | Self::WriteZero |
404 | | | Self::Other |
405 | 1 | | Self::UnexpectedEof => Code::Internal, |
406 | 0 | _ => Code::Unknown, |
407 | | } |
408 | 12 | } |
409 | | } |
410 | | |
411 | | // Serde definition for tonic::Code. See: https://serde.rs/remote-derive.html |
412 | | #[derive(Debug, Clone, Copy, Serialize, Deserialize)] |
413 | | #[serde(remote = "Code")] |
414 | | pub enum CodeDef { |
415 | | Ok = 0, |
416 | | Cancelled = 1, |
417 | | Unknown = 2, |
418 | | InvalidArgument = 3, |
419 | | DeadlineExceeded = 4, |
420 | | NotFound = 5, |
421 | | AlreadyExists = 6, |
422 | | PermissionDenied = 7, |
423 | | ResourceExhausted = 8, |
424 | | FailedPrecondition = 9, |
425 | | Aborted = 10, |
426 | | OutOfRange = 11, |
427 | | Unimplemented = 12, |
428 | | Internal = 13, |
429 | | Unavailable = 14, |
430 | | DataLoss = 15, |
431 | | Unauthenticated = 16, |
432 | | // NOTE: Additional codes must be added to stores.rs in ErrorCodes and also |
433 | | // in both match statements in retry.rs. |
434 | | } |