/build/source/nativelink-error/src/lib.rs

Source
// Copyright 2024 The NativeLink Authors. All rights reserved.
//
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    See LICENSE file for details
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use core::convert::Into;
use core::str::Utf8Error;
use std::sync::{MutexGuard, PoisonError};

use nativelink_metric::{
    MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent,
};
use prost_types::TimestampError;
use serde::{Deserialize, Serialize};
use tokio::sync::AcquireError;
// Reexport of tonic's error codes which we use as "nativelink_error::Code".
pub use tonic::Code;

#[macro_export]
macro_rules! make_err {
    ($code:expr, $($arg:tt)+) => {{
        $crate::Error::new(
            $code,
            format!("{}", format_args!($($arg)+)),
        )
    }};
}

#[macro_export]
macro_rules! make_input_err {
    ($($arg:tt)+) => {{
        $crate::make_err!($crate::Code::InvalidArgument, $($arg)+)
    }};
}

#[macro_export]
macro_rules! error_if {
    ($cond:expr, $($arg:tt)+) => {{
        if $cond {
            Err($crate::make_err!($crate::Code::InvalidArgument, $($arg)+))?;
        }
    }};
}

/// Typed metadata that travels with an [`Error`].
///
/// Used in place of string-parsing error messages when the producer
/// has structured information the consumer needs to act on. The
/// motivating case is missing-blob errors from `fast_slow_store` —
/// `to_execute_response` reads [`ErrorContext::MissingDigest`] and
/// surfaces a `FAILED_PRECONDITION` with a `PreconditionFailure`
/// detail naming the digest, which Bazel auto-retries on.
///
/// Default is [`ErrorContext::None`]; existing call sites that
/// construct [`Error`] via `make_err!` / `Error::new` do not need to
/// be updated.
#[derive(Default, Debug, Eq, PartialEq, Clone, Serialize, Deserialize)]
pub enum ErrorContext {
    #[default]
    None,
    /// The error refers to a specific CAS blob that could not be
    /// located. `hash` and `size` together form the digest the client
    /// should re-upload (`REv2` `blobs/{hash}/{size}`).
    MissingDigest { hash: String, size: i64 },
}

#[derive(Eq, PartialEq, Clone, Serialize, Deserialize)]
pub struct Error {
    #[serde(with = "CodeDef")]
    pub code: Code,
    pub messages: Vec<String>,
    #[serde(default, skip_serializing_if = "ErrorContext::is_none")]
    pub context: ErrorContext,
}

impl core::fmt::Debug for Error {
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
        let mut builder = f.debug_struct("Error");
        builder.field("code", &self.code);
        if !self.messages.is_empty() {
            builder.field("messages", &self.messages);
        }0
        if !self.context.is_none() {
            builder.field("context", &self.context);
        }
        builder.finish()
    }
}

impl ErrorContext {
    #[inline]
    #[must_use]
    pub const fn is_none(&self) -> bool {
        matches!0(self, Self::None)
    }
}

impl MetricsComponent for Error {
    fn publish(
        &self,
        kind: MetricKind,
        field_metadata: MetricFieldData,
    ) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> {
        self.to_string().publish(kind, field_metadata)
    }
}

impl Error {
    #[must_use]
    pub const fn new_with_messages(code: Code, messages: Vec<String>) -> Self {
        Self {
            code,
            messages,
            context: ErrorContext::None,
        }
    }

    #[must_use]
    pub fn new(code: Code, msg: String) -> Self {
        if msg.is_empty() {
            Self::new_with_messages(code, vec![])
        } else {
            Self::new_with_messages(code, vec![msg])
        }
    }

    #[must_use]
    pub fn from_std_err(code: Code, mut err: &dyn core::error::Error) -> Self {
        let mut messages = vec![format!("{err}")];
        while let Some(src6) = err.source() {
            messages.push(format!("{src}"));
            err = src;
        }
        messages.reverse();
        Self::new_with_messages(code, messages)
    }

    #[inline]
    #[must_use]
    pub fn append<S: Into<String>>(mut self, msg: S) -> Self {
        self.messages.push(msg.into());
        self
    }

    #[inline]
    #[must_use]
    pub fn with_context(mut self, context: ErrorContext) -> Self {
        self.context = context;
        self
    }

    #[must_use]
    pub fn merge<E: Into<Self>>(mut self, other: E) -> Self {
        let mut other: Self = other.into();
        // This will help with knowing which messages are tied to different errors.
        self.messages.push("---".to_string());
        self.messages.append(&mut other.messages);
        self
    }

    #[must_use]
    pub fn merge_option<T: Into<Self>, U: Into<Self>>(
        this: Option<T>,
        other: Option<U>,
    ) -> Option<Self> {
        if let Some(this5) = this {
            if let Some(other1) = other {
                return Some(this.into().merge(other));
            }
            return Some(this.into());
        }
        other.map(Into::into)
    }

    #[must_use]
    pub fn to_std_err(self) -> std::io::Error {
        std::io::Error::new(self.code.into_error_kind(), self.messages.join(" : "))
    }

    #[must_use]
    pub fn message_string(&self) -> String {
        self.messages.join(" : ")
    }
}

impl core::error::Error for Error {}

impl From<Error> for nativelink_proto::google::rpc::Status {
    fn from(val: Error) -> Self {
        Self {
            code: val.code as i32,
            message: val.message_string(),
            details: vec![],
        }
    }
}

impl From<nativelink_proto::google::rpc::Status> for Error {
    fn from(val: nativelink_proto::google::rpc::Status) -> Self {
        Self {
            code: val.code.into(),
            messages: vec![val.message],
            context: ErrorContext::None,
        }
    }
}

impl core::fmt::Display for Error {
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
        // A manual impl to reduce the noise of frequently empty fields.
        let mut builder = f.debug_struct("Error");

        builder.field("code", &self.code);

        if !self.messages.is_empty() {
            builder.field("messages", &self.messages);
        }0

        builder.finish()
    }
}

impl From<prost::DecodeError> for Error {
    fn from(err: prost::DecodeError) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<prost::EncodeError> for Error {
    fn from(err: prost::EncodeError) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<prost::UnknownEnumValue> for Error {
    fn from(err: prost::UnknownEnumValue) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<core::num::TryFromIntError> for Error {
    fn from(err: core::num::TryFromIntError) -> Self {
        Self::from_std_err(Code::InvalidArgument, &err)
    }
}

impl From<tokio::task::JoinError> for Error {
    fn from(err: tokio::task::JoinError) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl<T> From<PoisonError<MutexGuard<'_, T>>> for Error {
    fn from(err: PoisonError<MutexGuard<'_, T>>) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<serde_json5::Error> for Error {
    fn from(err: serde_json5::Error) -> Self {
        match err {
            serde_json5::Error::Message { msg, location } => {
                if let Some(has_location) = location {
                    make_err!(
                        Code::Internal,
                        "line {}, column {} - {}",
                        has_location.line,
                        has_location.column,
                        msg
                    )
                } else {
                    Self::new(Code::Internal, msg)
                }
            }
        }
    }
}

impl From<core::num::ParseIntError> for Error {
    fn from(err: core::num::ParseIntError) -> Self {
        Self::from_std_err(Code::InvalidArgument, &err)
    }
}

impl From<core::convert::Infallible> for Error {
    fn from(_err: core::convert::Infallible) -> Self {
        // Infallible is an error type that can never happen.
        unreachable!();
    }
}

impl From<TimestampError> for Error {
    fn from(err: TimestampError) -> Self {
        Self::from_std_err(Code::InvalidArgument, &err)
    }
}

impl From<AcquireError> for Error {
    fn from(err: AcquireError) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<Utf8Error> for Error {
    fn from(err: Utf8Error) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<std::io::Error> for Error {
    fn from(err: std::io::Error) -> Self {
        Self {
            code: err.kind().into_code(),
            messages: vec![err.to_string()],
            context: ErrorContext::None,
        }
    }
}

impl From<redis::RedisError> for Error {
    fn from(error: redis::RedisError) -> Self {
        use redis::ErrorKind::{
            AuthenticationFailed, ClusterConnectionNotFound, EmptySentinelList,
            InvalidClientConfig, Io as IoError, MasterNameNotFoundBySentinel,
            NoValidReplicasFoundBySentinel, Parse as ParseError, RESP3NotSupported, Server,
            UnexpectedReturnType,
        };
        use redis::ServerErrorKind::{
            BusyLoading, ClusterDown, MasterDown, NoPerm, ReadOnly, TryAgain,
        };

        // Conversions here are based on https://grpc.github.io/grpc/core/md_doc_statuscodes.html.
        //
        // The distinction that matters most is retryable vs not. These statuses
        // reach REAPI clients directly, and Bazel treats INVALID_ARGUMENT as
        // permanent: it fails the build instantly rather than retrying. So
        // anything caused by the state of the Redis deployment — a failover in
        // progress, a dropped connection, a node still loading its dataset —
        // must NOT land there, or a routine blip becomes a failed CI run.
        let code = match error.kind() {
            AuthenticationFailed | Server(NoPerm) => Code::PermissionDenied,

            // Topology in flux. Sentinel failover makes these normal and
            // brief: the master moved and the client has not caught up yet.
            // Surfacing MasterNameNotFoundBySentinel as INVALID_ARGUMENT is
            // what turned a 1-2 minute sentinel failover into a dead build.
            MasterNameNotFoundBySentinel
            | NoValidReplicasFoundBySentinel
            | ClusterConnectionNotFound
            | Server(ClusterDown | MasterDown | TryAgain | BusyLoading | ReadOnly) => {
                Code::Unavailable
            }

            // A timeout is worth distinguishing from a dead connection, but
            // both are transient and both are retryable.
            IoError => {
                if error.is_timeout() {
                    Code::DeadlineExceeded
                } else {
                    Code::Unavailable
                }
            }

            // Genuine misconfiguration by the operator — retrying cannot help.
            InvalidClientConfig | EmptySentinelList | RESP3NotSupported => Code::InvalidArgument,

            // Server-side or protocol faults. Not the caller's argument, so
            // not InvalidArgument: a malformed reply is our problem or the
            // server's, and it is at least worth a retry.
            ParseError | UnexpectedReturnType => Code::Internal,

            _ => Code::Unknown,
        };

        let kind = error.kind();
        make_err!(code, "{kind:?}: {error}")
    }
}

impl From<tonic::Status> for Error {
    fn from(status: tonic::Status) -> Self {
        Self::new(status.code(), status.to_string())
    }
}

impl From<Error> for tonic::Status {
    fn from(val: Error) -> Self {
        Self::new(val.code, val.messages.join(" : "))
    }
}

impl From<walkdir::Error> for Error {
    fn from(err: walkdir::Error) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<uuid::Error> for Error {
    fn from(err: uuid::Error) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<rustls_pki_types::pem::Error> for Error {
    fn from(err: rustls_pki_types::pem::Error) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<tokio::time::error::Elapsed> for Error {
    fn from(err: tokio::time::error::Elapsed) -> Self {
        Self::from_std_err(Code::DeadlineExceeded, &err)
    }
}

impl From<url::ParseError> for Error {
    fn from(err: url::ParseError) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<mongodb::error::Error> for Error {
    fn from(err: mongodb::error::Error) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<reqwest::Error> for Error {
    fn from(err: reqwest::Error) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<zip::result::ZipError> for Error {
    fn from(err: zip::result::ZipError) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<std::ffi::NulError> for Error {
    fn from(err: std::ffi::NulError) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

impl From<base64::DecodeError> for Error {
    fn from(err: base64::DecodeError) -> Self {
        Self::from_std_err(Code::Internal, &err)
    }
}

pub trait ResultExt<T> {
    /// # Errors
    ///
    /// Will return `Err` if we can't convert the error.
    fn err_tip_with_code<F, S>(self, tip_fn: F) -> Result<T, Error>
    where
        Self: Sized,
        S: ToString,
        F: (FnOnce(&Error) -> (Code, S)) + Sized;

    /// # Errors
    ///
    /// Will return `Err` if we can't convert the error.
    #[inline]
    fn err_tip<F, S>(self, tip_fn: F) -> Result<T, Error>
    where
        Self: Sized,
        S: ToString,
        F: (FnOnce() -> S) + Sized,
    {
        self.err_tip_with_code(|e| (e.code251, tip_fn()251))
    }

    /// # Errors
    ///
    /// Will return `Err` if we can't merge the errors.
    fn merge<U>(self, _other: Result<U, Error>) -> Result<U, Error>
    where
        Self: Sized,
    {
        unreachable!();
    }
}

impl<T, E: Into<Error>> ResultExt<T> for Result<T, E> {
    #[inline]
    fn err_tip_with_code<F, S>(self, tip_fn: F) -> Result<T, Error>
    where
        Self: Sized,
        S: ToString,
        F: (FnOnce(&Error) -> (Code, S)) + Sized,
    {
        self.map_err(|e| {238
            let mut error: Error = e.into();
            let (code, message) = tip_fn(&error);
            error.code = code;
            error.messages.push(message.to_string());
            error
        })
    }

    fn merge<U>(self, other: Result<U, Error>) -> Result<U, Error>
    where
        Self: Sized,
    {
        if let Err(e44) = self {
            let mut e: Error = e.into();
            if let Err(other_err40) = other {
                let mut other_err: Error = other_err;
                // This will help with knowing which messages are tied to different errors.
                e.messages.push("---".to_string());
                e.messages.append(&mut other_err.messages);
            }4
            return Err(e);
        }
        other
    }
}

impl<T> ResultExt<T> for Option<T> {
    #[inline]
    fn err_tip_with_code<F, S>(self, tip_fn: F) -> Result<T, Error>
    where
        Self: Sized,
        S: ToString,
        F: (FnOnce(&Error) -> (Code, S)) + Sized,
    {
        self.ok_or_else(|| {30
            let mut error = Error {
                code: Code::Internal,
                messages: vec![],
                context: ErrorContext::None,
            };
            let (code, message) = tip_fn(&error);
            error.code = code;
            error.messages.push(message.to_string());
            error
        })
    }
}

trait CodeExt {
    fn into_error_kind(self) -> std::io::ErrorKind;
}

impl CodeExt for Code {
    fn into_error_kind(self) -> std::io::ErrorKind {
        match self {
            Self::Aborted => std::io::ErrorKind::Interrupted,
            Self::AlreadyExists => std::io::ErrorKind::AlreadyExists,
            Self::DeadlineExceeded => std::io::ErrorKind::TimedOut,
            Self::InvalidArgument => std::io::ErrorKind::InvalidInput,
            Self::NotFound => std::io::ErrorKind::NotFound,
            Self::PermissionDenied => std::io::ErrorKind::PermissionDenied,
            Self::Unavailable => std::io::ErrorKind::ConnectionRefused,
            _ => std::io::ErrorKind::Other,
        }
    }
}

trait ErrorKindExt {
    fn into_code(self) -> Code;
}

impl ErrorKindExt for std::io::ErrorKind {
    fn into_code(self) -> Code {
        match self {
            Self::NotFound => Code::NotFound,
            Self::PermissionDenied => Code::PermissionDenied,
            Self::ConnectionRefused | Self::ConnectionReset | Self::ConnectionAborted => {
                Code::Unavailable
            }
            Self::AlreadyExists => Code::AlreadyExists,
            Self::InvalidInput | Self::InvalidData => Code::InvalidArgument,
            Self::TimedOut => Code::DeadlineExceeded,
            Self::Interrupted => Code::Aborted,
            Self::NotConnected
            | Self::AddrInUse
            | Self::AddrNotAvailable
            | Self::BrokenPipe
            | Self::WouldBlock
            | Self::WriteZero
            | Self::Other
            | Self::UnexpectedEof => Code::Internal,
            _ => Code::Unknown,
        }
    }
}

// Serde definition for tonic::Code. See: https://serde.rs/remote-derive.html
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[serde(remote = "Code")]
pub enum CodeDef {
    Ok = 0,
    Cancelled = 1,
    Unknown = 2,
    InvalidArgument = 3,
    DeadlineExceeded = 4,
    NotFound = 5,
    AlreadyExists = 6,
    PermissionDenied = 7,
    ResourceExhausted = 8,
    FailedPrecondition = 9,
    Aborted = 10,
    OutOfRange = 11,
    Unimplemented = 12,
    Internal = 13,
    Unavailable = 14,
    DataLoss = 15,
    Unauthenticated = 16,
    // NOTE: Additional codes must be added to stores.rs in ErrorCodes and also
    // in both match statements in retry.rs.
}

Coverage Report

Created: 2026-08-24 08:15

Line	Count	Source
1		// Copyright 2024 The NativeLink Authors. All rights reserved.
2		//
3		// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4		// you may not use this file except in compliance with the License.
5		// You may obtain a copy of the License at
6		//
7		// See LICENSE file for details
8		//
9		// Unless required by applicable law or agreed to in writing, software
10		// distributed under the License is distributed on an "AS IS" BASIS,
11		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		// See the License for the specific language governing permissions and
13		// limitations under the License.
14
15		use core::convert::Into;
16		use core::str::Utf8Error;
17		use std::sync::{MutexGuard, PoisonError};
18
19		use nativelink_metric::{
20		MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent,
21		};
22		use prost_types::TimestampError;
23		use serde::{Deserialize, Serialize};
24		use tokio::sync::AcquireError;
25		// Reexport of tonic's error codes which we use as "nativelink_error::Code".
26		pub use tonic::Code;
27
28		#[macro_export]
29		macro_rules! make_err {
30		($code:expr, $($arg:tt)+) => {{
31		$crate::Error::new(
32		$code,
33		format!("{}", format_args!($($arg)+)),
34		)
35		}};
36		}
37
38		#[macro_export]
39		macro_rules! make_input_err {
40		($($arg:tt)+) => {{
41		$crate::make_err!($crate::Code::InvalidArgument, $($arg)+)
42		}};
43		}
44
45		#[macro_export]
46		macro_rules! error_if {
47		($cond:expr, $($arg:tt)+) => {{
48		if $cond {
49		Err($crate::make_err!($crate::Code::InvalidArgument, $($arg)+))?;
50		}
51		}};
52		}
53
54		/// Typed metadata that travels with an [`Error`].
55		///
56		/// Used in place of string-parsing error messages when the producer
57		/// has structured information the consumer needs to act on. The
58		/// motivating case is missing-blob errors from `fast_slow_store` —
59		/// `to_execute_response` reads [`ErrorContext::MissingDigest`] and
60		/// surfaces a `FAILED_PRECONDITION` with a `PreconditionFailure`
61		/// detail naming the digest, which Bazel auto-retries on.
62		///
63		/// Default is [`ErrorContext::None`]; existing call sites that
64		/// construct [`Error`] via `make_err!` / `Error::new` do not need to
65		/// be updated.
66		#[derive(Default, Debug, Eq, PartialEq, Clone, Serialize, Deserialize)]
67		pub enum ErrorContext {
68		#[default]
69		None,
70		/// The error refers to a specific CAS blob that could not be
71		/// located. `hash` and `size` together form the digest the client
72		/// should re-upload (`REv2` `blobs/{hash}/{size}`).
73		MissingDigest { hash: String, size: i64 },
74		}
75
76		#[derive(Eq, PartialEq, Clone, Serialize, Deserialize)]
77		pub struct Error {
78		#[serde(with = "CodeDef")]
79		pub code: Code,
80		pub messages: Vec<String>,
81		#[serde(default, skip_serializing_if = "ErrorContext::is_none")]
82		pub context: ErrorContext,
83		}
84
85		impl core::fmt::Debug for Error {
86	55	fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
87	55	let mut builder = f.debug_struct("Error");
88	55	builder.field("code", &self.code);
89	55	if !self.messages.is_empty() {
90	55	builder.field("messages", &self.messages);
91	55	}0
92	55	if !self.context.is_none() {
93	0	builder.field("context", &self.context);
94	55	}
95	55	builder.finish()
96	55	}
97		}
98
99		impl ErrorContext {
100		#[inline]
101		#[must_use]
102	57	pub const fn is_none(&self) -> bool {
103	57	matches!0 (self, Self::None)
104	57	}
105		}
106
107		impl MetricsComponent for Error {
108	1	fn publish(
109	1	&self,
110	1	kind: MetricKind,
111	1	field_metadata: MetricFieldData,
112	1	) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> {
113	1	self.to_string().publish(kind, field_metadata)
114	1	}
115		}
116
117		impl Error {
118		#[must_use]
119	397	pub const fn new_with_messages(code: Code, messages: Vec<String>) -> Self {
120	397	Self {
121	397	code,
122	397	messages,
123	397	context: ErrorContext::None,
124	397	}
125	397	}
126
127		#[must_use]
128	341	pub fn new(code: Code, msg: String) -> Self {
129	341	if msg.is_empty() {
130	0	Self::new_with_messages(code, vec![])
131		} else {
132	341	Self::new_with_messages(code, vec![msg])
133		}
134	341	}
135
136		#[must_use]
137	53	pub fn from_std_err(code: Code, mut err: &dyn core::error::Error) -> Self {
138	53	let mut messages = vec![format!("{err}")];
139	59	while let Some( src6 ) = err.source() {
140	6	messages.push(format!("{src}"));
141	6	err = src;
142	6	}
143	53	messages.reverse();
144	53	Self::new_with_messages(code, messages)
145	53	}
146
147		#[inline]
148		#[must_use]
149	74	pub fn append<S: Into<String>>(mut self, msg: S) -> Self {
150	74	self.messages.push(msg.into());
151	74	self
152	74	}
153
154		#[inline]
155		#[must_use]
156	0	pub fn with_context(mut self, context: ErrorContext) -> Self {
157	0	self.context = context;
158	0	self
159	0	}
160
161		#[must_use]
162	5	pub fn merge<E: Into<Self>>(mut self, other: E) -> Self {
163	5	let mut other: Self = other.into();
164		// This will help with knowing which messages are tied to different errors.
165	5	self.messages.push("---".to_string());
166	5	self.messages.append(&mut other.messages);
167	5	self
168	5	}
169
170		#[must_use]
171	23	pub fn merge_option<T: Into<Self>, U: Into<Self>>(
172	23	this: Option<T>,
173	23	other: Option<U>,
174	23	) -> Option<Self> {
175	23	if let Some( this5 ) = this {
176	5	if let Some( other1 ) = other {
177	1	return Some(this.into().merge(other));
178	4	}
179	4	return Some(this.into());
180	18	}
181	18	other.map(Into::into)
182	23	}
183
184		#[must_use]
185	2	pub fn to_std_err(self) -> std::io::Error {
186	2	std::io::Error::new(self.code.into_error_kind(), self.messages.join(" : "))
187	2	}
188
189		#[must_use]
190	35	pub fn message_string(&self) -> String {
191	35	self.messages.join(" : ")
192	35	}
193		}
194
195		impl core::error::Error for Error {}
196
197		impl From<Error> for nativelink_proto::google::rpc::Status {
198	15	fn from(val: Error) -> Self {
199	15	Self {
200	15	code: val.code as i32,
201	15	message: val.message_string(),
202	15	details: vec![],
203	15	}
204	15	}
205		}
206
207		impl From<nativelink_proto::google::rpc::Status> for Error {
208	4	fn from(val: nativelink_proto::google::rpc::Status) -> Self {
209	4	Self {
210	4	code: val.code.into(),
211	4	messages: vec![val.message],
212	4	context: ErrorContext::None,
213	4	}
214	4	}
215		}
216
217		impl core::fmt::Display for Error {
218	37	fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
219		// A manual impl to reduce the noise of frequently empty fields.
220	37	let mut builder = f.debug_struct("Error");
221
222	37	builder.field("code", &self.code);
223
224	37	if !self.messages.is_empty() {
225	37	builder.field("messages", &self.messages);
226	37	}0
227
228	37	builder.finish()
229	37	}
230		}
231
232		impl From<prost::DecodeError> for Error {
233	1	fn from(err: prost::DecodeError) -> Self {
234	1	Self::from_std_err(Code::Internal, &err)
235	1	}
236		}
237
238		impl From<prost::EncodeError> for Error {
239	0	fn from(err: prost::EncodeError) -> Self {
240	0	Self::from_std_err(Code::Internal, &err)
241	0	}
242		}
243
244		impl From<prost::UnknownEnumValue> for Error {
245	0	fn from(err: prost::UnknownEnumValue) -> Self {
246	0	Self::from_std_err(Code::Internal, &err)
247	0	}
248		}
249
250		impl From<core::num::TryFromIntError> for Error {
251	1	fn from(err: core::num::TryFromIntError) -> Self {
252	1	Self::from_std_err(Code::InvalidArgument, &err)
253	1	}
254		}
255
256		impl From<tokio::task::JoinError> for Error {
257	1	fn from(err: tokio::task::JoinError) -> Self {
258	1	Self::from_std_err(Code::Internal, &err)
259	1	}
260		}
261
262		impl<T> From<PoisonError<MutexGuard<'_, T>>> for Error {
263	0	fn from(err: PoisonError<MutexGuard<'_, T>>) -> Self {
264	0	Self::from_std_err(Code::Internal, &err)
265	0	}
266		}
267
268		impl From<serde_json5::Error> for Error {
269	0	fn from(err: serde_json5::Error) -> Self {
270	0	match err {
271	0	serde_json5::Error::Message { msg, location } => {
272	0	if let Some(has_location) = location {
273	0	make_err!(
274	0	Code::Internal,
275		"line {}, column {} - {}",
276		has_location.line,
277		has_location.column,
278		msg
279		)
280		} else {
281	0	Self::new(Code::Internal, msg)
282		}
283		}
284		}
285	0	}
286		}
287
288		impl From<core::num::ParseIntError> for Error {
289	2	fn from(err: core::num::ParseIntError) -> Self {
290	2	Self::from_std_err(Code::InvalidArgument, &err)
291	2	}
292		}
293
294		impl From<core::convert::Infallible> for Error {
295	0	fn from(_err: core::convert::Infallible) -> Self {
296		// Infallible is an error type that can never happen.
297	0	unreachable!();
298		}
299		}
300
301		impl From<TimestampError> for Error {
302	0	fn from(err: TimestampError) -> Self {
303	0	Self::from_std_err(Code::InvalidArgument, &err)
304	0	}
305		}
306
307		impl From<AcquireError> for Error {
308	0	fn from(err: AcquireError) -> Self {
309	0	Self::from_std_err(Code::Internal, &err)
310	0	}
311		}
312
313		impl From<Utf8Error> for Error {
314	0	fn from(err: Utf8Error) -> Self {
315	0	Self::from_std_err(Code::Internal, &err)
316	0	}
317		}
318
319		impl From<std::io::Error> for Error {
320	61	fn from(err: std::io::Error) -> Self {
321	61	Self {
322	61	code: err.kind().into_code(),
323	61	messages: vec![err.to_string()],
324	61	context: ErrorContext::None,
325	61	}
326	61	}
327		}
328
329		impl From<redis::RedisError> for Error {
330	29	fn from(error: redis::RedisError) -> Self {
331		use redis::ErrorKind::{
332		AuthenticationFailed, ClusterConnectionNotFound, EmptySentinelList,
333		InvalidClientConfig, Io as IoError, MasterNameNotFoundBySentinel,
334		NoValidReplicasFoundBySentinel, Parse as ParseError, RESP3NotSupported, Server,
335		UnexpectedReturnType,
336		};
337		use redis::ServerErrorKind::{
338		BusyLoading, ClusterDown, MasterDown, NoPerm, ReadOnly, TryAgain,
339		};
340
341		// Conversions here are based on https://grpc.github.io/grpc/core/md_doc_statuscodes.html.
342		//
343		// The distinction that matters most is retryable vs not. These statuses
344		// reach REAPI clients directly, and Bazel treats INVALID_ARGUMENT as
345		// permanent: it fails the build instantly rather than retrying. So
346		// anything caused by the state of the Redis deployment — a failover in
347		// progress, a dropped connection, a node still loading its dataset —
348		// must NOT land there, or a routine blip becomes a failed CI run.
349	29	let code = match error.kind() {
350	3	AuthenticationFailed \| Server(NoPerm) => Code::PermissionDenied,
351
352		// Topology in flux. Sentinel failover makes these normal and
353		// brief: the master moved and the client has not caught up yet.
354		// Surfacing MasterNameNotFoundBySentinel as INVALID_ARGUMENT is
355		// what turned a 1-2 minute sentinel failover into a dead build.
356		MasterNameNotFoundBySentinel
357		\| NoValidReplicasFoundBySentinel
358		\| ClusterConnectionNotFound
359		\| Server(ClusterDown \| MasterDown \| TryAgain \| BusyLoading \| ReadOnly) => {
360	9	Code::Unavailable
361		}
362
363		// A timeout is worth distinguishing from a dead connection, but
364		// both are transient and both are retryable.
365		IoError => {
366	6	if error.is_timeout() {
367	4	Code::DeadlineExceeded
368		} else {
369	2	Code::Unavailable
370		}
371		}
372
373		// Genuine misconfiguration by the operator — retrying cannot help.
374	4	InvalidClientConfig \| EmptySentinelList \| RESP3NotSupported => Code::InvalidArgument,
375
376		// Server-side or protocol faults. Not the caller's argument, so
377		// not InvalidArgument: a malformed reply is our problem or the
378		// server's, and it is at least worth a retry.
379	2	ParseError \| UnexpectedReturnType => Code::Internal,
380
381	5	_ => Code::Unknown,
382		};
383
384	29	let kind = error.kind();
385	29	make_err!(code, "{kind:?}: {error}")
386	29	}
387		}
388
389		impl From<tonic::Status> for Error {
390	7	fn from(status: tonic::Status) -> Self {
391	7	Self::new(status.code(), status.to_string())
392	7	}
393		}
394
395		impl From<Error> for tonic::Status {
396	30	fn from(val: Error) -> Self {
397	30	Self::new(val.code, val.messages.join(" : "))
398	30	}
399		}
400
401		impl From<walkdir::Error> for Error {
402	1	fn from(err: walkdir::Error) -> Self {
403	1	Self::from_std_err(Code::Internal, &err)
404	1	}
405		}
406
407		impl From<uuid::Error> for Error {
408	0	fn from(err: uuid::Error) -> Self {
409	0	Self::from_std_err(Code::Internal, &err)
410	0	}
411		}
412
413		impl From<rustls_pki_types::pem::Error> for Error {
414	0	fn from(err: rustls_pki_types::pem::Error) -> Self {
415	0	Self::from_std_err(Code::Internal, &err)
416	0	}
417		}
418
419		impl From<tokio::time::error::Elapsed> for Error {
420	0	fn from(err: tokio::time::error::Elapsed) -> Self {
421	0	Self::from_std_err(Code::DeadlineExceeded, &err)
422	0	}
423		}
424
425		impl From<url::ParseError> for Error {
426	0	fn from(err: url::ParseError) -> Self {
427	0	Self::from_std_err(Code::Internal, &err)
428	0	}
429		}
430
431		impl From<mongodb::error::Error> for Error {
432	0	fn from(err: mongodb::error::Error) -> Self {
433	0	Self::from_std_err(Code::Internal, &err)
434	0	}
435		}
436
437		impl From<reqwest::Error> for Error {
438	0	fn from(err: reqwest::Error) -> Self {
439	0	Self::from_std_err(Code::Internal, &err)
440	0	}
441		}
442
443		impl From<zip::result::ZipError> for Error {
444	0	fn from(err: zip::result::ZipError) -> Self {
445	0	Self::from_std_err(Code::Internal, &err)
446	0	}
447		}
448
449		impl From<std::ffi::NulError> for Error {
450	0	fn from(err: std::ffi::NulError) -> Self {
451	0	Self::from_std_err(Code::Internal, &err)
452	0	}
453		}
454
455		impl From<base64::DecodeError> for Error {
456	0	fn from(err: base64::DecodeError) -> Self {
457	0	Self::from_std_err(Code::Internal, &err)
458	0	}
459		}
460
461		pub trait ResultExt<T> {
462		/// # Errors
463		///
464		/// Will return `Err` if we can't convert the error.
465		fn err_tip_with_code<F, S>(self, tip_fn: F) -> Result<T, Error>
466		where
467		Self: Sized,
468		S: ToString,
469		F: (FnOnce(&Error) -> (Code, S)) + Sized;
470
471		/// # Errors
472		///
473		/// Will return `Err` if we can't convert the error.
474		#[inline]
475	405k	fn err_tip<F, S>(self, tip_fn: F) -> Result<T, Error>
476	405k	where
477	405k	Self: Sized,
478	405k	S: ToString,
479	405k	F: (FnOnce() -> S) + Sized,
480		{
481	405k	self.err_tip_with_code(\|e\| ( e.code251 , tip_fn()251 ))
482	405k	}
483
484		/// # Errors
485		///
486		/// Will return `Err` if we can't merge the errors.
487	0	fn merge<U>(self, _other: Result<U, Error>) -> Result<U, Error>
488	0	where
489	0	Self: Sized,
490		{
491	0	unreachable!();
492		}
493		}
494
495		impl<T, E: Into<Error>> ResultExt<T> for Result<T, E> {
496		#[inline]
497	402k	fn err_tip_with_code<F, S>(self, tip_fn: F) -> Result<T, Error>
498	402k	where
499	402k	Self: Sized,
500	402k	S: ToString,
501	402k	F: (FnOnce(&Error) -> (Code, S)) + Sized,
502		{
503	402k	self.map_err(\|e\| {238
504	238	let mut error: Error = e.into();
505	238	let (code, message) = tip_fn(&error);
506	238	error.code = code;
507	238	error.messages.push(message.to_string());
508	238	error
509	238	})
510	402k	}
511
512	11.3k	fn merge<U>(self, other: Result<U, Error>) -> Result<U, Error>
513	11.3k	where
514	11.3k	Self: Sized,
515		{
516	11.3k	if let Err( e44 ) = self {
517	44	let mut e: Error = e.into();
518	44	if let Err( other_err40 ) = other {
519	40	let mut other_err: Error = other_err;
520	40	// This will help with knowing which messages are tied to different errors.
521	40	e.messages.push("---".to_string());
522	40	e.messages.append(&mut other_err.messages);
523	40	}4
524	44	return Err(e);
525	11.2k	}
526	11.2k	other
527	11.3k	}
528		}
529
530		impl<T> ResultExt<T> for Option<T> {
531		#[inline]
532	9.33k	fn err_tip_with_code<F, S>(self, tip_fn: F) -> Result<T, Error>
533	9.33k	where
534	9.33k	Self: Sized,
535	9.33k	S: ToString,
536	9.33k	F: (FnOnce(&Error) -> (Code, S)) + Sized,
537		{
538	9.33k	self.ok_or_else(\|\| {30
539	30	let mut error = Error {
540	30	code: Code::Internal,
541	30	messages: vec![],
542	30	context: ErrorContext::None,
543	30	};
544	30	let (code, message) = tip_fn(&error);
545	30	error.code = code;
546	30	error.messages.push(message.to_string());
547	30	error
548	30	})
549	9.33k	}
550		}
551
552		trait CodeExt {
553		fn into_error_kind(self) -> std::io::ErrorKind;
554		}
555
556		impl CodeExt for Code {
557	2	fn into_error_kind(self) -> std::io::ErrorKind {
558	2	match self {
559	0	Self::Aborted => std::io::ErrorKind::Interrupted,
560	0	Self::AlreadyExists => std::io::ErrorKind::AlreadyExists,
561	0	Self::DeadlineExceeded => std::io::ErrorKind::TimedOut,
562	0	Self::InvalidArgument => std::io::ErrorKind::InvalidInput,
563	0	Self::NotFound => std::io::ErrorKind::NotFound,
564	0	Self::PermissionDenied => std::io::ErrorKind::PermissionDenied,
565	0	Self::Unavailable => std::io::ErrorKind::ConnectionRefused,
566	2	_ => std::io::ErrorKind::Other,
567		}
568	2	}
569		}
570
571		trait ErrorKindExt {
572		fn into_code(self) -> Code;
573		}
574
575		impl ErrorKindExt for std::io::ErrorKind {
576	61	fn into_code(self) -> Code {
577	61	match self {
578	58	Self::NotFound => Code::NotFound,
579	0	Self::PermissionDenied => Code::PermissionDenied,
580		Self::ConnectionRefused \| Self::ConnectionReset \| Self::ConnectionAborted => {
581	0	Code::Unavailable
582		}
583	1	Self::AlreadyExists => Code::AlreadyExists,
584	0	Self::InvalidInput \| Self::InvalidData => Code::InvalidArgument,
585	0	Self::TimedOut => Code::DeadlineExceeded,
586	0	Self::Interrupted => Code::Aborted,
587		Self::NotConnected
588		\| Self::AddrInUse
589		\| Self::AddrNotAvailable
590		\| Self::BrokenPipe
591		\| Self::WouldBlock
592		\| Self::WriteZero
593		\| Self::Other
594	1	\| Self::UnexpectedEof => Code::Internal,
595	1	_ => Code::Unknown,
596		}
597	61	}
598		}
599
600		// Serde definition for tonic::Code. See: https://serde.rs/remote-derive.html
601		#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
602		#[serde(remote = "Code")]
603		pub enum CodeDef {
604		Ok = 0,
605		Cancelled = 1,
606		Unknown = 2,
607		InvalidArgument = 3,
608		DeadlineExceeded = 4,
609		NotFound = 5,
610		AlreadyExists = 6,
611		PermissionDenied = 7,
612		ResourceExhausted = 8,
613		FailedPrecondition = 9,
614		Aborted = 10,
615		OutOfRange = 11,
616		Unimplemented = 12,
617		Internal = 13,
618		Unavailable = 14,
619		DataLoss = 15,
620		Unauthenticated = 16,
621		// NOTE: Additional codes must be added to stores.rs in ErrorCodes and also
622		// in both match statements in retry.rs.
623		}