Coverage Report

Created: 2026-11-13 23:29

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-store/src/common_s3_utils.rs
Line
Count
Source
1
// Copyright 2025 Trace Machina, Inc. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::future::Future;
16
use core::pin::Pin;
17
use core::task::{Context, Poll};
18
use std::sync::Arc;
19
20
use aws_config::retry::ErrorKind::TransientError;
21
use aws_smithy_runtime_api::client::http::{
22
    HttpClient as SmithyHttpClient, HttpConnector as SmithyHttpConnector, HttpConnectorFuture,
23
    HttpConnectorSettings, SharedHttpConnector,
24
};
25
use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
26
use aws_smithy_runtime_api::client::result::ConnectorError;
27
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
28
use aws_smithy_runtime_api::http::Response;
29
use aws_smithy_types::body::SdkBody;
30
use bytes::{Bytes, BytesMut};
31
use futures::Stream;
32
use http_body::{Frame, SizeHint};
33
use http_body_util::BodyExt;
34
use hyper::{Method, Request};
35
use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder};
36
use hyper_util::client::legacy::Client as LegacyClient;
37
use hyper_util::client::legacy::connect::HttpConnector as LegacyHttpConnector;
38
use hyper_util::rt::TokioExecutor;
39
use nativelink_config::stores::CommonObjectSpec;
40
// Note: S3 store should be very careful about the error codes it returns
41
// when in a retryable wrapper. Always prefer Code::Aborted or another
42
// retryable code over Code::InvalidArgument or make_input_err!().
43
// ie: Don't import make_input_err!() to help prevent this.
44
use nativelink_error::{Code, Error, make_err};
45
use nativelink_util::buf_channel::DropCloserReadHalf;
46
use nativelink_util::fs;
47
use nativelink_util::retry::{Retrier, RetryResult};
48
use tokio::time::sleep;
49
50
39
pub(crate) fn install_default_rustls_crypto_provider() {
51
39
    drop(rustls::crypto::ring::default_provider().install_default());
52
39
}
53
54
#[derive(Clone)]
55
pub struct TlsClient {
56
    client: LegacyClient<HttpsConnector<LegacyHttpConnector>, SdkBody>,
57
    retrier: Retrier,
58
}
59
60
impl TlsClient {
61
5
    pub fn new(common: &CommonObjectSpec) -> Result<Self, Error> {
62
5
        Self::new_with_http_support(common, common.insecure_allow_http)
63
5
    }
64
65
    /// Creates a client for AWS credential discovery.
66
    ///
67
    /// The default credential chain uses plain HTTP for link-local metadata
68
    /// services, including the EKS Pod Identity, ECS, and EC2 providers.
69
2
    pub fn new_for_credentials(common: &CommonObjectSpec) -> Result<Self, Error> {
70
2
        Self::new_with_http_support(common, true)
71
2
    }
72
73
7
    fn new_with_http_support(common: &CommonObjectSpec, allow_http: bool) -> Result<Self, Error> {
74
7
        install_default_rustls_crypto_provider();
75
76
7
        let 
connector_with_roots5
= HttpsConnectorBuilder::new()
77
7
            .try_with_platform_verifier()
78
7
            .map_err(|e| 
{2
79
2
                make_err!(
80
2
                    Code::InvalidArgument,
81
                    "Failed to load CA root certificates for the TLS client: {e}. \
82
                     Mount a CA bundle into the container and point SSL_CERT_FILE \
83
                     or SSL_CERT_DIR at it."
84
                )
85
2
            })?;
86
87
5
        let connector_with_schemes = if allow_http {
88
1
            connector_with_roots.https_or_http()
89
        } else {
90
4
            connector_with_roots.https_only()
91
        };
92
93
5
        let connector = if common.disable_http2 {
94
0
            connector_with_schemes.enable_http1().build()
95
        } else {
96
5
            connector_with_schemes.enable_http1().enable_http2().build()
97
        };
98
99
5
        Ok(Self::with_https_connector(common, connector))
100
7
    }
101
102
8
    pub fn with_https_connector(
103
8
        common: &CommonObjectSpec,
104
8
        connector: HttpsConnector<LegacyHttpConnector>,
105
8
    ) -> Self {
106
8
        install_default_rustls_crypto_provider();
107
108
8
        let client = LegacyClient::builder(TokioExecutor::new()).build(connector);
109
110
        Self {
111
8
            client,
112
8
            retrier: Retrier::new(
113
8
                Arc::new(|duration| 
Box::pin0
(
sleep0
(
duration0
))),
114
8
                common.retry.make_jitter_fn(),
115
8
                common.retry.clone(),
116
            )
117
8
            .with_client("s3_http"),
118
        }
119
8
    }
120
}
121
122
impl core::fmt::Debug for TlsClient {
123
42
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> {
124
42
        f.debug_struct("TlsClient").finish_non_exhaustive()
125
42
    }
126
}
127
128
impl SmithyHttpClient for TlsClient {
129
6
    fn http_connector(
130
6
        &self,
131
6
        _settings: &HttpConnectorSettings,
132
6
        _components: &RuntimeComponents,
133
6
    ) -> SharedHttpConnector {
134
6
        SharedHttpConnector::new(self.clone())
135
6
    }
136
}
137
138
struct RequestBuilder<'a> {
139
    components: &'a RequestComponents,
140
}
141
142
impl<'a> RequestBuilder<'a> {
143
    #[inline]
144
8
    const fn new(components: &'a RequestComponents) -> Self {
145
8
        Self { components }
146
8
    }
147
148
    #[inline]
149
    #[allow(unused_qualifications, reason = "false positive on hyper::http::Error")]
150
8
    fn build(&self) -> Result<Request<SdkBody>, hyper::http::Error> {
151
8
        let mut req_builder = Request::builder()
152
8
            .method(self.components.method.clone())
153
8
            .uri(self.components.uri.clone())
154
8
            .version(self.components.version);
155
156
8
        let headers_map = req_builder.headers_mut().unwrap();
157
18
        for (name, value) in 
&self.components.headers8
{
158
18
            headers_map.insert(name, value.clone());
159
18
        }
160
161
8
        match &self.components.body_data {
162
6
            BufferedBodyState::Cloneable(body) => {
163
6
                let cloned_body = body.try_clone().expect("Body should be cloneable");
164
6
                req_builder.body(cloned_body)
165
            }
166
0
            BufferedBodyState::Buffered(bytes) => req_builder.body(SdkBody::from(bytes.clone())),
167
2
            BufferedBodyState::Empty => req_builder.body(SdkBody::empty()),
168
        }
169
8
    }
170
}
171
172
mod execution {
173
    use super::conversions::ResponseExt;
174
    use super::{
175
        Code, HttpsConnector, LegacyClient, LegacyHttpConnector, RequestBuilder, RequestComponents,
176
        Response, RetryResult, SdkBody, fs, make_err,
177
    };
178
179
    #[inline]
180
8
    pub(crate) async fn execute_request(
181
8
        client: LegacyClient<HttpsConnector<LegacyHttpConnector>, SdkBody>,
182
8
        components: &RequestComponents,
183
8
    ) -> RetryResult<Response<SdkBody>> {
184
8
        let _permit = match fs::get_permit().await {
185
8
            Ok(permit) => permit,
186
0
            Err(e) => {
187
0
                return RetryResult::Retry(make_err!(
188
0
                    Code::Unavailable,
189
0
                    "Failed to acquire permit: {e}"
190
0
                ));
191
            }
192
        };
193
194
8
        let request = match RequestBuilder::new(components).build() {
195
8
            Ok(req) => req,
196
0
            Err(e) => {
197
0
                return RetryResult::Err(make_err!(
198
0
                    Code::Internal,
199
0
                    "Failed to create request: {e}",
200
0
                ));
201
            }
202
        };
203
204
8
        match client.request(request).await {
205
1
            Ok(resp) => RetryResult::Ok(resp.into_smithy_response()),
206
7
            Err(e) => RetryResult::Retry(make_err!(
207
7
                Code::Unavailable,
208
7
                "Failed request in S3Store: {e}"
209
7
            )),
210
        }
211
8
    }
212
213
    #[inline]
214
8
    pub(crate) fn create_retry_stream(
215
8
        client: LegacyClient<HttpsConnector<LegacyHttpConnector>, SdkBody>,
216
8
        components: RequestComponents,
217
8
    ) -> impl futures::Stream<Item = RetryResult<Response<SdkBody>>> {
218
8
        futures::stream::unfold(components, move |components| {
219
8
            let client_clone = client.clone();
220
8
            async move {
221
8
                let result = execute_request(client_clone, &components).await;
222
223
8
                Some((result, components))
224
8
            }
225
8
        })
226
8
    }
227
}
228
229
enum BufferedBodyState {
230
    Cloneable(SdkBody),
231
    Buffered(Bytes),
232
    Empty,
233
}
234
235
mod body_processing {
236
    use super::{BodyExt, BufferedBodyState, BytesMut, ConnectorError, SdkBody, TransientError};
237
238
    /// Buffer a request body fully into memory.
239
    ///
240
    /// TODO(aaronmondal): This could lead to OOMs in extremely constrained
241
    ///                    environments. Probably better to implement something
242
    ///                    like a rewindable stream logic.
243
    #[inline]
244
0
    pub(crate) async fn buffer_body(body: SdkBody) -> Result<BufferedBodyState, ConnectorError> {
245
0
        let mut bytes = BytesMut::new();
246
0
        let mut body_stream = body;
247
0
        while let Some(frame) = body_stream.frame().await {
248
0
            match frame {
249
0
                Ok(frame) => {
250
0
                    if let Some(data) = frame.data_ref() {
251
0
                        bytes.extend_from_slice(data);
252
0
                    }
253
                }
254
0
                Err(e) => {
255
0
                    return Err(ConnectorError::other(
256
0
                        format!("Failed to read request body: {e}").into(),
257
0
                        Some(TransientError),
258
0
                    ));
259
                }
260
            }
261
        }
262
263
0
        Ok(BufferedBodyState::Buffered(bytes.freeze()))
264
0
    }
265
}
266
267
pub(crate) struct RequestComponents {
268
    method: Method,
269
    uri: hyper::Uri,
270
    version: hyper::Version,
271
    headers: hyper::HeaderMap,
272
    body_data: BufferedBodyState,
273
}
274
275
mod conversions {
276
    use super::{
277
        BufferedBodyState, ConnectorError, Future, HttpRequest, Method, RequestComponents,
278
        Response, SdkBody, TransientError, body_processing,
279
    };
280
281
    pub(crate) trait RequestExt {
282
        fn into_components(self)
283
        -> impl Future<Output = Result<RequestComponents, ConnectorError>>;
284
    }
285
286
    impl RequestExt for HttpRequest {
287
8
        async fn into_components(self) -> Result<RequestComponents, ConnectorError> {
288
            // Note: This does *not* refer the the HTTP protocol, but to the
289
            //       version of the http crate.
290
8
            let hyper_req = self.try_into_http1x().map_err(|e| 
{0
291
0
                ConnectorError::other(
292
0
                    format!("Failed to convert to HTTP request: {e}").into(),
293
0
                    Some(TransientError),
294
                )
295
0
            })?;
296
297
8
            let method = hyper_req.method().clone();
298
8
            let uri = hyper_req.uri().clone();
299
8
            let version = hyper_req.version();
300
8
            let headers = hyper_req.headers().clone();
301
302
8
            let body = hyper_req.into_body();
303
304
            // Only buffer bodies for methods likely to have payloads.
305
8
            let needs_buffering = 
matches!2
(method, Method::POST | Method::PUT);
306
307
            // Preserve the body in case we need to retry.
308
8
            let body_data = if needs_buffering {
309
6
                if let Some(cloneable_body) = body.try_clone() {
310
6
                    BufferedBodyState::Cloneable(cloneable_body)
311
                } else {
312
0
                    body_processing::buffer_body(body).await?
313
                }
314
            } else {
315
2
                BufferedBodyState::Empty
316
            };
317
318
8
            Ok(RequestComponents {
319
8
                method,
320
8
                uri,
321
8
                version,
322
8
                headers,
323
8
                body_data,
324
8
            })
325
8
        }
326
    }
327
328
    pub(crate) trait ResponseExt {
329
        fn into_smithy_response(self) -> Response<SdkBody>;
330
    }
331
332
    impl ResponseExt for hyper::Response<hyper::body::Incoming> {
333
1
        fn into_smithy_response(self) -> Response<SdkBody> {
334
1
            let (parts, body) = self.into_parts();
335
1
            let sdk_body = SdkBody::from_body_1_x(body);
336
1
            let mut smithy_resp = Response::new(parts.status.into(), sdk_body);
337
1
            let header_pairs: Vec<(String, String)> = parts
338
1
                .headers
339
1
                .iter()
340
1
                .filter_map(|(name, value)| {
341
1
                    value
342
1
                        .to_str()
343
1
                        .ok()
344
1
                        .map(|value_str| (name.as_str().to_owned(), value_str.to_owned()))
345
1
                })
346
1
                .collect();
347
348
1
            for (name, value) in header_pairs {
349
1
                smithy_resp.headers_mut().insert(name, value);
350
1
            }
351
352
1
            smithy_resp
353
1
        }
354
    }
355
}
356
357
impl SmithyHttpConnector for TlsClient {
358
8
    fn call(&self, req: HttpRequest) -> HttpConnectorFuture {
359
        use conversions::RequestExt;
360
361
8
        let client = self.client.clone();
362
8
        let retrier = self.retrier.clone();
363
364
8
        HttpConnectorFuture::new(Box::pin(async move {
365
8
            let components = req.into_components().await
?0
;
366
367
8
            let retry_stream = execution::create_retry_stream(client, components);
368
369
8
            match retrier.retry(retry_stream).await {
370
1
                Ok(response) => Ok(response),
371
7
                Err(e) => Err(ConnectorError::other(
372
7
                    format!("Connection failed after retries: {e}").into(),
373
7
                    Some(TransientError),
374
7
                )),
375
            }
376
8
        }))
377
8
    }
378
}
379
380
#[derive(Debug)]
381
pub struct BodyWrapper {
382
    pub reader: DropCloserReadHalf,
383
    pub size: u64,
384
}
385
386
impl http_body::Body for BodyWrapper {
387
    type Data = Bytes;
388
    type Error = std::io::Error;
389
390
17
    fn poll_frame(
391
17
        self: Pin<&mut Self>,
392
17
        cx: &mut Context<'_>,
393
17
    ) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
394
17
        let reader = Pin::new(&mut Pin::get_mut(self).reader);
395
17
        reader
396
17
            .poll_next(cx)
397
17
            .map(|maybe_bytes_res| maybe_bytes_res.map(|res| 
res1
.
map1
(Frame::data)))
398
17
    }
399
400
10
    fn size_hint(&self) -> SizeHint {
401
10
        SizeHint::with_exact(self.size)
402
10
    }
403
}