Coverage Report

Created: 2026-09-18 20:40

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-store/src/common_s3_utils.rs
Line
Count
Source
1
// Copyright 2025 The NativeLink Authors. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::future::Future;
16
use core::pin::Pin;
17
use core::task::{Context, Poll};
18
use std::sync::Arc;
19
20
use aws_config::retry::ErrorKind::TransientError;
21
use aws_smithy_runtime_api::client::http::{
22
    HttpClient as SmithyHttpClient, HttpConnector as SmithyHttpConnector, HttpConnectorFuture,
23
    HttpConnectorSettings, SharedHttpConnector,
24
};
25
use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
26
use aws_smithy_runtime_api::client::result::ConnectorError;
27
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
28
use aws_smithy_runtime_api::http::Response;
29
use aws_smithy_types::body::SdkBody;
30
use bytes::{Bytes, BytesMut};
31
use futures::Stream;
32
use http_body::{Frame, SizeHint};
33
use http_body_util::BodyExt;
34
use hyper::{Method, Request};
35
use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder};
36
use hyper_util::client::legacy::Client as LegacyClient;
37
use hyper_util::client::legacy::connect::HttpConnector as LegacyHttpConnector;
38
use hyper_util::rt::TokioExecutor;
39
use nativelink_config::stores::CommonObjectSpec;
40
// Note: S3 store should be very careful about the error codes it returns
41
// when in a retryable wrapper. Always prefer Code::Aborted or another
42
// retryable code over Code::InvalidArgument or make_input_err!().
43
// ie: Don't import make_input_err!() to help prevent this.
44
use nativelink_error::{Code, Error, make_err};
45
use nativelink_util::buf_channel::DropCloserReadHalf;
46
use nativelink_util::fs;
47
use nativelink_util::retry::{Retrier, RetryResult};
48
use tokio::time::sleep;
49
50
39
pub(crate) fn install_default_rustls_crypto_provider() {
51
39
    drop(rustls::crypto::ring::default_provider().install_default());
52
39
}
53
54
#[derive(Clone)]
55
pub struct TlsClient {
56
    client: LegacyClient<HttpsConnector<LegacyHttpConnector>, SdkBody>,
57
    retrier: Retrier,
58
}
59
60
impl TlsClient {
61
5
    pub fn new(common: &CommonObjectSpec) -> Result<Self, Error> {
62
5
        Self::new_with_http_support(common, common.insecure_allow_http)
63
5
    }
64
65
    /// Creates a client for AWS credential discovery.
66
    ///
67
    /// The default credential chain uses plain HTTP for link-local metadata
68
    /// services, including the EKS Pod Identity, ECS, and EC2 providers.
69
2
    pub fn new_for_credentials(common: &CommonObjectSpec) -> Result<Self, Error> {
70
2
        Self::new_with_http_support(common, true)
71
2
    }
72
73
7
    fn new_with_http_support(common: &CommonObjectSpec, allow_http: bool) -> Result<Self, Error> {
74
7
        install_default_rustls_crypto_provider();
75
76
7
        let 
connector_with_roots5
= HttpsConnectorBuilder::new()
77
7
            .try_with_platform_verifier()
78
7
            .map_err(|e| 
{2
79
2
                make_err!(
80
2
                    Code::InvalidArgument,
81
                    "Failed to load CA root certificates for the TLS client: {e}. \
82
                     Mount a CA bundle into the container and point SSL_CERT_FILE \
83
                     or SSL_CERT_DIR at it."
84
                )
85
2
            })?;
86
87
5
        let connector_with_schemes = if allow_http {
88
1
            connector_with_roots.https_or_http()
89
        } else {
90
4
            connector_with_roots.https_only()
91
        };
92
93
5
        let connector = if common.disable_http2 {
94
0
            connector_with_schemes.enable_http1().build()
95
        } else {
96
5
            connector_with_schemes.enable_http1().enable_http2().build()
97
        };
98
99
5
        Ok(Self::with_https_connector(common, connector))
100
7
    }
101
102
8
    pub fn with_https_connector(
103
8
        common: &CommonObjectSpec,
104
8
        connector: HttpsConnector<LegacyHttpConnector>,
105
8
    ) -> Self {
106
8
        install_default_rustls_crypto_provider();
107
108
8
        let client = LegacyClient::builder(TokioExecutor::new()).build(connector);
109
110
        Self {
111
8
            client,
112
8
            retrier: Retrier::new(
113
8
                Arc::new(|duration| 
Box::pin0
(
sleep0
(
duration0
))),
114
8
                common.retry.make_jitter_fn(),
115
8
                common.retry.clone(),
116
            ),
117
        }
118
8
    }
119
}
120
121
impl core::fmt::Debug for TlsClient {
122
42
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> {
123
42
        f.debug_struct("TlsClient").finish_non_exhaustive()
124
42
    }
125
}
126
127
impl SmithyHttpClient for TlsClient {
128
6
    fn http_connector(
129
6
        &self,
130
6
        _settings: &HttpConnectorSettings,
131
6
        _components: &RuntimeComponents,
132
6
    ) -> SharedHttpConnector {
133
6
        SharedHttpConnector::new(self.clone())
134
6
    }
135
}
136
137
struct RequestBuilder<'a> {
138
    components: &'a RequestComponents,
139
}
140
141
impl<'a> RequestBuilder<'a> {
142
    #[inline]
143
8
    const fn new(components: &'a RequestComponents) -> Self {
144
8
        Self { components }
145
8
    }
146
147
    #[inline]
148
    #[allow(unused_qualifications, reason = "false positive on hyper::http::Error")]
149
8
    fn build(&self) -> Result<Request<SdkBody>, hyper::http::Error> {
150
8
        let mut req_builder = Request::builder()
151
8
            .method(self.components.method.clone())
152
8
            .uri(self.components.uri.clone())
153
8
            .version(self.components.version);
154
155
8
        let headers_map = req_builder.headers_mut().unwrap();
156
18
        for (name, value) in 
&self.components.headers8
{
157
18
            headers_map.insert(name, value.clone());
158
18
        }
159
160
8
        match &self.components.body_data {
161
6
            BufferedBodyState::Cloneable(body) => {
162
6
                let cloned_body = body.try_clone().expect("Body should be cloneable");
163
6
                req_builder.body(cloned_body)
164
            }
165
0
            BufferedBodyState::Buffered(bytes) => req_builder.body(SdkBody::from(bytes.clone())),
166
2
            BufferedBodyState::Empty => req_builder.body(SdkBody::empty()),
167
        }
168
8
    }
169
}
170
171
mod execution {
172
    use super::conversions::ResponseExt;
173
    use super::{
174
        Code, HttpsConnector, LegacyClient, LegacyHttpConnector, RequestBuilder, RequestComponents,
175
        Response, RetryResult, SdkBody, fs, make_err,
176
    };
177
178
    #[inline]
179
8
    pub(crate) async fn execute_request(
180
8
        client: LegacyClient<HttpsConnector<LegacyHttpConnector>, SdkBody>,
181
8
        components: &RequestComponents,
182
8
    ) -> RetryResult<Response<SdkBody>> {
183
8
        let _permit = match fs::get_permit().await {
184
8
            Ok(permit) => permit,
185
0
            Err(e) => {
186
0
                return RetryResult::Retry(make_err!(
187
0
                    Code::Unavailable,
188
0
                    "Failed to acquire permit: {e}"
189
0
                ));
190
            }
191
        };
192
193
8
        let request = match RequestBuilder::new(components).build() {
194
8
            Ok(req) => req,
195
0
            Err(e) => {
196
0
                return RetryResult::Err(make_err!(
197
0
                    Code::Internal,
198
0
                    "Failed to create request: {e}",
199
0
                ));
200
            }
201
        };
202
203
8
        match client.request(request).await {
204
1
            Ok(resp) => RetryResult::Ok(resp.into_smithy_response()),
205
7
            Err(e) => RetryResult::Retry(make_err!(
206
7
                Code::Unavailable,
207
7
                "Failed request in S3Store: {e}"
208
7
            )),
209
        }
210
8
    }
211
212
    #[inline]
213
8
    pub(crate) fn create_retry_stream(
214
8
        client: LegacyClient<HttpsConnector<LegacyHttpConnector>, SdkBody>,
215
8
        components: RequestComponents,
216
8
    ) -> impl futures::Stream<Item = RetryResult<Response<SdkBody>>> {
217
8
        futures::stream::unfold(components, move |components| {
218
8
            let client_clone = client.clone();
219
8
            async move {
220
8
                let result = execute_request(client_clone, &components).await;
221
222
8
                Some((result, components))
223
8
            }
224
8
        })
225
8
    }
226
}
227
228
enum BufferedBodyState {
229
    Cloneable(SdkBody),
230
    Buffered(Bytes),
231
    Empty,
232
}
233
234
mod body_processing {
235
    use super::{BodyExt, BufferedBodyState, BytesMut, ConnectorError, SdkBody, TransientError};
236
237
    /// Buffer a request body fully into memory.
238
    ///
239
    /// TODO(aaronmondal): This could lead to OOMs in extremely constrained
240
    ///                    environments. Probably better to implement something
241
    ///                    like a rewindable stream logic.
242
    #[inline]
243
0
    pub(crate) async fn buffer_body(body: SdkBody) -> Result<BufferedBodyState, ConnectorError> {
244
0
        let mut bytes = BytesMut::new();
245
0
        let mut body_stream = body;
246
0
        while let Some(frame) = body_stream.frame().await {
247
0
            match frame {
248
0
                Ok(frame) => {
249
0
                    if let Some(data) = frame.data_ref() {
250
0
                        bytes.extend_from_slice(data);
251
0
                    }
252
                }
253
0
                Err(e) => {
254
0
                    return Err(ConnectorError::other(
255
0
                        format!("Failed to read request body: {e}").into(),
256
0
                        Some(TransientError),
257
0
                    ));
258
                }
259
            }
260
        }
261
262
0
        Ok(BufferedBodyState::Buffered(bytes.freeze()))
263
0
    }
264
}
265
266
pub(crate) struct RequestComponents {
267
    method: Method,
268
    uri: hyper::Uri,
269
    version: hyper::Version,
270
    headers: hyper::HeaderMap,
271
    body_data: BufferedBodyState,
272
}
273
274
mod conversions {
275
    use super::{
276
        BufferedBodyState, ConnectorError, Future, HttpRequest, Method, RequestComponents,
277
        Response, SdkBody, TransientError, body_processing,
278
    };
279
280
    pub(crate) trait RequestExt {
281
        fn into_components(self)
282
        -> impl Future<Output = Result<RequestComponents, ConnectorError>>;
283
    }
284
285
    impl RequestExt for HttpRequest {
286
8
        async fn into_components(self) -> Result<RequestComponents, ConnectorError> {
287
            // Note: This does *not* refer the the HTTP protocol, but to the
288
            //       version of the http crate.
289
8
            let hyper_req = self.try_into_http1x().map_err(|e| 
{0
290
0
                ConnectorError::other(
291
0
                    format!("Failed to convert to HTTP request: {e}").into(),
292
0
                    Some(TransientError),
293
                )
294
0
            })?;
295
296
8
            let method = hyper_req.method().clone();
297
8
            let uri = hyper_req.uri().clone();
298
8
            let version = hyper_req.version();
299
8
            let headers = hyper_req.headers().clone();
300
301
8
            let body = hyper_req.into_body();
302
303
            // Only buffer bodies for methods likely to have payloads.
304
8
            let needs_buffering = 
matches!2
(method, Method::POST | Method::PUT);
305
306
            // Preserve the body in case we need to retry.
307
8
            let body_data = if needs_buffering {
308
6
                if let Some(cloneable_body) = body.try_clone() {
309
6
                    BufferedBodyState::Cloneable(cloneable_body)
310
                } else {
311
0
                    body_processing::buffer_body(body).await?
312
                }
313
            } else {
314
2
                BufferedBodyState::Empty
315
            };
316
317
8
            Ok(RequestComponents {
318
8
                method,
319
8
                uri,
320
8
                version,
321
8
                headers,
322
8
                body_data,
323
8
            })
324
8
        }
325
    }
326
327
    pub(crate) trait ResponseExt {
328
        fn into_smithy_response(self) -> Response<SdkBody>;
329
    }
330
331
    impl ResponseExt for hyper::Response<hyper::body::Incoming> {
332
1
        fn into_smithy_response(self) -> Response<SdkBody> {
333
1
            let (parts, body) = self.into_parts();
334
1
            let sdk_body = SdkBody::from_body_1_x(body);
335
1
            let mut smithy_resp = Response::new(parts.status.into(), sdk_body);
336
1
            let header_pairs: Vec<(String, String)> = parts
337
1
                .headers
338
1
                .iter()
339
1
                .filter_map(|(name, value)| {
340
1
                    value
341
1
                        .to_str()
342
1
                        .ok()
343
1
                        .map(|value_str| (name.as_str().to_owned(), value_str.to_owned()))
344
1
                })
345
1
                .collect();
346
347
1
            for (name, value) in header_pairs {
348
1
                smithy_resp.headers_mut().insert(name, value);
349
1
            }
350
351
1
            smithy_resp
352
1
        }
353
    }
354
}
355
356
impl SmithyHttpConnector for TlsClient {
357
8
    fn call(&self, req: HttpRequest) -> HttpConnectorFuture {
358
        use conversions::RequestExt;
359
360
8
        let client = self.client.clone();
361
8
        let retrier = self.retrier.clone();
362
363
8
        HttpConnectorFuture::new(Box::pin(async move {
364
8
            let components = req.into_components().await
?0
;
365
366
8
            let retry_stream = execution::create_retry_stream(client, components);
367
368
8
            match retrier.retry(retry_stream).await {
369
1
                Ok(response) => Ok(response),
370
7
                Err(e) => Err(ConnectorError::other(
371
7
                    format!("Connection failed after retries: {e}").into(),
372
7
                    Some(TransientError),
373
7
                )),
374
            }
375
8
        }))
376
8
    }
377
}
378
379
#[derive(Debug)]
380
pub struct BodyWrapper {
381
    pub reader: DropCloserReadHalf,
382
    pub size: u64,
383
}
384
385
impl http_body::Body for BodyWrapper {
386
    type Data = Bytes;
387
    type Error = std::io::Error;
388
389
91
    fn poll_frame(
390
91
        self: Pin<&mut Self>,
391
91
        cx: &mut Context<'_>,
392
91
    ) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
393
91
        let reader = Pin::new(&mut Pin::get_mut(self).reader);
394
91
        reader
395
91
            .poll_next(cx)
396
91
            .map(|maybe_bytes_res| 
maybe_bytes_res66
.
map66
(|res|
res50
.
map50
(Frame::data)))
397
91
    }
398
399
10
    fn size_hint(&self) -> SizeHint {
400
10
        SizeHint::with_exact(self.size)
401
10
    }
402
}