Coverage Report

Created: 2026-11-13 23:29

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-util/src/common.rs
Line
Count
Source
1
// Copyright 2024 Trace Machina, Inc. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::cmp::{Eq, Ordering};
16
use core::hash::{BuildHasher, Hash};
17
use core::ops::{Deref, DerefMut};
18
use std::collections::HashMap;
19
use std::io::{Cursor, Write};
20
use std::{env, fmt};
21
22
use bytes::{Buf, BufMut, Bytes, BytesMut};
23
use nativelink_error::{Error, ResultExt, make_input_err};
24
use nativelink_metric::{
25
    MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent,
26
};
27
use nativelink_proto::build::bazel::remote::execution::v2::Digest;
28
use prost::Message;
29
use rand::Rng;
30
use serde::de::Visitor;
31
use serde::ser::Error as _;
32
use serde::{Deserialize, Deserializer, Serialize, Serializer};
33
use tonic::Code;
34
use tracing::error;
35
36
pub use crate::fs;
37
38
0
#[derive(Default, Clone, Copy, Eq, PartialEq, Hash, wincode::SchemaRead, wincode::SchemaWrite)]
39
#[repr(C)]
40
pub struct DigestInfo {
41
    /// Raw hash in packed form.
42
    packed_hash: PackedHash,
43
44
    /// Possibly the size of the digest in bytes.
45
    size_bytes: u64,
46
}
47
48
impl MetricsComponent for DigestInfo {
49
0
    fn publish(
50
0
        &self,
51
0
        _kind: MetricKind,
52
0
        field_metadata: MetricFieldData,
53
0
    ) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> {
54
0
        format!("{self}").publish(MetricKind::String, field_metadata)
55
0
    }
56
}
57
58
impl DigestInfo {
59
83.5k
    pub const fn new(packed_hash: [u8; 32], size_bytes: u64) -> Self {
60
83.5k
        Self {
61
83.5k
            size_bytes,
62
83.5k
            packed_hash: PackedHash(packed_hash),
63
83.5k
        }
64
83.5k
    }
65
66
2.20M
    pub fn try_new<T>(hash: &str, size_bytes: T) -> Result<Self, Error>
67
2.20M
    where
68
2.20M
        T: TryInto<u64> + fmt::Display + Copy,
69
    {
70
2.20M
        let packed_hash =
71
2.20M
            PackedHash::from_hex(hash).err_tip(|| 
format!9
("Invalid sha256 hash: {hash}"))
?9
;
72
2.20M
        let size_bytes = size_bytes
73
2.20M
            .try_into()
74
2.20M
            .map_err(|_| 
make_input_err!0
("Could not convert {} into u64", size_bytes))
?0
;
75
        // The proto `Digest` takes an i64, so to keep compatibility
76
        // we only allow sizes that can fit into an i64.
77
2.20M
        if size_bytes > i64::MAX as u64 {
78
2
            return Err(make_input_err!(
79
2
                "Size bytes is too large: {} - max: {}",
80
2
                size_bytes,
81
2
                i64::MAX
82
2
            ));
83
2.20M
        }
84
2.20M
        Ok(Self {
85
2.20M
            packed_hash,
86
2.20M
            size_bytes,
87
2.20M
        })
88
2.20M
    }
89
90
186
    pub const fn zero_digest() -> Self {
91
186
        Self {
92
186
            size_bytes: 0,
93
186
            packed_hash: PackedHash::new(),
94
186
        }
95
186
    }
96
97
61.4k
    pub const fn packed_hash(&self) -> &PackedHash {
98
61.4k
        &self.packed_hash
99
61.4k
    }
100
101
2.16k
    pub const fn set_packed_hash(&mut self, packed_hash: [u8; 32]) {
102
2.16k
        self.packed_hash = PackedHash(packed_hash);
103
2.16k
    }
104
105
216k
    pub const fn size_bytes(&self) -> u64 {
106
216k
        self.size_bytes
107
216k
    }
108
109
    /// Returns a struct that can turn the `DigestInfo` into a string.
110
79.0k
    const fn stringifier(&self) -> DigestStackStringifier<'_> {
111
79.0k
        DigestStackStringifier::new(self)
112
79.0k
    }
113
}
114
115
/// Counts the number of digits a number needs if it were to be
116
/// converted to a string.
117
0
const fn count_digits(mut num: u64) -> usize {
118
0
    let mut count = 0;
119
0
    while num != 0 {
120
0
        count += 1;
121
0
        num /= 10;
122
0
    }
123
0
    count
124
0
}
125
126
/// An optimized version of a function that can convert a `DigestInfo`
127
/// into a str on the stack.
128
struct DigestStackStringifier<'a> {
129
    digest: &'a DigestInfo,
130
    /// Buffer that can hold the string representation of the `DigestInfo`.
131
    /// - Hex is '2 * sizeof(PackedHash)'.
132
    /// - Digits can be at most `count_digits(u64::MAX)`.
133
    /// - We also have a hyphen separator.
134
    buf: [u8; size_of::<PackedHash>() * 2 + count_digits(u64::MAX) + 1],
135
}
136
137
impl<'a> DigestStackStringifier<'a> {
138
79.0k
    const fn new(digest: &'a DigestInfo) -> Self {
139
79.0k
        DigestStackStringifier {
140
79.0k
            digest,
141
79.0k
            buf: [b'-'; size_of::<PackedHash>() * 2 + count_digits(u64::MAX) + 1],
142
79.0k
        }
143
79.0k
    }
144
145
79.0k
    fn as_str(&mut self) -> Result<&str, Error> {
146
        // Populate the buffer and return the amount of bytes written
147
        // to the buffer.
148
79.0k
        let len = {
149
79.0k
            let mut cursor = Cursor::new(&mut self.buf[..]);
150
79.0k
            let hex = self.digest.packed_hash.to_hex().map_err(|e| 
{0
151
0
                Error::from_std_err(Code::InvalidArgument, &e).append(format!(
152
                    "Could not convert PackedHash to hex - {:?}",
153
                    self.digest
154
                ))
155
0
            })?;
156
79.0k
            cursor
157
79.0k
                .write_all(&hex)
158
79.0k
                .err_tip(|| 
format!0
("Could not write hex to buffer - {hex:?} - {hex:?}"))
?0
;
159
            // Note: We already have a hyphen at this point because we
160
            // initialized the buffer with hyphens.
161
79.0k
            cursor.advance(1);
162
79.0k
            cursor
163
79.0k
                .write_fmt(format_args!("{}", self.digest.size_bytes()))
164
79.0k
                .err_tip(|| 
format!0
("Could not write size_bytes to buffer - {hex:?}"))
?0
;
165
79.0k
            cursor.position().try_into().map_err(|e| 
{0
166
0
                Error::from_std_err(Code::InvalidArgument, &e)
167
0
                    .append("Cursor position exceeds usize bounds")
168
0
            })?
169
        };
170
        // Convert the buffer into utf8 string.
171
79.0k
        core::str::from_utf8(&self.buf[..len]).map_err(|e| 
{0
172
0
            Error::from_std_err(Code::InvalidArgument, &e).append(format!(
173
                "Could not convert [u8] to string - {} - {:?}",
174
                self.digest, self.buf
175
            ))
176
0
        })
177
79.0k
    }
178
}
179
180
/// Custom serializer for `DigestInfo` because the default Serializer
181
/// would try to encode the data as a byte array, but we use {hex}-{size}.
182
impl Serialize for DigestInfo {
183
34.8k
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
184
34.8k
    where
185
34.8k
        S: Serializer,
186
    {
187
34.8k
        let mut stringifier = self.stringifier();
188
34.8k
        serializer.serialize_str(
189
34.8k
            stringifier
190
34.8k
                .as_str()
191
34.8k
                .err_tip(|| "During serialization of DigestInfo")
192
34.8k
                .map_err(S::Error::custom)
?0
,
193
        )
194
34.8k
    }
195
}
196
197
/// Custom deserializer for `DigestInfo` because the default Deserializer
198
/// would try to decode the data as a byte array, but we use {hex}-{size}.
199
impl<'de> Deserialize<'de> for DigestInfo {
200
2.19M
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
201
2.19M
    where
202
2.19M
        D: Deserializer<'de>,
203
    {
204
        struct DigestInfoVisitor;
205
        impl Visitor<'_> for DigestInfoVisitor {
206
            type Value = DigestInfo;
207
208
0
            fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
209
0
                formatter.write_str("a string representing a DigestInfo")
210
0
            }
211
212
2.19M
            fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>
213
2.19M
            where
214
2.19M
                E: serde::de::Error,
215
            {
216
2.19M
                let Some((
hash2.19M
,
size2.19M
)) = s.split_once('-') else {
217
26
                    return Err(E::custom(
218
26
                        "Invalid DigestInfo format, expected '-' separator",
219
26
                    ));
220
                };
221
2.19M
                let 
size_bytes2.19M
= size
222
2.19M
                    .parse::<u64>()
223
2.19M
                    .map_err(|e| 
E::custom1
(
format!1
("Could not parse size_bytes: {e:?}")))
?1
;
224
2.19M
                DigestInfo::try_new(hash, size_bytes)
225
2.19M
                    .map_err(|e| 
E::custom1
(
format!1
("Could not create DigestInfo: {e:?}")))
226
2.19M
            }
227
        }
228
2.19M
        deserializer.deserialize_str(DigestInfoVisitor)
229
2.19M
    }
230
}
231
232
impl fmt::Display for DigestInfo {
233
33.7k
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
234
33.7k
        let mut stringifier = self.stringifier();
235
33.7k
        f.write_str(
236
33.7k
            stringifier
237
33.7k
                .as_str()
238
33.7k
                .err_tip(|| "During serialization of DigestInfo")
239
33.7k
                .map_err(|e| 
{0
240
0
                    error!("Could not convert DigestInfo to string - {e:?}");
241
0
                    fmt::Error
242
0
                })?,
243
        )
244
33.7k
    }
245
}
246
247
impl fmt::Debug for DigestInfo {
248
10.5k
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
249
10.5k
        let mut stringifier = self.stringifier();
250
10.5k
        match stringifier.as_str() {
251
10.5k
            Ok(s) => f.debug_tuple("DigestInfo").field(&s).finish(),
252
0
            Err(e) => {
253
0
                error!("Could not convert DigestInfo to string - {e:?}");
254
0
                Err(fmt::Error)
255
            }
256
        }
257
10.5k
    }
258
}
259
260
impl Ord for DigestInfo {
261
1
    fn cmp(&self, other: &Self) -> Ordering {
262
1
        self.packed_hash
263
1
            .cmp(&other.packed_hash)
264
1
            .then_with(|| self.size_bytes.cmp(&other.size_bytes))
265
1
    }
266
}
267
268
impl PartialOrd for DigestInfo {
269
0
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
270
0
        Some(self.cmp(other))
271
0
    }
272
}
273
274
impl TryFrom<Digest> for DigestInfo {
275
    type Error = Error;
276
277
114k
    fn try_from(digest: Digest) -> Result<Self, Self::Error> {
278
114k
        let 
packed_hash114k
= PackedHash::from_hex(&digest.hash)
279
114k
            .err_tip(|| 
format!1
("Invalid sha256 hash: {}", digest.hash))
?1
;
280
114k
        let size_bytes = digest.size_bytes.try_into().map_err(|e| 
{0
281
0
            Error::from_std_err(Code::InvalidArgument, &e)
282
0
                .append(format!("Could not convert {} into u64", digest.size_bytes))
283
0
        })?;
284
114k
        Ok(Self {
285
114k
            packed_hash,
286
114k
            size_bytes,
287
114k
        })
288
114k
    }
289
}
290
291
impl TryFrom<&Digest> for DigestInfo {
292
    type Error = Error;
293
294
23.2k
    fn try_from(digest: &Digest) -> Result<Self, Self::Error> {
295
23.2k
        let packed_hash = PackedHash::from_hex(&digest.hash)
296
23.2k
            .err_tip(|| 
format!0
("Invalid sha256 hash: {}", digest.hash))
?0
;
297
23.2k
        let size_bytes = digest.size_bytes.try_into().map_err(|e| 
{0
298
0
            Error::from_std_err(Code::InvalidArgument, &e)
299
0
                .append(format!("Could not convert {} into u64", digest.size_bytes))
300
0
        })?;
301
23.2k
        Ok(Self {
302
23.2k
            packed_hash,
303
23.2k
            size_bytes,
304
23.2k
        })
305
23.2k
    }
306
}
307
308
impl From<DigestInfo> for Digest {
309
143k
    fn from(val: DigestInfo) -> Self {
310
        Self {
311
143k
            hash: val.packed_hash.to_string(),
312
143k
            size_bytes: val.size_bytes.try_into().unwrap_or_else(|e| 
{0
313
0
                error!("Could not convert {} into u64 - {e:?}", val.size_bytes);
314
                // This is a placeholder value that can help a user identify
315
                // that the conversion failed.
316
0
                -255
317
0
            }),
318
        }
319
143k
    }
320
}
321
322
impl From<&DigestInfo> for Digest {
323
272
    fn from(val: &DigestInfo) -> Self {
324
        Self {
325
272
            hash: val.packed_hash.to_string(),
326
272
            size_bytes: val.size_bytes.try_into().unwrap_or_else(|e| 
{0
327
0
                error!("Could not convert {} into u64 - {e:?}", val.size_bytes);
328
                // This is a placeholder value that can help a user identify
329
                // that the conversion failed.
330
0
                -255
331
0
            }),
332
        }
333
272
    }
334
}
335
336
#[derive(
337
    Debug,
338
    Serialize,
339
    Deserialize,
340
    Default,
341
    Clone,
342
    Copy,
343
    Eq,
344
    PartialEq,
345
    Hash,
346
    PartialOrd,
347
    Ord,
348
    wincode::SchemaRead,
349
0
    wincode::SchemaWrite,
350
)]
351
pub struct PackedHash([u8; 32]);
352
353
const SIZE_OF_PACKED_HASH: usize = 32;
354
impl PackedHash {
355
186
    const fn new() -> Self {
356
186
        Self([0; SIZE_OF_PACKED_HASH])
357
186
    }
358
359
2.33M
    fn from_hex(hash: &str) -> Result<Self, Error> {
360
2.33M
        let mut packed_hash = [0u8; 32];
361
2.33M
        hex::decode_to_slice(hash, &mut packed_hash).map_err(|e| 
{10
362
10
            Error::from_std_err(Code::InvalidArgument, &e)
363
10
                .append(format!("Invalid sha256 hash: {hash}"))
364
10
        })?;
365
2.33M
        Ok(Self(packed_hash))
366
2.33M
    }
367
368
    /// Converts the packed hash into a hex string.
369
    #[inline]
370
240k
    fn to_hex(self) -> Result<[u8; SIZE_OF_PACKED_HASH * 2], fmt::Error> {
371
240k
        let mut hash = [0u8; SIZE_OF_PACKED_HASH * 2];
372
240k
        hex::encode_to_slice(self.0, &mut hash).map_err(|e| 
{0
373
0
            error!("Could not convert PackedHash to hex - {e:?} - {:?}", self.0);
374
0
            fmt::Error
375
0
        })?;
376
240k
        Ok(hash)
377
240k
    }
378
}
379
380
impl fmt::Display for PackedHash {
381
161k
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
382
161k
        let hash = self.to_hex()
?0
;
383
161k
        match core::str::from_utf8(&hash) {
384
161k
            Ok(hash) => f.write_str(hash)
?0
,
385
0
            Err(_) => f.write_str(&format!("Could not convert hash to utf8 {:?}", self.0))?,
386
        }
387
161k
        Ok(())
388
161k
    }
389
}
390
391
impl Deref for PackedHash {
392
    type Target = [u8; 32];
393
394
0
    fn deref(&self) -> &Self::Target {
395
0
        &self.0
396
0
    }
397
}
398
399
impl DerefMut for PackedHash {
400
2.16k
    fn deref_mut(&mut self) -> &mut Self::Target {
401
2.16k
        &mut self.0
402
2.16k
    }
403
}
404
405
// Simple utility trait that makes it easier to apply `.try_map` to Vec.
406
// This will convert one vector into another vector with a different type.
407
pub trait VecExt<T> {
408
    fn try_map<F, U>(self, f: F) -> Result<Vec<U>, Error>
409
    where
410
        Self: Sized,
411
        F: (Fn(T) -> Result<U, Error>) + Sized;
412
}
413
414
impl<T> VecExt<T> for Vec<T> {
415
40.1k
    fn try_map<F, U>(self, f: F) -> Result<Vec<U>, Error>
416
40.1k
    where
417
40.1k
        Self: Sized,
418
40.1k
        F: (Fn(T) -> Result<U, Error>) + Sized,
419
    {
420
40.1k
        let mut output = Vec::with_capacity(self.len());
421
40.1k
        for 
item6
in self {
422
6
            output.push((f)(item)
?0
);
423
        }
424
40.1k
        Ok(output)
425
40.1k
    }
426
}
427
428
// Simple utility trait that makes it easier to apply `.try_map` to HashMap.
429
// This will convert one HashMap into another keeping the key the same, but
430
// different value type.
431
pub trait HashMapExt<K: Eq + Hash, T, S: BuildHasher> {
432
    fn try_map<F, U>(self, f: F) -> Result<HashMap<K, U, S>, Error>
433
    where
434
        Self: Sized,
435
        F: (Fn(T) -> Result<U, Error>) + Sized;
436
}
437
438
impl<K: Eq + Hash, T, S: BuildHasher + Clone> HashMapExt<K, T, S> for HashMap<K, T, S> {
439
10.0k
    fn try_map<F, U>(self, f: F) -> Result<HashMap<K, U, S>, Error>
440
10.0k
    where
441
10.0k
        Self: Sized,
442
10.0k
        F: (Fn(T) -> Result<U, Error>) + Sized,
443
    {
444
10.0k
        let mut output = HashMap::with_capacity_and_hasher(self.len(), (*self.hasher()).clone());
445
10.0k
        for (
k2
,
v2
) in self {
446
2
            output.insert(k, (f)(v)
?0
);
447
        }
448
10.0k
        Ok(output)
449
10.0k
    }
450
}
451
452
// Utility to encode our proto into GRPC stream format.
453
12.7k
pub fn encode_stream_proto<T: Message>(proto: &T) -> Result<Bytes, Box<dyn core::error::Error>> {
454
    // See below comment on spec.
455
    use core::mem::size_of;
456
    const PREFIX_BYTES: usize = size_of::<u8>() + size_of::<u32>();
457
458
12.7k
    let mut buf = BytesMut::new();
459
460
63.8k
    for _ in 
0..PREFIX_BYTES12.7k
{
461
63.8k
        // Advance our buffer first.
462
63.8k
        // We will backfill it once we know the size of the message.
463
63.8k
        buf.put_u8(0);
464
63.8k
    }
465
12.7k
    proto.encode(&mut buf)
?0
;
466
12.7k
    let len = buf.len() - PREFIX_BYTES;
467
    {
468
12.7k
        let mut buf = &mut buf[0..PREFIX_BYTES];
469
        // See: https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#:~:text=Compressed-Flag
470
        // for more details on spec.
471
        // Compressed-Flag -> 0 / 1 # encoded as 1 byte unsigned integer.
472
12.7k
        buf.put_u8(0);
473
        // Message-Length -> {length of Message} # encoded as 4 byte unsigned integer (big endian).
474
12.7k
        buf.put_u32(u32::try_from(len)
?0
);
475
        // Message -> *{binary octet}.
476
    }
477
478
12.7k
    Ok(buf.freeze())
479
12.7k
}
480
481
/// Small utility to reseed the global RNG.
482
/// Done this way because we use it in a macro
483
/// and macro's can't load external crates.
484
#[inline]
485
1.35k
pub fn reseed_rng_for_test() -> Result<(), Error> {
486
1.35k
    rand::rng()
487
1.35k
        .reseed()
488
1.35k
        .map_err(|e| 
Error::from_std_err0
(
Code::InvalidArgument0
,
&e0
).
append0
("Could not reseed RNG"))
489
1.35k
}
490
491
/// Get temporary path from either `TEST_TMPDIR` or best effort temp directory if
492
/// not set.
493
473
pub fn make_temp_path(data: &str) -> String {
494
    #[cfg(target_family = "unix")]
495
473
    return format!(
496
        "{}/{}/{}",
497
473
        env::var("TEST_TMPDIR").unwrap_or_else(|_| env::temp_dir().to_str().unwrap().to_string()),
498
473
        rand::rng().random::<u64>(),
499
        data
500
    );
501
    #[cfg(target_family = "windows")]
502
    return format!(
503
        "{}\\{}\\{}",
504
        env::var("TEST_TMPDIR").unwrap_or_else(|_| env::temp_dir().to_str().unwrap().to_string()),
505
        rand::rng().random::<u64>(),
506
        data
507
    );
508
473
}
509
510
// Constant for PreconditionFailure
511
pub const VIOLATION_TYPE_MISSING: &str = "MISSING";