Coverage Report

Created: 2025-05-30 16:37

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-util/src/common.rs
Line
Count
Source
1
// Copyright 2024 The NativeLink Authors. All rights reserved.
2
//
3
// Licensed under the Apache License, Version 2.0 (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    http://www.apache.org/licenses/LICENSE-2.0
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::cmp::{Eq, Ordering};
16
use core::hash::{BuildHasher, Hash};
17
use core::ops::{Deref, DerefMut};
18
use std::collections::HashMap;
19
use std::fmt;
20
use std::io::{Cursor, Write};
21
22
use bytes::{Buf, BufMut, Bytes, BytesMut};
23
use nativelink_error::{Error, ResultExt, make_input_err};
24
use nativelink_metric::{
25
    MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent,
26
};
27
use nativelink_proto::build::bazel::remote::execution::v2::Digest;
28
use prost::Message;
29
use serde::de::Visitor;
30
use serde::ser::Error as _;
31
use serde::{Deserialize, Deserializer, Serialize, Serializer};
32
use tracing::error;
33
34
pub use crate::fs;
35
36
#[derive(Default, Clone, Copy, Eq, PartialEq, Hash)]
37
#[repr(C)]
38
pub struct DigestInfo {
39
    /// Raw hash in packed form.
40
    packed_hash: PackedHash,
41
42
    /// Possibly the size of the digest in bytes.
43
    size_bytes: u64,
44
}
45
46
impl MetricsComponent for DigestInfo {
47
0
    fn publish(
48
0
        &self,
49
0
        _kind: MetricKind,
50
0
        field_metadata: MetricFieldData,
51
0
    ) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> {
52
0
        format!("{self}").publish(MetricKind::String, field_metadata)
53
0
    }
54
}
55
56
impl DigestInfo {
57
5.39k
    pub const fn new(packed_hash: [u8; 32], size_bytes: u64) -> Self {
58
5.39k
        Self {
59
5.39k
            size_bytes,
60
5.39k
            packed_hash: PackedHash(packed_hash),
61
5.39k
        }
62
5.39k
    }
63
64
5.26k
    pub fn try_new<T>(hash: &str, size_bytes: T) -> Result<Self, Error>
65
5.26k
    where
66
5.26k
        T: TryInto<u64> + fmt::Display + Copy,
67
    {
68
5.25k
        let packed_hash =
69
5.26k
            PackedHash::from_hex(hash).err_tip(|| format!(
"Invalid sha256 hash: {hash}"9
))
?9
;
70
5.25k
        let size_bytes = size_bytes
71
5.25k
            .try_into()
72
5.25k
            .map_err(|_| make_input_err!("Could not convert {} into u64", size_bytes))
?0
;
73
        // The proto `Digest` takes an i64, so to keep compatibility
74
        // we only allow sizes that can fit into an i64.
75
5.25k
        if size_bytes > i64::MAX as u64 {
  Branch (75:12): [True: 0, False: 3]
  Branch (75:12): [True: 0, False: 8]
  Branch (75:12): [True: 0, False: 21]
  Branch (75:12): [True: 0, False: 4]
  Branch (75:12): [True: 0, False: 2]
  Branch (75:12): [True: 0, False: 4.58k]
  Branch (75:12): [Folded - Ignored]
  Branch (75:12): [True: 0, False: 1]
  Branch (75:12): [True: 0, False: 8]
  Branch (75:12): [True: 0, False: 1]
  Branch (75:12): [True: 0, False: 6]
  Branch (75:12): [True: 2, False: 5]
  Branch (75:12): [True: 0, False: 6]
  Branch (75:12): [True: 0, False: 9]
  Branch (75:12): [True: 0, False: 58]
  Branch (75:12): [True: 0, False: 4]
  Branch (75:12): [True: 0, False: 1]
  Branch (75:12): [True: 0, False: 5]
  Branch (75:12): [True: 0, False: 16]
  Branch (75:12): [True: 0, False: 9]
  Branch (75:12): [True: 0, False: 2]
  Branch (75:12): [True: 0, False: 405]
  Branch (75:12): [True: 0, False: 1]
  Branch (75:12): [True: 0, False: 2]
  Branch (75:12): [Folded - Ignored]
  Branch (75:12): [True: 0, False: 6]
  Branch (75:12): [True: 0, False: 6]
  Branch (75:12): [True: 0, False: 27]
  Branch (75:12): [True: 0, False: 3]
  Branch (75:12): [True: 0, False: 3]
  Branch (75:12): [True: 0, False: 4]
  Branch (75:12): [True: 0, False: 3]
  Branch (75:12): [True: 0, False: 13]
  Branch (75:12): [True: 0, False: 12]
  Branch (75:12): [True: 0, False: 10]
76
2
            return Err(make_input_err!(
77
2
                "Size bytes is too large: {} - max: {}",
78
2
                size_bytes,
79
2
                i64::MAX
80
2
            ));
81
5.25k
        }
82
5.25k
        Ok(Self {
83
5.25k
            packed_hash,
84
5.25k
            size_bytes,
85
5.25k
        })
86
5.26k
    }
87
88
0
    pub const fn zero_digest() -> Self {
89
0
        Self {
90
0
            size_bytes: 0,
91
0
            packed_hash: PackedHash::new(),
92
0
        }
93
0
    }
94
95
40.2k
    pub const fn packed_hash(&self) -> &PackedHash {
96
40.2k
        &self.packed_hash
97
40.2k
    }
98
99
135
    pub const fn set_packed_hash(&mut self, packed_hash: [u8; 32]) {
100
135
        self.packed_hash = PackedHash(packed_hash);
101
135
    }
102
103
15.9k
    pub const fn size_bytes(&self) -> u64 {
104
15.9k
        self.size_bytes
105
15.9k
    }
106
107
    /// Returns a struct that can turn the `DigestInfo` into a string.
108
680
    const fn stringifier(&self) -> DigestStackStringifier<'_> {
109
680
        DigestStackStringifier::new(self)
110
680
    }
111
}
112
113
/// Counts the number of digits a number needs if it were to be
114
/// converted to a string.
115
0
const fn count_digits(mut num: u64) -> usize {
116
0
    let mut count = 0;
117
0
    while num != 0 {
  Branch (117:11): [Folded - Ignored]
  Branch (117:11): [Folded - Ignored]
118
0
        count += 1;
119
0
        num /= 10;
120
0
    }
121
0
    count
122
0
}
123
124
/// An optimized version of a function that can convert a `DigestInfo`
125
/// into a str on the stack.
126
struct DigestStackStringifier<'a> {
127
    digest: &'a DigestInfo,
128
    /// Buffer that can hold the string representation of the `DigestInfo`.
129
    /// - Hex is '2 * sizeof(PackedHash)'.
130
    /// - Digits can be at most `count_digits(u64::MAX)`.
131
    /// - We also have a hyphen separator.
132
    buf: [u8; size_of::<PackedHash>() * 2 + count_digits(u64::MAX) + 1],
133
}
134
135
impl<'a> DigestStackStringifier<'a> {
136
680
    const fn new(digest: &'a DigestInfo) -> Self {
137
680
        DigestStackStringifier {
138
680
            digest,
139
680
            buf: [b'-'; size_of::<PackedHash>() * 2 + count_digits(u64::MAX) + 1],
140
680
        }
141
680
    }
142
143
680
    fn as_str(&mut self) -> Result<&str, Error> {
144
        // Populate the buffer and return the amount of bytes written
145
        // to the buffer.
146
680
        let len = {
147
680
            let mut cursor = Cursor::new(&mut self.buf[..]);
148
680
            let hex = self.digest.packed_hash.to_hex().map_err(|e| 
{0
149
0
                make_input_err!(
150
                    "Could not convert PackedHash to hex - {e:?} - {:?}",
151
                    self.digest
152
                )
153
0
            })?;
154
680
            cursor
155
680
                .write_all(&hex)
156
680
                .err_tip(|| format!(
"Could not write hex to buffer - {hex:?} - {hex:?}"0
,))
?0
;
157
            // Note: We already have a hyphen at this point because we
158
            // initialized the buffer with hyphens.
159
680
            cursor.advance(1);
160
680
            cursor
161
680
                .write_fmt(format_args!("{}", self.digest.size_bytes()))
162
680
                .err_tip(|| format!(
"Could not write size_bytes to buffer - {hex:?}"0
,))
?0
;
163
680
            cursor.position() as usize
164
        };
165
        // Convert the buffer into utf8 string.
166
680
        core::str::from_utf8(&self.buf[..len]).map_err(|e| 
{0
167
0
            make_input_err!(
168
                "Could not convert [u8] to string - {} - {:?} - {:?}",
169
                self.digest,
170
                self.buf,
171
                e,
172
            )
173
0
        })
174
680
    }
175
}
176
177
/// Custom serializer for `DigestInfo` because the default Serializer
178
/// would try to encode the data as a byte array, but we use {hex}-{size}.
179
impl Serialize for DigestInfo {
180
110
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
181
110
    where
182
110
        S: Serializer,
183
    {
184
110
        let mut stringifier = self.stringifier();
185
110
        serializer.serialize_str(
186
110
            stringifier
187
110
                .as_str()
188
110
                .err_tip(|| "During serialization of DigestInfo")
189
110
                .map_err(S::Error::custom)
?0
,
190
        )
191
110
    }
192
}
193
194
/// Custom deserializer for `DigestInfo` because the default Deserializer
195
/// would try to decode the data as a byte array, but we use {hex}-{size}.
196
impl<'de> Deserialize<'de> for DigestInfo {
197
4.60k
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
198
4.60k
    where
199
4.60k
        D: Deserializer<'de>,
200
    {
201
        struct DigestInfoVisitor;
202
        impl Visitor<'_> for DigestInfoVisitor {
203
            type Value = DigestInfo;
204
205
0
            fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
206
0
                formatter.write_str("a string representing a DigestInfo")
207
0
            }
208
209
4.60k
            fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>
210
4.60k
            where
211
4.60k
                E: serde::de::Error,
212
            {
213
4.60k
                let Some((hash, size)) = s.split_once('-') else {
  Branch (213:21): [True: 8, False: 0]
  Branch (213:21): [True: 4.58k, False: 0]
  Branch (213:21): [Folded - Ignored]
  Branch (213:21): [True: 3, False: 0]
  Branch (213:21): [Folded - Ignored]
214
0
                    return Err(E::custom(
215
0
                        "Invalid DigestInfo format, expected '-' separator",
216
0
                    ));
217
                };
218
4.60k
                let size_bytes = size
219
4.60k
                    .parse::<u64>()
220
4.60k
                    .map_err(|e| 
E::custom0
(
format!0
(
"Could not parse size_bytes: {e:?}"0
)))
?0
;
221
4.60k
                DigestInfo::try_new(hash, size_bytes)
222
4.60k
                    .map_err(|e| 
E::custom1
(
format!1
(
"Could not create DigestInfo: {e:?}"1
)))
223
4.60k
            }
224
        }
225
4.60k
        deserializer.deserialize_str(DigestInfoVisitor)
226
4.60k
    }
227
}
228
229
impl fmt::Display for DigestInfo {
230
535
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
231
535
        let mut stringifier = self.stringifier();
232
535
        f.write_str(
233
535
            stringifier
234
535
                .as_str()
235
535
                .err_tip(|| "During serialization of DigestInfo")
236
535
                .map_err(|e| 
{0
237
0
                    error!("Could not convert DigestInfo to string - {e:?}");
238
0
                    fmt::Error
239
0
                })?,
240
        )
241
535
    }
242
}
243
244
impl fmt::Debug for DigestInfo {
245
35
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
246
35
        let mut stringifier = self.stringifier();
247
35
        match stringifier.as_str() {
248
35
            Ok(s) => f.debug_tuple("DigestInfo").field(&s).finish(),
249
0
            Err(e) => {
250
0
                error!("Could not convert DigestInfo to string - {e:?}");
251
0
                Err(fmt::Error)
252
            }
253
        }
254
35
    }
255
}
256
257
impl Ord for DigestInfo {
258
0
    fn cmp(&self, other: &Self) -> Ordering {
259
0
        self.packed_hash
260
0
            .cmp(&other.packed_hash)
261
0
            .then_with(|| self.size_bytes.cmp(&other.size_bytes))
262
0
    }
263
}
264
265
impl PartialOrd for DigestInfo {
266
0
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
267
0
        Some(self.cmp(other))
268
0
    }
269
}
270
271
impl TryFrom<Digest> for DigestInfo {
272
    type Error = Error;
273
274
192
    fn try_from(digest: Digest) -> Result<Self, Self::Error> {
275
192
        let 
packed_hash191
= PackedHash::from_hex(&digest.hash)
276
192
            .err_tip(|| format!(
"Invalid sha256 hash: {}"1
, digest.hash))
?1
;
277
191
        let size_bytes = digest
278
191
            .size_bytes
279
191
            .try_into()
280
191
            .map_err(|_| make_input_err!("Could not convert {} into u64", digest.size_bytes))
?0
;
281
191
        Ok(Self {
282
191
            packed_hash,
283
191
            size_bytes,
284
191
        })
285
192
    }
286
}
287
288
impl TryFrom<&Digest> for DigestInfo {
289
    type Error = Error;
290
291
0
    fn try_from(digest: &Digest) -> Result<Self, Self::Error> {
292
0
        let packed_hash = PackedHash::from_hex(&digest.hash)
293
0
            .err_tip(|| format!("Invalid sha256 hash: {}", digest.hash))?;
294
0
        let size_bytes = digest
295
0
            .size_bytes
296
0
            .try_into()
297
0
            .map_err(|_| make_input_err!("Could not convert {} into u64", digest.size_bytes))?;
298
0
        Ok(Self {
299
0
            packed_hash,
300
0
            size_bytes,
301
0
        })
302
0
    }
303
}
304
305
impl From<DigestInfo> for Digest {
306
222
    fn from(val: DigestInfo) -> Self {
307
        Self {
308
222
            hash: val.packed_hash.to_string(),
309
222
            size_bytes: val.size_bytes.try_into().unwrap_or_else(|e| 
{0
310
0
                error!("Could not convert {} into u64 - {e:?}", val.size_bytes);
311
                // This is a placeholder value that can help a user identify
312
                // that the conversion failed.
313
0
                -255
314
0
            }),
315
        }
316
222
    }
317
}
318
319
impl From<&DigestInfo> for Digest {
320
10
    fn from(val: &DigestInfo) -> Self {
321
        Self {
322
10
            hash: val.packed_hash.to_string(),
323
10
            size_bytes: val.size_bytes.try_into().unwrap_or_else(|e| 
{0
324
0
                error!("Could not convert {} into u64 - {e:?}", val.size_bytes);
325
                // This is a placeholder value that can help a user identify
326
                // that the conversion failed.
327
0
                -255
328
0
            }),
329
        }
330
10
    }
331
}
332
333
#[derive(
334
    Debug, Serialize, Deserialize, Default, Clone, Copy, Eq, PartialEq, Hash, PartialOrd, Ord,
335
)]
336
pub struct PackedHash([u8; 32]);
337
338
const SIZE_OF_PACKED_HASH: usize = 32;
339
impl PackedHash {
340
19
    const fn new() -> Self {
341
19
        Self([0; SIZE_OF_PACKED_HASH])
342
19
    }
343
344
5.45k
    fn from_hex(hash: &str) -> Result<Self, Error> {
345
5.45k
        let mut packed_hash = [0u8; 32];
346
5.45k
        hex::decode_to_slice(hash, &mut packed_hash)
347
5.45k
            .map_err(|e| make_input_err!("Invalid sha256 hash: {hash} - {e:?}"))
?10
;
348
5.44k
        Ok(Self(packed_hash))
349
5.45k
    }
350
351
    /// Converts the packed hash into a hex string.
352
    #[inline]
353
928
    fn to_hex(self) -> Result<[u8; SIZE_OF_PACKED_HASH * 2], fmt::Error> {
354
928
        let mut hash = [0u8; SIZE_OF_PACKED_HASH * 2];
355
928
        hex::encode_to_slice(self.0, &mut hash).map_err(|e| 
{0
356
0
            error!("Could not convert PackedHash to hex - {e:?} - {:?}", self.0);
357
0
            fmt::Error
358
0
        })?;
359
928
        Ok(hash)
360
928
    }
361
}
362
363
impl fmt::Display for PackedHash {
364
248
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
365
248
        let hash = self.to_hex()
?0
;
366
248
        match core::str::from_utf8(&hash) {
367
248
            Ok(hash) => f.write_str(hash)
?0
,
368
0
            Err(_) => f.write_str(&format!("Could not convert hash to utf8 {:?}", self.0))?,
369
        }
370
248
        Ok(())
371
248
    }
372
}
373
374
impl Deref for PackedHash {
375
    type Target = [u8; 32];
376
377
40.2k
    fn deref(&self) -> &Self::Target {
378
40.2k
        &self.0
379
40.2k
    }
380
}
381
382
impl DerefMut for PackedHash {
383
135
    fn deref_mut(&mut self) -> &mut Self::Target {
384
135
        &mut self.0
385
135
    }
386
}
387
388
// Simple utility trait that makes it easier to apply `.try_map` to Vec.
389
// This will convert one vector into another vector with a different type.
390
pub trait VecExt<T> {
391
    fn try_map<F, U>(self, f: F) -> Result<Vec<U>, Error>
392
    where
393
        Self: Sized,
394
        F: (Fn(T) -> Result<U, Error>) + Sized;
395
}
396
397
impl<T> VecExt<T> for Vec<T> {
398
12
    fn try_map<F, U>(self, f: F) -> Result<Vec<U>, Error>
399
12
    where
400
12
        Self: Sized,
401
12
        F: (Fn(T) -> Result<U, Error>) + Sized,
402
    {
403
12
        let mut output = Vec::with_capacity(self.len());
404
18
        for 
item6
in self {
405
6
            output.push((f)(item)
?0
);
406
        }
407
12
        Ok(output)
408
12
    }
409
}
410
411
// Simple utility trait that makes it easier to apply `.try_map` to HashMap.
412
// This will convert one HashMap into another keeping the key the same, but
413
// different value type.
414
pub trait HashMapExt<K: Eq + Hash, T, S: BuildHasher> {
415
    fn try_map<F, U>(self, f: F) -> Result<HashMap<K, U, S>, Error>
416
    where
417
        Self: Sized,
418
        F: (Fn(T) -> Result<U, Error>) + Sized;
419
}
420
421
impl<K: Eq + Hash, T, S: BuildHasher + Clone> HashMapExt<K, T, S> for HashMap<K, T, S> {
422
3
    fn try_map<F, U>(self, f: F) -> Result<HashMap<K, U, S>, Error>
423
3
    where
424
3
        Self: Sized,
425
3
        F: (Fn(T) -> Result<U, Error>) + Sized,
426
    {
427
3
        let mut output = HashMap::with_capacity_and_hasher(self.len(), (*self.hasher()).clone());
428
5
        for (
k2
,
v2
) in self {
429
2
            output.insert(k, (f)(v)
?0
);
430
        }
431
3
        Ok(output)
432
3
    }
433
}
434
435
// Utility to encode our proto into GRPC stream format.
436
38
pub fn encode_stream_proto<T: Message>(proto: &T) -> Result<Bytes, Box<dyn core::error::Error>> {
437
    // See below comment on spec.
438
    use core::mem::size_of;
439
    const PREFIX_BYTES: usize = size_of::<u8>() + size_of::<u32>();
440
441
38
    let mut buf = BytesMut::new();
442
443
228
    for _ in 0..PREFIX_BYTES {
444
190
        // Advance our buffer first.
445
190
        // We will backfill it once we know the size of the message.
446
190
        buf.put_u8(0);
447
190
    }
448
38
    proto.encode(&mut buf)
?0
;
449
38
    let len = buf.len() - PREFIX_BYTES;
450
38
    {
451
38
        let mut buf = &mut buf[0..PREFIX_BYTES];
452
38
        // See: https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#:~:text=Compressed-Flag
453
38
        // for more details on spec.
454
38
        // Compressed-Flag -> 0 / 1 # encoded as 1 byte unsigned integer.
455
38
        buf.put_u8(0);
456
38
        // Message-Length -> {length of Message} # encoded as 4 byte unsigned integer (big endian).
457
38
        buf.put_u32(len as u32);
458
38
        // Message -> *{binary octet}.
459
38
    }
460
461
38
    Ok(buf.freeze())
462
38
}
463
464
/// Small utility to reseed the global RNG.
465
/// Done this way because we use it in a macro
466
/// and macro's can't load external crates.
467
#[inline]
468
343
pub fn reseed_rng_for_test() -> Result<(), Error> {
469
343
    rand::rng()
470
343
        .reseed()
471
343
        .map_err(|e| make_input_err!("Could not reseed RNG - {e:?}"))
472
343
}