Coverage Report

Created: 2026-05-23 21:09

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-util/src/common.rs
Line
Count
Source
1
// Copyright 2024 The NativeLink Authors. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::cmp::{Eq, Ordering};
16
use core::hash::{BuildHasher, Hash};
17
use core::ops::{Deref, DerefMut};
18
use std::collections::HashMap;
19
use std::io::{Cursor, Write};
20
use std::{env, fmt};
21
22
use bytes::{Buf, BufMut, Bytes, BytesMut};
23
use nativelink_error::{Error, ResultExt, make_input_err};
24
use nativelink_metric::{
25
    MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent,
26
};
27
use nativelink_proto::build::bazel::remote::execution::v2::Digest;
28
use prost::Message;
29
use rand::Rng;
30
use serde::de::Visitor;
31
use serde::ser::Error as _;
32
use serde::{Deserialize, Deserializer, Serialize, Serializer};
33
use tonic::Code;
34
use tracing::error;
35
36
pub use crate::fs;
37
38
0
#[derive(Default, Clone, Copy, Eq, PartialEq, Hash, wincode::SchemaRead, wincode::SchemaWrite)]
39
#[repr(C)]
40
pub struct DigestInfo {
41
    /// Raw hash in packed form.
42
    packed_hash: PackedHash,
43
44
    /// Possibly the size of the digest in bytes.
45
    size_bytes: u64,
46
}
47
48
impl MetricsComponent for DigestInfo {
49
0
    fn publish(
50
0
        &self,
51
0
        _kind: MetricKind,
52
0
        field_metadata: MetricFieldData,
53
0
    ) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> {
54
0
        format!("{self}").publish(MetricKind::String, field_metadata)
55
0
    }
56
}
57
58
impl DigestInfo {
59
5.55k
    pub const fn new(packed_hash: [u8; 32], size_bytes: u64) -> Self {
60
5.55k
        Self {
61
5.55k
            size_bytes,
62
5.55k
            packed_hash: PackedHash(packed_hash),
63
5.55k
        }
64
5.55k
    }
65
66
980
    pub fn try_new<T>(hash: &str, size_bytes: T) -> Result<Self, Error>
67
980
    where
68
980
        T: TryInto<u64> + fmt::Display + Copy,
69
    {
70
971
        let packed_hash =
71
980
            PackedHash::from_hex(hash).err_tip(|| 
format!9
("Invalid sha256 hash: {hash}"))
?9
;
72
971
        let size_bytes = size_bytes
73
971
            .try_into()
74
971
            .map_err(|_| 
make_input_err!0
("Could not convert {} into u64", size_bytes))
?0
;
75
        // The proto `Digest` takes an i64, so to keep compatibility
76
        // we only allow sizes that can fit into an i64.
77
971
        if size_bytes > i64::MAX as u64 {
78
2
            return Err(make_input_err!(
79
2
                "Size bytes is too large: {} - max: {}",
80
2
                size_bytes,
81
2
                i64::MAX
82
2
            ));
83
969
        }
84
969
        Ok(Self {
85
969
            packed_hash,
86
969
            size_bytes,
87
969
        })
88
980
    }
89
90
26
    pub const fn zero_digest() -> Self {
91
26
        Self {
92
26
            size_bytes: 0,
93
26
            packed_hash: PackedHash::new(),
94
26
        }
95
26
    }
96
97
40.3k
    pub const fn packed_hash(&self) -> &PackedHash {
98
40.3k
        &self.packed_hash
99
40.3k
    }
100
101
117
    pub const fn set_packed_hash(&mut self, packed_hash: [u8; 32]) {
102
117
        self.packed_hash = PackedHash(packed_hash);
103
117
    }
104
105
17.0k
    pub const fn size_bytes(&self) -> u64 {
106
17.0k
        self.size_bytes
107
17.0k
    }
108
109
    /// Returns a struct that can turn the `DigestInfo` into a string.
110
1.23k
    const fn stringifier(&self) -> DigestStackStringifier<'_> {
111
1.23k
        DigestStackStringifier::new(self)
112
1.23k
    }
113
}
114
115
/// Counts the number of digits a number needs if it were to be
116
/// converted to a string.
117
0
const fn count_digits(mut num: u64) -> usize {
118
0
    let mut count = 0;
119
0
    while num != 0 {
120
0
        count += 1;
121
0
        num /= 10;
122
0
    }
123
0
    count
124
0
}
125
126
/// An optimized version of a function that can convert a `DigestInfo`
127
/// into a str on the stack.
128
struct DigestStackStringifier<'a> {
129
    digest: &'a DigestInfo,
130
    /// Buffer that can hold the string representation of the `DigestInfo`.
131
    /// - Hex is '2 * sizeof(PackedHash)'.
132
    /// - Digits can be at most `count_digits(u64::MAX)`.
133
    /// - We also have a hyphen separator.
134
    buf: [u8; size_of::<PackedHash>() * 2 + count_digits(u64::MAX) + 1],
135
}
136
137
impl<'a> DigestStackStringifier<'a> {
138
1.23k
    const fn new(digest: &'a DigestInfo) -> Self {
139
1.23k
        DigestStackStringifier {
140
1.23k
            digest,
141
1.23k
            buf: [b'-'; size_of::<PackedHash>() * 2 + count_digits(u64::MAX) + 1],
142
1.23k
        }
143
1.23k
    }
144
145
1.23k
    fn as_str(&mut self) -> Result<&str, Error> {
146
        // Populate the buffer and return the amount of bytes written
147
        // to the buffer.
148
1.23k
        let len = {
149
1.23k
            let mut cursor = Cursor::new(&mut self.buf[..]);
150
1.23k
            let hex = self.digest.packed_hash.to_hex().map_err(|e| 
{0
151
0
                Error::from_std_err(Code::InvalidArgument, &e).append(format!(
152
                    "Could not convert PackedHash to hex - {:?}",
153
                    self.digest
154
                ))
155
0
            })?;
156
1.23k
            cursor
157
1.23k
                .write_all(&hex)
158
1.23k
                .err_tip(|| 
format!0
("Could not write hex to buffer - {hex:?} - {hex:?}",))
?0
;
159
            // Note: We already have a hyphen at this point because we
160
            // initialized the buffer with hyphens.
161
1.23k
            cursor.advance(1);
162
1.23k
            cursor
163
1.23k
                .write_fmt(format_args!("{}", self.digest.size_bytes()))
164
1.23k
                .err_tip(|| 
format!0
("Could not write size_bytes to buffer - {hex:?}",))
?0
;
165
1.23k
            cursor.position().try_into().map_err(|e| 
{0
166
0
                Error::from_std_err(Code::InvalidArgument, &e)
167
0
                    .append("Cursor position exceeds usize bounds")
168
0
            })?
169
        };
170
        // Convert the buffer into utf8 string.
171
1.23k
        core::str::from_utf8(&self.buf[..len]).map_err(|e| 
{0
172
0
            Error::from_std_err(Code::InvalidArgument, &e).append(format!(
173
                "Could not convert [u8] to string - {} - {:?}",
174
                self.digest, self.buf
175
            ))
176
0
        })
177
1.23k
    }
178
}
179
180
/// Custom serializer for `DigestInfo` because the default Serializer
181
/// would try to encode the data as a byte array, but we use {hex}-{size}.
182
impl Serialize for DigestInfo {
183
78
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
184
78
    where
185
78
        S: Serializer,
186
    {
187
78
        let mut stringifier = self.stringifier();
188
78
        serializer.serialize_str(
189
78
            stringifier
190
78
                .as_str()
191
78
                .err_tip(|| "During serialization of DigestInfo")
192
78
                .map_err(S::Error::custom)
?0
,
193
        )
194
78
    }
195
}
196
197
/// Custom deserializer for `DigestInfo` because the default Deserializer
198
/// would try to decode the data as a byte array, but we use {hex}-{size}.
199
impl<'de> Deserialize<'de> for DigestInfo {
200
235
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
201
235
    where
202
235
        D: Deserializer<'de>,
203
    {
204
        struct DigestInfoVisitor;
205
        impl Visitor<'_> for DigestInfoVisitor {
206
            type Value = DigestInfo;
207
208
0
            fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
209
0
                formatter.write_str("a string representing a DigestInfo")
210
0
            }
211
212
235
            fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>
213
235
            where
214
235
                E: serde::de::Error,
215
            {
216
235
                let Some((hash, size)) = s.split_once('-') else {
217
0
                    return Err(E::custom(
218
0
                        "Invalid DigestInfo format, expected '-' separator",
219
0
                    ));
220
                };
221
235
                let size_bytes = size
222
235
                    .parse::<u64>()
223
235
                    .map_err(|e| 
E::custom0
(
format!0
("Could not parse size_bytes: {e:?}")))
?0
;
224
235
                DigestInfo::try_new(hash, size_bytes)
225
235
                    .map_err(|e| 
E::custom1
(
format!1
("Could not create DigestInfo: {e:?}")))
226
235
            }
227
        }
228
235
        deserializer.deserialize_str(DigestInfoVisitor)
229
235
    }
230
}
231
232
impl fmt::Display for DigestInfo {
233
684
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
234
684
        let mut stringifier = self.stringifier();
235
684
        f.write_str(
236
684
            stringifier
237
684
                .as_str()
238
684
                .err_tip(|| "During serialization of DigestInfo")
239
684
                .map_err(|e| 
{0
240
0
                    error!("Could not convert DigestInfo to string - {e:?}");
241
0
                    fmt::Error
242
0
                })?,
243
        )
244
684
    }
245
}
246
247
impl fmt::Debug for DigestInfo {
248
474
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
249
474
        let mut stringifier = self.stringifier();
250
474
        match stringifier.as_str() {
251
474
            Ok(s) => f.debug_tuple("DigestInfo").field(&s).finish(),
252
0
            Err(e) => {
253
0
                error!("Could not convert DigestInfo to string - {e:?}");
254
0
                Err(fmt::Error)
255
            }
256
        }
257
474
    }
258
}
259
260
impl Ord for DigestInfo {
261
0
    fn cmp(&self, other: &Self) -> Ordering {
262
0
        self.packed_hash
263
0
            .cmp(&other.packed_hash)
264
0
            .then_with(|| self.size_bytes.cmp(&other.size_bytes))
265
0
    }
266
}
267
268
impl PartialOrd for DigestInfo {
269
0
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
270
0
        Some(self.cmp(other))
271
0
    }
272
}
273
274
impl TryFrom<Digest> for DigestInfo {
275
    type Error = Error;
276
277
260
    fn try_from(digest: Digest) -> Result<Self, Self::Error> {
278
260
        let 
packed_hash259
= PackedHash::from_hex(&digest.hash)
279
260
            .err_tip(|| 
format!1
("Invalid sha256 hash: {}", digest.hash))
?1
;
280
259
        let size_bytes = digest.size_bytes.try_into().map_err(|e| 
{0
281
0
            Error::from_std_err(Code::InvalidArgument, &e)
282
0
                .append(format!("Could not convert {} into u64", digest.size_bytes))
283
0
        })?;
284
259
        Ok(Self {
285
259
            packed_hash,
286
259
            size_bytes,
287
259
        })
288
260
    }
289
}
290
291
impl TryFrom<&Digest> for DigestInfo {
292
    type Error = Error;
293
294
0
    fn try_from(digest: &Digest) -> Result<Self, Self::Error> {
295
0
        let packed_hash = PackedHash::from_hex(&digest.hash)
296
0
            .err_tip(|| format!("Invalid sha256 hash: {}", digest.hash))?;
297
0
        let size_bytes = digest.size_bytes.try_into().map_err(|e| {
298
0
            Error::from_std_err(Code::InvalidArgument, &e)
299
0
                .append(format!("Could not convert {} into u64", digest.size_bytes))
300
0
        })?;
301
0
        Ok(Self {
302
0
            packed_hash,
303
0
            size_bytes,
304
0
        })
305
0
    }
306
}
307
308
impl From<DigestInfo> for Digest {
309
299
    fn from(val: DigestInfo) -> Self {
310
        Self {
311
299
            hash: val.packed_hash.to_string(),
312
299
            size_bytes: val.size_bytes.try_into().unwrap_or_else(|e| 
{0
313
0
                error!("Could not convert {} into u64 - {e:?}", val.size_bytes);
314
                // This is a placeholder value that can help a user identify
315
                // that the conversion failed.
316
0
                -255
317
0
            }),
318
        }
319
299
    }
320
}
321
322
impl From<&DigestInfo> for Digest {
323
11
    fn from(val: &DigestInfo) -> Self {
324
        Self {
325
11
            hash: val.packed_hash.to_string(),
326
11
            size_bytes: val.size_bytes.try_into().unwrap_or_else(|e| 
{0
327
0
                error!("Could not convert {} into u64 - {e:?}", val.size_bytes);
328
                // This is a placeholder value that can help a user identify
329
                // that the conversion failed.
330
0
                -255
331
0
            }),
332
        }
333
11
    }
334
}
335
336
#[derive(
337
    Debug,
338
    Serialize,
339
    Deserialize,
340
    Default,
341
    Clone,
342
    Copy,
343
    Eq,
344
    PartialEq,
345
    Hash,
346
    PartialOrd,
347
    Ord,
348
    wincode::SchemaRead,
349
0
    wincode::SchemaWrite,
350
)]
351
pub struct PackedHash([u8; 32]);
352
353
const SIZE_OF_PACKED_HASH: usize = 32;
354
impl PackedHash {
355
0
    const fn new() -> Self {
356
0
        Self([0; SIZE_OF_PACKED_HASH])
357
0
    }
358
359
1.24k
    fn from_hex(hash: &str) -> Result<Self, Error> {
360
1.24k
        let mut packed_hash = [0u8; 32];
361
1.24k
        hex::decode_to_slice(hash, &mut packed_hash).map_err(|e| 
{10
362
10
            Error::from_std_err(Code::InvalidArgument, &e)
363
10
                .append(format!("Invalid sha256 hash: {hash}"))
364
10
        })?;
365
1.23k
        Ok(Self(packed_hash))
366
1.24k
    }
367
368
    /// Converts the packed hash into a hex string.
369
    #[inline]
370
1.59k
    fn to_hex(self) -> Result<[u8; SIZE_OF_PACKED_HASH * 2], fmt::Error> {
371
1.59k
        let mut hash = [0u8; SIZE_OF_PACKED_HASH * 2];
372
1.59k
        hex::encode_to_slice(self.0, &mut hash).map_err(|e| 
{0
373
0
            error!("Could not convert PackedHash to hex - {e:?} - {:?}", self.0);
374
0
            fmt::Error
375
0
        })?;
376
1.59k
        Ok(hash)
377
1.59k
    }
378
}
379
380
impl fmt::Display for PackedHash {
381
363
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
382
363
        let hash = self.to_hex()
?0
;
383
363
        match core::str::from_utf8(&hash) {
384
363
            Ok(hash) => f.write_str(hash)
?0
,
385
0
            Err(_) => f.write_str(&format!("Could not convert hash to utf8 {:?}", self.0))?,
386
        }
387
363
        Ok(())
388
363
    }
389
}
390
391
impl Deref for PackedHash {
392
    type Target = [u8; 32];
393
394
40.2k
    fn deref(&self) -> &Self::Target {
395
40.2k
        &self.0
396
40.2k
    }
397
}
398
399
impl DerefMut for PackedHash {
400
117
    fn deref_mut(&mut self) -> &mut Self::Target {
401
117
        &mut self.0
402
117
    }
403
}
404
405
// Simple utility trait that makes it easier to apply `.try_map` to Vec.
406
// This will convert one vector into another vector with a different type.
407
pub trait VecExt<T> {
408
    fn try_map<F, U>(self, f: F) -> Result<Vec<U>, Error>
409
    where
410
        Self: Sized,
411
        F: (Fn(T) -> Result<U, Error>) + Sized;
412
}
413
414
impl<T> VecExt<T> for Vec<T> {
415
12
    fn try_map<F, U>(self, f: F) -> Result<Vec<U>, Error>
416
12
    where
417
12
        Self: Sized,
418
12
        F: (Fn(T) -> Result<U, Error>) + Sized,
419
    {
420
12
        let mut output = Vec::with_capacity(self.len());
421
12
        for 
item6
in self {
422
6
            output.push((f)(item)
?0
);
423
        }
424
12
        Ok(output)
425
12
    }
426
}
427
428
// Simple utility trait that makes it easier to apply `.try_map` to HashMap.
429
// This will convert one HashMap into another keeping the key the same, but
430
// different value type.
431
pub trait HashMapExt<K: Eq + Hash, T, S: BuildHasher> {
432
    fn try_map<F, U>(self, f: F) -> Result<HashMap<K, U, S>, Error>
433
    where
434
        Self: Sized,
435
        F: (Fn(T) -> Result<U, Error>) + Sized;
436
}
437
438
impl<K: Eq + Hash, T, S: BuildHasher + Clone> HashMapExt<K, T, S> for HashMap<K, T, S> {
439
3
    fn try_map<F, U>(self, f: F) -> Result<HashMap<K, U, S>, Error>
440
3
    where
441
3
        Self: Sized,
442
3
        F: (Fn(T) -> Result<U, Error>) + Sized,
443
    {
444
3
        let mut output = HashMap::with_capacity_and_hasher(self.len(), (*self.hasher()).clone());
445
3
        for (
k2
,
v2
) in self {
446
2
            output.insert(k, (f)(v)
?0
);
447
        }
448
3
        Ok(output)
449
3
    }
450
}
451
452
// Utility to encode our proto into GRPC stream format.
453
43
pub fn encode_stream_proto<T: Message>(proto: &T) -> Result<Bytes, Box<dyn core::error::Error>> {
454
    // See below comment on spec.
455
    use core::mem::size_of;
456
    const PREFIX_BYTES: usize = size_of::<u8>() + size_of::<u32>();
457
458
43
    let mut buf = BytesMut::new();
459
460
215
    for _ in 
0..PREFIX_BYTES43
{
461
215
        // Advance our buffer first.
462
215
        // We will backfill it once we know the size of the message.
463
215
        buf.put_u8(0);
464
215
    }
465
43
    proto.encode(&mut buf)
?0
;
466
43
    let len = buf.len() - PREFIX_BYTES;
467
    {
468
43
        let mut buf = &mut buf[0..PREFIX_BYTES];
469
        // See: https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#:~:text=Compressed-Flag
470
        // for more details on spec.
471
        // Compressed-Flag -> 0 / 1 # encoded as 1 byte unsigned integer.
472
43
        buf.put_u8(0);
473
        // Message-Length -> {length of Message} # encoded as 4 byte unsigned integer (big endian).
474
43
        buf.put_u32(u32::try_from(len)
?0
);
475
        // Message -> *{binary octet}.
476
    }
477
478
43
    Ok(buf.freeze())
479
43
}
480
481
/// Small utility to reseed the global RNG.
482
/// Done this way because we use it in a macro
483
/// and macro's can't load external crates.
484
#[inline]
485
550
pub fn reseed_rng_for_test() -> Result<(), Error> {
486
550
    rand::rng()
487
550
        .reseed()
488
550
        .map_err(|e| 
Error::from_std_err0
(
Code::InvalidArgument0
,
&e0
).
append0
("Could not reseed RNG"))
489
550
}
490
491
/// Get temporary path from either `TEST_TMPDIR` or best effort temp directory if
492
/// not set.
493
152
pub fn make_temp_path(data: &str) -> String {
494
    #[cfg(target_family = "unix")]
495
152
    return format!(
496
        "{}/{}/{}",
497
152
        env::var("TEST_TMPDIR").unwrap_or_else(|_| env::temp_dir().to_str().unwrap().to_string()),
498
152
        rand::rng().random::<u64>(),
499
        data
500
    );
501
    #[cfg(target_family = "windows")]
502
    return format!(
503
        "{}\\{}\\{}",
504
        env::var("TEST_TMPDIR").unwrap_or_else(|_| env::temp_dir().to_str().unwrap().to_string()),
505
        rand::rng().random::<u64>(),
506
        data
507
    );
508
152
}
509
510
// Constant for PreconditionFailure
511
pub const VIOLATION_TYPE_MISSING: &str = "MISSING";