Coverage Report

Created: 2026-07-21 15:28

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-util/src/origin_event_publisher.rs
Line
Count
Source
1
// Copyright 2024 The NativeLink Authors. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::time::Duration;
16
17
use bytes::BytesMut;
18
use futures::{FutureExt, future};
19
use nativelink_proto::com::github::trace_machina::nativelink::events::{OriginEvent, OriginEvents};
20
use prost::Message;
21
use tokio::sync::{broadcast, mpsc};
22
use tokio::time::sleep;
23
use tracing::{error, warn};
24
use uuid::{Timestamp, Uuid};
25
26
use crate::shutdown_guard::{Priority, ShutdownGuard};
27
use crate::store_trait::{Store, StoreLike};
28
29
/// Publishes origin events to the store.
30
#[derive(Debug)]
31
pub struct OriginEventPublisher {
32
    store: Store,
33
    rx: mpsc::Receiver<OriginEvent>,
34
    shutdown_tx: broadcast::Sender<ShutdownGuard>,
35
    node_id: [u8; 6],
36
}
37
38
impl OriginEventPublisher {
39
1
    pub fn new(
40
1
        store: Store,
41
1
        rx: mpsc::Receiver<OriginEvent>,
42
1
        shutdown_tx: broadcast::Sender<ShutdownGuard>,
43
1
    ) -> Self {
44
        // Generate a random node_id for this instance
45
        use rand::Rng;
46
1
        let mut rng = rand::rng();
47
1
        let mut node_id = [0u8; 6];
48
1
        rng.fill(&mut node_id);
49
50
1
        Self {
51
1
            store,
52
1
            rx,
53
1
            shutdown_tx,
54
1
            node_id,
55
1
        }
56
1
    }
57
58
    /// Runs the origin event publisher.
59
1
    pub async fn run(mut self) {
60
        const MAX_EVENTS_PER_BATCH: usize = 1024;
61
1
        let mut batch: Vec<OriginEvent> = Vec::with_capacity(MAX_EVENTS_PER_BATCH);
62
1
        let mut shutdown_rx = self.shutdown_tx.subscribe();
63
1
        let shutdown_fut = shutdown_rx.recv().fuse();
64
1
        tokio::pin!(shutdown_fut);
65
1
        let shutdown_guard = future::pending().left_future();
66
1
        tokio::pin!(shutdown_guard);
67
        loop {
68
2
            tokio::select! {
69
                biased;
70
2
                _ = self.rx.recv_many(&mut batch, MAX_EVENTS_PER_BATCH) => {
71
1
                    self.handle_batch(&mut batch).await;
72
                }
73
2
                
shutdown_guard_res0
= &mut shutdown_fut => {
74
0
                    tracing::info!("Received shutdown down in origin event publisher");
75
0
                    let Ok(mut local_shutdown_guard) = shutdown_guard_res else {
76
0
                        tracing::error!("Received shutdown down in origin event publisher but failed to get shutdown guard");
77
0
                        return;
78
                    };
79
0
                    shutdown_guard.set(async move {
80
0
                        local_shutdown_guard.wait_for(Priority::P0).await;
81
0
                    }
82
0
                    .right_future());
83
                }
84
2
                () = &mut shutdown_guard => {
85
                    // All other services with less priority have completed.
86
                    // We may still need to process any remaining events.
87
0
                    while !self.rx.is_empty() {
88
0
                        self.rx.recv_many(&mut batch, MAX_EVENTS_PER_BATCH).await;
89
0
                        self.handle_batch(&mut batch).await;
90
                    }
91
0
                    return;
92
                }
93
            }
94
        }
95
0
    }
96
97
1
    async fn handle_batch(&self, batch: &mut Vec<OriginEvent>) {
98
        // Bounded so a sustained store outage can't block the publisher (and
99
        // thus the schedulers feeding it) forever; enough to ride out a
100
        // Sentinel failover.
101
        const MAX_UPLOAD_ATTEMPTS: u32 = 5;
102
        // UUID v6 requires a timestamp and node ID
103
        // Create timestamp from current system time with nanosecond precision
104
1
        let now = std::time::SystemTime::now()
105
1
            .duration_since(std::time::UNIX_EPOCH)
106
1
            .unwrap();
107
1
        let ts = Timestamp::from_unix(
108
1
            uuid::timestamp::context::NoContext,
109
1
            now.as_secs(),
110
1
            now.subsec_nanos(),
111
        );
112
1
        let uuid = Uuid::new_v6(ts, &self.node_id);
113
1
        let events = OriginEvents {
114
1
            #[expect(
115
1
                clippy::drain_collect,
116
1
                reason = "Clippy wants us to use use `mem::take`, but this would move all capacity \
117
1
                    as well to the new vector. Since it is much more likely that we will have a \
118
1
                    small number of events in the batch, we prefer to use `drain` and `collect` \
119
1
                    here, so we only need to allocate the exact amount of memory needed and let \
120
1
                    the batch vector's capacity be reused."
121
1
            )]
122
1
            events: batch.drain(..).collect(),
123
1
        };
124
1
        let mut data = BytesMut::new();
125
1
        if let Err(
e0
) = events.encode(&mut data) {
126
0
            error!("Failed to encode origin events: {}", e);
127
0
            return;
128
1
        }
129
        // The batch has already been drained out of `batch`, so a failed store
130
        // write would silently lose these events (the source of the recurring
131
        // "No action-level resource sizing records" — origin events dropped
132
        // during a transient Redis disruption such as a Sentinel failover).
133
        // Retry the upload, re-resolving the master each time, so a transient
134
        // failure doesn't drop the events.
135
1
        let key = format!("OriginEvents:{}", uuid.hyphenated());
136
1
        let data = data.freeze();
137
3
        for attempt in 
1..=MAX_UPLOAD_ATTEMPTS1
{
138
3
            match self
139
3
                .store
140
3
                .as_store_driver_pin()
141
3
                .update_oneshot(key.clone().into(), data.clone())
142
3
                .await
143
            {
144
1
                Ok(()) => return,
145
2
                Err(err) if attempt < MAX_UPLOAD_ATTEMPTS => {
146
2
                    warn!(
147
                        attempt,
148
                        max = MAX_UPLOAD_ATTEMPTS,
149
                        ?err,
150
                        "Failed to upload origin events, retrying"
151
                    );
152
2
                    sleep(Duration::from_secs_f32(0.1 * attempt as f32)).await;
153
                }
154
0
                Err(err) => {
155
0
                    error!(
156
                        attempts = MAX_UPLOAD_ATTEMPTS,
157
                        ?err,
158
                        "Failed to upload origin events after retries"
159
                    );
160
                }
161
            }
162
        }
163
1
    }
164
}