/build/source/nativelink-util/src/origin_event_publisher.rs
Line | Count | Source |
1 | | // Copyright 2024 The NativeLink Authors. All rights reserved. |
2 | | // |
3 | | // Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // See LICENSE file for details |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | use core::time::Duration; |
16 | | |
17 | | use bytes::BytesMut; |
18 | | use futures::{FutureExt, future}; |
19 | | use nativelink_proto::com::github::trace_machina::nativelink::events::{OriginEvent, OriginEvents}; |
20 | | use prost::Message; |
21 | | use tokio::sync::{broadcast, mpsc}; |
22 | | use tokio::time::sleep; |
23 | | use tracing::{error, warn}; |
24 | | use uuid::{Timestamp, Uuid}; |
25 | | |
26 | | use crate::shutdown_guard::{Priority, ShutdownGuard}; |
27 | | use crate::store_trait::{Store, StoreLike}; |
28 | | |
29 | | /// Publishes origin events to the store. |
30 | | #[derive(Debug)] |
31 | | pub struct OriginEventPublisher { |
32 | | store: Store, |
33 | | rx: mpsc::Receiver<OriginEvent>, |
34 | | shutdown_tx: broadcast::Sender<ShutdownGuard>, |
35 | | node_id: [u8; 6], |
36 | | } |
37 | | |
38 | | impl OriginEventPublisher { |
39 | 1 | pub fn new( |
40 | 1 | store: Store, |
41 | 1 | rx: mpsc::Receiver<OriginEvent>, |
42 | 1 | shutdown_tx: broadcast::Sender<ShutdownGuard>, |
43 | 1 | ) -> Self { |
44 | | // Generate a random node_id for this instance |
45 | | use rand::Rng; |
46 | 1 | let mut rng = rand::rng(); |
47 | 1 | let mut node_id = [0u8; 6]; |
48 | 1 | rng.fill(&mut node_id); |
49 | | |
50 | 1 | Self { |
51 | 1 | store, |
52 | 1 | rx, |
53 | 1 | shutdown_tx, |
54 | 1 | node_id, |
55 | 1 | } |
56 | 1 | } |
57 | | |
58 | | /// Runs the origin event publisher. |
59 | 1 | pub async fn run(mut self) { |
60 | | const MAX_EVENTS_PER_BATCH: usize = 1024; |
61 | 1 | let mut batch: Vec<OriginEvent> = Vec::with_capacity(MAX_EVENTS_PER_BATCH); |
62 | 1 | let mut shutdown_rx = self.shutdown_tx.subscribe(); |
63 | 1 | let shutdown_fut = shutdown_rx.recv().fuse(); |
64 | 1 | tokio::pin!(shutdown_fut); |
65 | 1 | let shutdown_guard = future::pending().left_future(); |
66 | 1 | tokio::pin!(shutdown_guard); |
67 | | loop { |
68 | 2 | tokio::select! { |
69 | | biased; |
70 | 2 | _ = self.rx.recv_many(&mut batch, MAX_EVENTS_PER_BATCH) => { |
71 | 1 | self.handle_batch(&mut batch).await; |
72 | | } |
73 | 2 | shutdown_guard_res0 = &mut shutdown_fut => { |
74 | 0 | tracing::info!("Received shutdown down in origin event publisher"); |
75 | 0 | let Ok(mut local_shutdown_guard) = shutdown_guard_res else { |
76 | 0 | tracing::error!("Received shutdown down in origin event publisher but failed to get shutdown guard"); |
77 | 0 | return; |
78 | | }; |
79 | 0 | shutdown_guard.set(async move { |
80 | 0 | local_shutdown_guard.wait_for(Priority::P0).await; |
81 | 0 | } |
82 | 0 | .right_future()); |
83 | | } |
84 | 2 | () = &mut shutdown_guard => { |
85 | | // All other services with less priority have completed. |
86 | | // We may still need to process any remaining events. |
87 | 0 | while !self.rx.is_empty() { |
88 | 0 | self.rx.recv_many(&mut batch, MAX_EVENTS_PER_BATCH).await; |
89 | 0 | self.handle_batch(&mut batch).await; |
90 | | } |
91 | 0 | return; |
92 | | } |
93 | | } |
94 | | } |
95 | 0 | } |
96 | | |
97 | 1 | async fn handle_batch(&self, batch: &mut Vec<OriginEvent>) { |
98 | | // Bounded so a sustained store outage can't block the publisher (and |
99 | | // thus the schedulers feeding it) forever; enough to ride out a |
100 | | // Sentinel failover. |
101 | | const MAX_UPLOAD_ATTEMPTS: u32 = 5; |
102 | | // UUID v6 requires a timestamp and node ID |
103 | | // Create timestamp from current system time with nanosecond precision |
104 | 1 | let now = std::time::SystemTime::now() |
105 | 1 | .duration_since(std::time::UNIX_EPOCH) |
106 | 1 | .unwrap(); |
107 | 1 | let ts = Timestamp::from_unix( |
108 | 1 | uuid::timestamp::context::NoContext, |
109 | 1 | now.as_secs(), |
110 | 1 | now.subsec_nanos(), |
111 | | ); |
112 | 1 | let uuid = Uuid::new_v6(ts, &self.node_id); |
113 | 1 | let events = OriginEvents { |
114 | 1 | #[expect( |
115 | 1 | clippy::drain_collect, |
116 | 1 | reason = "Clippy wants us to use use `mem::take`, but this would move all capacity \ |
117 | 1 | as well to the new vector. Since it is much more likely that we will have a \ |
118 | 1 | small number of events in the batch, we prefer to use `drain` and `collect` \ |
119 | 1 | here, so we only need to allocate the exact amount of memory needed and let \ |
120 | 1 | the batch vector's capacity be reused." |
121 | 1 | )] |
122 | 1 | events: batch.drain(..).collect(), |
123 | 1 | }; |
124 | 1 | let mut data = BytesMut::new(); |
125 | 1 | if let Err(e0 ) = events.encode(&mut data) { |
126 | 0 | error!("Failed to encode origin events: {}", e); |
127 | 0 | return; |
128 | 1 | } |
129 | | // The batch has already been drained out of `batch`, so a failed store |
130 | | // write would silently lose these events (the source of the recurring |
131 | | // "No action-level resource sizing records" — origin events dropped |
132 | | // during a transient Redis disruption such as a Sentinel failover). |
133 | | // Retry the upload, re-resolving the master each time, so a transient |
134 | | // failure doesn't drop the events. |
135 | 1 | let key = format!("OriginEvents:{}", uuid.hyphenated()); |
136 | 1 | let data = data.freeze(); |
137 | 3 | for attempt in 1..=MAX_UPLOAD_ATTEMPTS1 { |
138 | 3 | match self |
139 | 3 | .store |
140 | 3 | .as_store_driver_pin() |
141 | 3 | .update_oneshot(key.clone().into(), data.clone()) |
142 | 3 | .await |
143 | | { |
144 | 1 | Ok(()) => return, |
145 | 2 | Err(err) if attempt < MAX_UPLOAD_ATTEMPTS => { |
146 | 2 | warn!( |
147 | | attempt, |
148 | | max = MAX_UPLOAD_ATTEMPTS, |
149 | | ?err, |
150 | | "Failed to upload origin events, retrying" |
151 | | ); |
152 | 2 | sleep(Duration::from_secs_f32(0.1 * attempt as f32)).await; |
153 | | } |
154 | 0 | Err(err) => { |
155 | 0 | error!( |
156 | | attempts = MAX_UPLOAD_ATTEMPTS, |
157 | | ?err, |
158 | | "Failed to upload origin events after retries" |
159 | | ); |
160 | | } |
161 | | } |
162 | | } |
163 | 1 | } |
164 | | } |