/build/source/nativelink-service/src/bep_server.rs
Line | Count | Source |
1 | | // Copyright 2024 Trace Machina, Inc. All rights reserved. |
2 | | // |
3 | | // Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // See LICENSE file for details |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | use core::pin::Pin; |
16 | | use core::time::Duration; |
17 | | use std::borrow::Cow; |
18 | | |
19 | | use bytes::BytesMut; |
20 | | use futures::{Stream, StreamExt}; |
21 | | use nativelink_error::{Error, ResultExt}; |
22 | | use nativelink_proto::com::github::trace_machina::nativelink::events::{BepEvent, bep_event}; |
23 | | use nativelink_proto::google::devtools::build::v1::publish_build_event_server::{ |
24 | | PublishBuildEvent, PublishBuildEventServer, |
25 | | }; |
26 | | use nativelink_proto::google::devtools::build::v1::{ |
27 | | PublishBuildToolEventStreamRequest, PublishBuildToolEventStreamResponse, |
28 | | PublishLifecycleEventRequest, |
29 | | }; |
30 | | use nativelink_store::store_manager::StoreManager; |
31 | | use nativelink_util::store_trait::{Store, StoreDriver, StoreKey, StoreLike}; |
32 | | use opentelemetry::baggage::BaggageExt; |
33 | | use opentelemetry::context::Context; |
34 | | use opentelemetry_semantic_conventions::attribute::ENDUSER_ID; |
35 | | use prost::Message; |
36 | | use tokio::time::sleep; |
37 | | use tonic::{Request, Response, Result, Status, Streaming}; |
38 | | use tracing::{Level, instrument, warn}; |
39 | | |
40 | | /// Bounded retries for persisting a BEP event so a transient store failure |
41 | | /// (e.g. a Redis Sentinel failover) doesn't drop it — BEP events are the |
42 | | /// authoritative build record, the same durability class as origin events. |
43 | | const MAX_BEP_UPLOAD_ATTEMPTS: u32 = 5; |
44 | | /// Store writes in flight per stream. Bazel waits for every event's ack |
45 | | /// before it finishes, so a sequential write per event caps a build's |
46 | | /// upload at one store round trip per event; writes overlap up to this |
47 | | /// many while the acks still go out in sequence order. |
48 | | const MAX_IN_FLIGHT_BEP_EVENTS: usize = 64; |
49 | | |
50 | | /// Current version of the BEP event. This might be used in the future if |
51 | | /// there is a breaking change in the BEP event format. |
52 | | const BEP_EVENT_VERSION: u32 = 0; |
53 | | |
54 | | #[allow(clippy::result_large_err, reason = "TODO Fix this. Breaks on nightly")] |
55 | 5 | fn get_identity() -> Option<String> { |
56 | 5 | Context::current() |
57 | 5 | .baggage() |
58 | 5 | .get(ENDUSER_ID) |
59 | 5 | .map(|value| value.as_str()0 .to_string0 ()) |
60 | 5 | } |
61 | | |
62 | | #[derive(Debug)] |
63 | | pub struct BepServer { |
64 | | store: Store, |
65 | | } |
66 | | |
67 | | impl BepServer { |
68 | 5 | pub fn new( |
69 | 5 | config: &nativelink_config::cas_server::BepConfig, |
70 | 5 | store_manager: &StoreManager, |
71 | 5 | ) -> Result<Self, Error> { |
72 | 5 | let store = store_manager |
73 | 5 | .get_store(&config.store) |
74 | 5 | .err_tip(|| format!0 ("Expected store {} to exist in store manager", config.store))?0 ; |
75 | | |
76 | 5 | Ok(Self { store }) |
77 | 5 | } |
78 | | |
79 | 0 | pub fn into_service(self) -> PublishBuildEventServer<Self> { |
80 | 0 | PublishBuildEventServer::new(self) |
81 | 0 | } |
82 | | |
83 | 2 | async fn inner_publish_lifecycle_event( |
84 | 2 | &self, |
85 | 2 | request: PublishLifecycleEventRequest, |
86 | 2 | identity: Option<String>, |
87 | 2 | ) -> Result<Response<()>, Error> { |
88 | 2 | let build_event = request |
89 | 2 | .build_event |
90 | 2 | .as_ref() |
91 | 2 | .err_tip(|| "Expected build_event to be set")?0 ; |
92 | 2 | let stream_id = build_event |
93 | 2 | .stream_id |
94 | 2 | .as_ref() |
95 | 2 | .err_tip(|| "Expected stream_id to be set")?0 ; |
96 | | |
97 | 2 | let sequence_number = build_event.sequence_number; |
98 | | |
99 | 2 | let store_key = StoreKey::Str(Cow::Owned(format!( |
100 | 2 | "BepEvent:le:{}:{}:{}", |
101 | 2 | stream_id.build_id, stream_id.invocation_id, sequence_number, |
102 | 2 | ))); |
103 | | |
104 | 2 | let bep_event = BepEvent { |
105 | 2 | version: BEP_EVENT_VERSION, |
106 | 2 | identity: identity.unwrap_or_default(), |
107 | 2 | event: Some(bep_event::Event::LifecycleEvent(request)), |
108 | 2 | }; |
109 | 2 | let mut buf = BytesMut::new(); |
110 | 2 | bep_event |
111 | 2 | .encode(&mut buf) |
112 | 2 | .err_tip(|| "Could not encode PublishLifecycleEventRequest proto")?0 ; |
113 | | |
114 | 2 | let data = buf.freeze(); |
115 | 4 | for attempt in 1..=MAX_BEP_UPLOAD_ATTEMPTS2 { |
116 | 4 | match self |
117 | 4 | .store |
118 | 4 | .update_oneshot(store_key.borrow(), data.clone()) |
119 | 4 | .await |
120 | | { |
121 | 2 | Ok(()) => break, |
122 | 2 | Err(err) if attempt < MAX_BEP_UPLOAD_ATTEMPTS => { |
123 | 2 | warn!( |
124 | | attempt, |
125 | | max = MAX_BEP_UPLOAD_ATTEMPTS, |
126 | | ?err, |
127 | | %store_key, |
128 | | "Failed to store BEP lifecycle event, retrying" |
129 | | ); |
130 | 2 | sleep(Duration::from_secs_f32(0.1 * attempt as f32)).await; |
131 | | } |
132 | 0 | Err(err) => { |
133 | 0 | return Err(err).err_tip(|| { |
134 | 0 | format!( |
135 | | "Failed to store PublishLifecycleEventRequest for {store_key} after retries" |
136 | | ) |
137 | 0 | }); |
138 | | } |
139 | | } |
140 | | } |
141 | | |
142 | 2 | Ok(Response::new(())) |
143 | 2 | } |
144 | | |
145 | 3 | async fn inner_publish_build_tool_event_stream( |
146 | 3 | &self, |
147 | 3 | stream: Streaming<PublishBuildToolEventStreamRequest>, |
148 | 3 | identity: Option<String>, |
149 | 3 | ) -> Result<Response<PublishBuildToolEventStreamStream>, Error> { |
150 | 22 | async fn process_request( |
151 | 22 | store: Pin<&dyn StoreDriver>, |
152 | 22 | request: PublishBuildToolEventStreamRequest, |
153 | 22 | identity: String, |
154 | 22 | ) -> Result<PublishBuildToolEventStreamResponse, Status> { |
155 | 22 | let ordered_build_event = request |
156 | 22 | .ordered_build_event |
157 | 22 | .as_ref() |
158 | 22 | .err_tip(|| "Expected ordered_build_event to be set")?0 ; |
159 | 22 | let stream_id = ordered_build_event |
160 | 22 | .stream_id |
161 | 22 | .as_ref() |
162 | 22 | .err_tip(|| "Expected stream_id to be set")?0 |
163 | 22 | .clone(); |
164 | | |
165 | 22 | let sequence_number = ordered_build_event.sequence_number; |
166 | | |
167 | 22 | let bep_event = BepEvent { |
168 | 22 | version: BEP_EVENT_VERSION, |
169 | 22 | identity, |
170 | 22 | event: Some(bep_event::Event::BuildToolEvent(request)), |
171 | 22 | }; |
172 | 22 | let mut buf = BytesMut::new(); |
173 | | |
174 | 22 | bep_event |
175 | 22 | .encode(&mut buf) |
176 | 22 | .err_tip(|| "Could not encode PublishBuildToolEventStreamRequest proto")?0 ; |
177 | | |
178 | 22 | let store_key = StoreKey::Str(Cow::Owned(format!( |
179 | 22 | "BepEvent:be:{}:{}:{}", |
180 | 22 | stream_id.build_id, stream_id.invocation_id, sequence_number, |
181 | 22 | ))); |
182 | 22 | let data = buf.freeze(); |
183 | 22 | for attempt in 1..=MAX_BEP_UPLOAD_ATTEMPTS { |
184 | 22 | match store.update_oneshot(store_key.borrow(), data.clone()).await { |
185 | 22 | Ok(()) => break, |
186 | 0 | Err(err) if attempt < MAX_BEP_UPLOAD_ATTEMPTS => { |
187 | 0 | warn!( |
188 | | attempt, |
189 | | max = MAX_BEP_UPLOAD_ATTEMPTS, |
190 | | ?err, |
191 | | %store_key, |
192 | | "Failed to store BEP build-tool event, retrying" |
193 | | ); |
194 | 0 | sleep(Duration::from_secs_f32(0.1 * attempt as f32)).await; |
195 | | } |
196 | 0 | Err(err) => { |
197 | 0 | return Err(err).err_tip( |
198 | | || "Failed to store PublishBuildToolEventStreamRequest after retries", |
199 | 0 | )?; |
200 | | } |
201 | | } |
202 | | } |
203 | | |
204 | 22 | Ok(PublishBuildToolEventStreamResponse { |
205 | 22 | stream_id: Some(stream_id.clone()), |
206 | 22 | sequence_number, |
207 | 22 | }) |
208 | 22 | } |
209 | | |
210 | 3 | let store = self.store.clone(); |
211 | 3 | let identity = identity.unwrap_or_default(); |
212 | 3 | let response_stream = stream |
213 | 22 | .map3 (move |request| { |
214 | 22 | let store = store.clone(); |
215 | 22 | let identity = identity.clone(); |
216 | 22 | async move { |
217 | 22 | let request = request |
218 | 22 | .err_tip(|| "While receiving message in publish_build_tool_event_stream")?0 ; |
219 | 22 | process_request(store.as_store_driver_pin(), request, identity).await |
220 | 22 | } |
221 | 22 | }) |
222 | 3 | .buffered(MAX_IN_FLIGHT_BEP_EVENTS); |
223 | 3 | Ok(Response::new(Box::pin(response_stream))) |
224 | 3 | } |
225 | | } |
226 | | |
227 | | type PublishBuildToolEventStreamStream = Pin< |
228 | | Box<dyn Stream<Item = Result<PublishBuildToolEventStreamResponse, Status>> + Send + 'static>, |
229 | | >; |
230 | | |
231 | | #[tonic::async_trait] |
232 | | impl PublishBuildEvent for BepServer { |
233 | | type PublishBuildToolEventStreamStream = PublishBuildToolEventStreamStream; |
234 | | |
235 | | #[instrument( |
236 | | err, |
237 | | ret(level = Level::INFO), |
238 | | level = Level::ERROR, |
239 | | skip_all, |
240 | | fields(request = ?grpc_request.get_ref()) |
241 | | )] |
242 | | async fn publish_lifecycle_event( |
243 | | &self, |
244 | | grpc_request: Request<PublishLifecycleEventRequest>, |
245 | | ) -> Result<Response<()>, Status> { |
246 | | self.inner_publish_lifecycle_event(grpc_request.into_inner(), get_identity()) |
247 | | .await |
248 | | .map_err(Error::into) |
249 | | } |
250 | | |
251 | | #[instrument( |
252 | | err, |
253 | | level = Level::ERROR, |
254 | | skip_all, |
255 | | fields(request = ?grpc_request.get_ref()) |
256 | | )] |
257 | | async fn publish_build_tool_event_stream( |
258 | | &self, |
259 | | grpc_request: Request<Streaming<PublishBuildToolEventStreamRequest>>, |
260 | | ) -> Result<Response<Self::PublishBuildToolEventStreamStream>, Status> { |
261 | | self.inner_publish_build_tool_event_stream(grpc_request.into_inner(), get_identity()) |
262 | | .await |
263 | | .map_err(Error::into) |
264 | | } |
265 | | } |