Coverage Report

Created: 2026-11-13 23:29

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-service/src/bep_server.rs
Line
Count
Source
1
// Copyright 2024 Trace Machina, Inc. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::pin::Pin;
16
use core::time::Duration;
17
use std::borrow::Cow;
18
19
use bytes::BytesMut;
20
use futures::{Stream, StreamExt};
21
use nativelink_error::{Error, ResultExt};
22
use nativelink_proto::com::github::trace_machina::nativelink::events::{BepEvent, bep_event};
23
use nativelink_proto::google::devtools::build::v1::publish_build_event_server::{
24
    PublishBuildEvent, PublishBuildEventServer,
25
};
26
use nativelink_proto::google::devtools::build::v1::{
27
    PublishBuildToolEventStreamRequest, PublishBuildToolEventStreamResponse,
28
    PublishLifecycleEventRequest,
29
};
30
use nativelink_store::store_manager::StoreManager;
31
use nativelink_util::store_trait::{Store, StoreDriver, StoreKey, StoreLike};
32
use opentelemetry::baggage::BaggageExt;
33
use opentelemetry::context::Context;
34
use opentelemetry_semantic_conventions::attribute::ENDUSER_ID;
35
use prost::Message;
36
use tokio::time::sleep;
37
use tonic::{Request, Response, Result, Status, Streaming};
38
use tracing::{Level, instrument, warn};
39
40
/// Bounded retries for persisting a BEP event so a transient store failure
41
/// (e.g. a Redis Sentinel failover) doesn't drop it — BEP events are the
42
/// authoritative build record, the same durability class as origin events.
43
const MAX_BEP_UPLOAD_ATTEMPTS: u32 = 5;
44
/// Store writes in flight per stream. Bazel waits for every event's ack
45
/// before it finishes, so a sequential write per event caps a build's
46
/// upload at one store round trip per event; writes overlap up to this
47
/// many while the acks still go out in sequence order.
48
const MAX_IN_FLIGHT_BEP_EVENTS: usize = 64;
49
50
/// Current version of the BEP event. This might be used in the future if
51
/// there is a breaking change in the BEP event format.
52
const BEP_EVENT_VERSION: u32 = 0;
53
54
#[allow(clippy::result_large_err, reason = "TODO Fix this. Breaks on nightly")]
55
5
fn get_identity() -> Option<String> {
56
5
    Context::current()
57
5
        .baggage()
58
5
        .get(ENDUSER_ID)
59
5
        .map(|value| 
value.as_str()0
.
to_string0
())
60
5
}
61
62
#[derive(Debug)]
63
pub struct BepServer {
64
    store: Store,
65
}
66
67
impl BepServer {
68
5
    pub fn new(
69
5
        config: &nativelink_config::cas_server::BepConfig,
70
5
        store_manager: &StoreManager,
71
5
    ) -> Result<Self, Error> {
72
5
        let store = store_manager
73
5
            .get_store(&config.store)
74
5
            .err_tip(|| 
format!0
("Expected store {} to exist in store manager", config.store))
?0
;
75
76
5
        Ok(Self { store })
77
5
    }
78
79
0
    pub fn into_service(self) -> PublishBuildEventServer<Self> {
80
0
        PublishBuildEventServer::new(self)
81
0
    }
82
83
2
    async fn inner_publish_lifecycle_event(
84
2
        &self,
85
2
        request: PublishLifecycleEventRequest,
86
2
        identity: Option<String>,
87
2
    ) -> Result<Response<()>, Error> {
88
2
        let build_event = request
89
2
            .build_event
90
2
            .as_ref()
91
2
            .err_tip(|| "Expected build_event to be set")
?0
;
92
2
        let stream_id = build_event
93
2
            .stream_id
94
2
            .as_ref()
95
2
            .err_tip(|| "Expected stream_id to be set")
?0
;
96
97
2
        let sequence_number = build_event.sequence_number;
98
99
2
        let store_key = StoreKey::Str(Cow::Owned(format!(
100
2
            "BepEvent:le:{}:{}:{}",
101
2
            stream_id.build_id, stream_id.invocation_id, sequence_number,
102
2
        )));
103
104
2
        let bep_event = BepEvent {
105
2
            version: BEP_EVENT_VERSION,
106
2
            identity: identity.unwrap_or_default(),
107
2
            event: Some(bep_event::Event::LifecycleEvent(request)),
108
2
        };
109
2
        let mut buf = BytesMut::new();
110
2
        bep_event
111
2
            .encode(&mut buf)
112
2
            .err_tip(|| "Could not encode PublishLifecycleEventRequest proto")
?0
;
113
114
2
        let data = buf.freeze();
115
4
        for attempt in 
1..=MAX_BEP_UPLOAD_ATTEMPTS2
{
116
4
            match self
117
4
                .store
118
4
                .update_oneshot(store_key.borrow(), data.clone())
119
4
                .await
120
            {
121
2
                Ok(()) => break,
122
2
                Err(err) if attempt < MAX_BEP_UPLOAD_ATTEMPTS => {
123
2
                    warn!(
124
                        attempt,
125
                        max = MAX_BEP_UPLOAD_ATTEMPTS,
126
                        ?err,
127
                        %store_key,
128
                        "Failed to store BEP lifecycle event, retrying"
129
                    );
130
2
                    sleep(Duration::from_secs_f32(0.1 * attempt as f32)).await;
131
                }
132
0
                Err(err) => {
133
0
                    return Err(err).err_tip(|| {
134
0
                        format!(
135
                            "Failed to store PublishLifecycleEventRequest for {store_key} after retries"
136
                        )
137
0
                    });
138
                }
139
            }
140
        }
141
142
2
        Ok(Response::new(()))
143
2
    }
144
145
3
    async fn inner_publish_build_tool_event_stream(
146
3
        &self,
147
3
        stream: Streaming<PublishBuildToolEventStreamRequest>,
148
3
        identity: Option<String>,
149
3
    ) -> Result<Response<PublishBuildToolEventStreamStream>, Error> {
150
22
        async fn process_request(
151
22
            store: Pin<&dyn StoreDriver>,
152
22
            request: PublishBuildToolEventStreamRequest,
153
22
            identity: String,
154
22
        ) -> Result<PublishBuildToolEventStreamResponse, Status> {
155
22
            let ordered_build_event = request
156
22
                .ordered_build_event
157
22
                .as_ref()
158
22
                .err_tip(|| "Expected ordered_build_event to be set")
?0
;
159
22
            let stream_id = ordered_build_event
160
22
                .stream_id
161
22
                .as_ref()
162
22
                .err_tip(|| "Expected stream_id to be set")
?0
163
22
                .clone();
164
165
22
            let sequence_number = ordered_build_event.sequence_number;
166
167
22
            let bep_event = BepEvent {
168
22
                version: BEP_EVENT_VERSION,
169
22
                identity,
170
22
                event: Some(bep_event::Event::BuildToolEvent(request)),
171
22
            };
172
22
            let mut buf = BytesMut::new();
173
174
22
            bep_event
175
22
                .encode(&mut buf)
176
22
                .err_tip(|| "Could not encode PublishBuildToolEventStreamRequest proto")
?0
;
177
178
22
            let store_key = StoreKey::Str(Cow::Owned(format!(
179
22
                "BepEvent:be:{}:{}:{}",
180
22
                stream_id.build_id, stream_id.invocation_id, sequence_number,
181
22
            )));
182
22
            let data = buf.freeze();
183
22
            for attempt in 1..=MAX_BEP_UPLOAD_ATTEMPTS {
184
22
                match store.update_oneshot(store_key.borrow(), data.clone()).await {
185
22
                    Ok(()) => break,
186
0
                    Err(err) if attempt < MAX_BEP_UPLOAD_ATTEMPTS => {
187
0
                        warn!(
188
                            attempt,
189
                            max = MAX_BEP_UPLOAD_ATTEMPTS,
190
                            ?err,
191
                            %store_key,
192
                            "Failed to store BEP build-tool event, retrying"
193
                        );
194
0
                        sleep(Duration::from_secs_f32(0.1 * attempt as f32)).await;
195
                    }
196
0
                    Err(err) => {
197
0
                        return Err(err).err_tip(
198
                            || "Failed to store PublishBuildToolEventStreamRequest after retries",
199
0
                        )?;
200
                    }
201
                }
202
            }
203
204
22
            Ok(PublishBuildToolEventStreamResponse {
205
22
                stream_id: Some(stream_id.clone()),
206
22
                sequence_number,
207
22
            })
208
22
        }
209
210
3
        let store = self.store.clone();
211
3
        let identity = identity.unwrap_or_default();
212
3
        let response_stream = stream
213
22
            .
map3
(move |request| {
214
22
                let store = store.clone();
215
22
                let identity = identity.clone();
216
22
                async move {
217
22
                    let request = request
218
22
                        .err_tip(|| "While receiving message in publish_build_tool_event_stream")
?0
;
219
22
                    process_request(store.as_store_driver_pin(), request, identity).await
220
22
                }
221
22
            })
222
3
            .buffered(MAX_IN_FLIGHT_BEP_EVENTS);
223
3
        Ok(Response::new(Box::pin(response_stream)))
224
3
    }
225
}
226
227
type PublishBuildToolEventStreamStream = Pin<
228
    Box<dyn Stream<Item = Result<PublishBuildToolEventStreamResponse, Status>> + Send + 'static>,
229
>;
230
231
#[tonic::async_trait]
232
impl PublishBuildEvent for BepServer {
233
    type PublishBuildToolEventStreamStream = PublishBuildToolEventStreamStream;
234
235
    #[instrument(
236
        err,
237
        ret(level = Level::INFO),
238
        level = Level::ERROR,
239
        skip_all,
240
        fields(request = ?grpc_request.get_ref())
241
    )]
242
    async fn publish_lifecycle_event(
243
        &self,
244
        grpc_request: Request<PublishLifecycleEventRequest>,
245
    ) -> Result<Response<()>, Status> {
246
        self.inner_publish_lifecycle_event(grpc_request.into_inner(), get_identity())
247
            .await
248
            .map_err(Error::into)
249
    }
250
251
    #[instrument(
252
      err,
253
      level = Level::ERROR,
254
      skip_all,
255
      fields(request = ?grpc_request.get_ref())
256
    )]
257
    async fn publish_build_tool_event_stream(
258
        &self,
259
        grpc_request: Request<Streaming<PublishBuildToolEventStreamRequest>>,
260
    ) -> Result<Response<Self::PublishBuildToolEventStreamStream>, Status> {
261
        self.inner_publish_build_tool_event_stream(grpc_request.into_inner(), get_identity())
262
            .await
263
            .map_err(Error::into)
264
    }
265
}