Coverage Report

Created: 2026-07-21 15:28

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-scheduler/src/awaited_action_db/awaited_action.rs
Line
Count
Source
1
// Copyright 2024 The NativeLink Authors. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use std::sync::Arc;
16
use std::time::{SystemTime, UNIX_EPOCH};
17
18
use nativelink_error::{Error, ResultExt, make_input_err};
19
use nativelink_metric::{
20
    MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent,
21
};
22
use nativelink_util::action_messages::{
23
    ActionInfo, ActionStage, ActionState, OperationId, WorkerId,
24
};
25
use nativelink_util::origin_event::{
26
    BAZEL_METADATA_KEY, OriginMetadata, request_metadata_from_baggage,
27
};
28
use opentelemetry::baggage::BaggageExt;
29
use opentelemetry::context::Context;
30
use opentelemetry_semantic_conventions::attribute::ENDUSER_ID;
31
use serde::{Deserialize, Serialize};
32
use static_assertions::{assert_eq_size, const_assert, const_assert_eq};
33
34
/// The version of the awaited action.
35
/// This number will always increment by one each time
36
/// the action is updated.
37
#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)]
38
struct AwaitedActionVersion(i64);
39
40
impl MetricsComponent for AwaitedActionVersion {
41
0
    fn publish(
42
0
        &self,
43
0
        _kind: MetricKind,
44
0
        _field_metadata: MetricFieldData,
45
0
    ) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> {
46
0
        Ok(MetricPublishKnownKindData::Counter(u64::from_ne_bytes(
47
0
            self.0.to_ne_bytes(),
48
0
        )))
49
0
    }
50
}
51
52
/// An action that is being awaited on and last known state.
53
#[derive(Debug, Clone, MetricsComponent, Serialize, Deserialize)]
54
pub struct AwaitedAction {
55
    /// The current version of the action.
56
    #[metric(help = "The version of the AwaitedAction")]
57
    version: AwaitedActionVersion,
58
59
    /// The action that is being awaited on.
60
    #[metric(help = "The action info of the AwaitedAction")]
61
    action_info: Arc<ActionInfo>,
62
63
    /// The operation id of the action.
64
    // If you need the client operation id, it may be set in
65
    // ActionState::operation_id.
66
    #[metric(help = "The operation id of the AwaitedAction")]
67
    operation_id: OperationId,
68
69
    /// The currentsort key used to order the actions.
70
    #[metric(help = "The sort key of the AwaitedAction")]
71
    sort_key: AwaitedActionSortKey,
72
73
    /// The time the action was last updated.
74
    #[metric(help = "The last time the worker updated the AwaitedAction")]
75
    last_worker_updated_timestamp: SystemTime,
76
77
    /// The last time the client sent a keepalive message.
78
    #[metric(help = "The last time the client sent a keepalive message")]
79
    last_client_keepalive_timestamp: SystemTime,
80
81
    /// Worker that is currently running this action, None if unassigned.
82
    #[metric(help = "The worker id of the AwaitedAction")]
83
    worker_id: Option<WorkerId>,
84
85
    /// The current state of the action.
86
    #[metric(help = "The state of the AwaitedAction")]
87
    state: Arc<ActionState>,
88
89
    /// The origin metadata of the action.
90
    maybe_origin_metadata: Option<OriginMetadata>,
91
92
    /// Number of attempts the job has been tried.
93
    #[metric(help = "The number of attempts the AwaitedAction has been tried")]
94
    pub attempts: usize,
95
}
96
97
impl AwaitedAction {
98
46
    pub fn new(operation_id: OperationId, action_info: Arc<ActionInfo>, now: SystemTime) -> Self {
99
46
        let sort_key = AwaitedActionSortKey::new_with_unique_key(
100
46
            action_info.priority,
101
46
            &action_info.insert_timestamp,
102
        );
103
46
        let action_state = Arc::new(ActionState {
104
46
            stage: ActionStage::Queued,
105
46
            // Note: We don't use the real client_operation_id here because
106
46
            // the only place AwaitedAction::new should ever be called is
107
46
            // when the action is first created and this struct will be stored
108
46
            // in the database, so we don't want to accidentally leak the
109
46
            // client_operation_id to all clients.
110
46
            client_operation_id: operation_id.clone(),
111
46
            action_digest: action_info.unique_qualifier.digest(),
112
46
            last_transition_timestamp: now,
113
46
        });
114
115
46
        let ctx = Context::current();
116
46
        let baggage = ctx.baggage();
117
118
46
        let maybe_origin_metadata = if baggage.is_empty() {
119
41
            None
120
        } else {
121
5
            let bazel_metadata = baggage
122
5
                .get(BAZEL_METADATA_KEY)
123
5
                .and_then(|value| 
request_metadata_from_baggage1
(
value.as_str()1
).
ok1
());
124
            Some(OriginMetadata {
125
5
                identity: baggage
126
5
                    .get(ENDUSER_ID)
127
5
                    .map(|v| v.as_str().to_string())
128
5
                    .unwrap_or_default(),
129
5
                bazel_metadata,
130
            })
131
        };
132
133
46
        Self {
134
46
            version: AwaitedActionVersion(0),
135
46
            action_info,
136
46
            operation_id,
137
46
            sort_key,
138
46
            attempts: 0,
139
46
            last_worker_updated_timestamp: now,
140
46
            last_client_keepalive_timestamp: now,
141
46
            maybe_origin_metadata,
142
46
            worker_id: None,
143
46
            state: action_state,
144
46
        }
145
46
    }
146
147
113
    pub(crate) const fn version(&self) -> i64 {
148
113
        self.version.0
149
113
    }
150
151
57
    pub(crate) const fn set_version(&mut self, version: i64) {
152
57
        self.version = AwaitedActionVersion(version);
153
57
    }
154
155
46
    pub(crate) const fn increment_version(&mut self) {
156
46
        self.version = AwaitedActionVersion(self.version.0 + 1);
157
46
    }
158
159
444
    pub const fn action_info(&self) -> &Arc<ActionInfo> {
160
444
        &self.action_info
161
444
    }
162
163
161
    pub const fn operation_id(&self) -> &OperationId {
164
161
        &self.operation_id
165
161
    }
166
167
92
    pub(crate) const fn sort_key(&self) -> AwaitedActionSortKey {
168
92
        self.sort_key
169
92
    }
170
171
1.22k
    pub const fn state(&self) -> &Arc<ActionState> {
172
1.22k
        &self.state
173
1.22k
    }
174
175
17
    pub fn is_complete(&self) -> bool {
176
17
        match &self.state.stage {
177
            ActionStage::Unknown
178
            | ActionStage::CacheCheck
179
            | ActionStage::Queued
180
16
            | ActionStage::Executing => false,
181
1
            ActionStage::Completed(_) | ActionStage::CompletedFromCache(_) => true,
182
        }
183
17
    }
184
185
159
    pub(crate) const fn maybe_origin_metadata(&self) -> Option<&OriginMetadata> {
186
159
        self.maybe_origin_metadata.as_ref()
187
159
    }
188
189
142
    pub(crate) const fn worker_id(&self) -> Option<&WorkerId> {
190
142
        self.worker_id.as_ref()
191
142
    }
192
193
567
    pub(crate) const fn last_worker_updated_timestamp(&self) -> SystemTime {
194
567
        self.last_worker_updated_timestamp
195
567
    }
196
197
111
    pub(crate) const fn worker_keep_alive(&mut self, now: SystemTime) {
198
111
        self.last_worker_updated_timestamp = now;
199
111
    }
200
201
609
    pub(crate) const fn last_client_keepalive_timestamp(&self) -> SystemTime {
202
609
        self.last_client_keepalive_timestamp
203
609
    }
204
205
61
    pub(crate) const fn update_client_keep_alive(&mut self, now: SystemTime) {
206
61
        self.last_client_keepalive_timestamp = now;
207
61
    }
208
209
72
    pub(crate) fn set_client_operation_id(&mut self, client_operation_id: OperationId) {
210
72
        Arc::make_mut(&mut self.state).client_operation_id = client_operation_id;
211
72
    }
212
213
    /// Sets the worker id that is currently processing this action.
214
59
    pub(crate) fn set_worker_id(&mut self, new_maybe_worker_id: Option<WorkerId>, now: SystemTime) {
215
59
        if self.worker_id != new_maybe_worker_id {
216
51
            self.worker_id = new_maybe_worker_id;
217
51
            self.worker_keep_alive(now);
218
51
        
}8
219
59
    }
220
221
    /// Sets the current state of the action and updates the last worker updated timestamp.
222
60
    pub fn worker_set_state(&mut self, mut state: Arc<ActionState>, now: SystemTime) {
223
60
        core::mem::swap(&mut self.state, &mut state);
224
60
        self.worker_keep_alive(now);
225
60
    }
226
}
227
228
impl TryFrom<&[u8]> for AwaitedAction {
229
    type Error = Error;
230
0
    fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
231
0
        serde_json::from_slice(value)
232
0
            .map_err(|e| make_input_err!("{}", e.to_string()))
233
0
            .err_tip(|| "In AwaitedAction::TryFrom::&[u8]")
234
0
    }
235
}
236
237
/// The key used to sort the awaited actions.
238
///
239
/// The rules for sorting are as follows:
240
/// 1. priority of the action
241
/// 2. insert order of the action (lower = higher priority)
242
/// 3. (mostly random hash based on the action info)
243
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
244
#[repr(transparent)]
245
pub struct AwaitedActionSortKey(u64);
246
247
impl MetricsComponent for AwaitedActionSortKey {
248
0
    fn publish(
249
0
        &self,
250
0
        _kind: MetricKind,
251
0
        _field_metadata: MetricFieldData,
252
0
    ) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> {
253
0
        Ok(MetricPublishKnownKindData::Counter(self.0))
254
0
    }
255
}
256
257
impl AwaitedActionSortKey {
258
46
    const fn new(priority: i32, insert_timestamp: u32) -> Self {
259
        // Shift the signed i32 range [i32::MIN, i32::MAX] to the unsigned u32 range
260
        // [0, u32::MAX] to preserve ordering when we convert to bytes for sorting.
261
46
        let priority_u32 = i32::MIN.unsigned_abs().wrapping_add_signed(priority);
262
46
        let priority = priority_u32.to_be_bytes();
263
264
        // Invert our timestamp so the larger the timestamp the lower the number.
265
        // This makes timestamp descending order instead of ascending.
266
46
        let timestamp = (insert_timestamp ^ u32::MAX).to_be_bytes();
267
268
46
        Self(u64::from_be_bytes([
269
46
            priority[0],
270
46
            priority[1],
271
46
            priority[2],
272
46
            priority[3],
273
46
            timestamp[0],
274
46
            timestamp[1],
275
46
            timestamp[2],
276
46
            timestamp[3],
277
46
        ]))
278
46
    }
279
280
46
    fn new_with_unique_key(priority: i32, insert_timestamp: &SystemTime) -> Self {
281
46
        let timestamp = u32::try_from(
282
46
            insert_timestamp
283
46
                .duration_since(UNIX_EPOCH)
284
46
                .unwrap()
285
46
                .as_secs(),
286
        )
287
46
        .unwrap_or(u32::MAX);
288
46
        Self::new(priority, timestamp)
289
46
    }
290
291
17
    pub(crate) const fn as_u64(self) -> u64 {
292
17
        self.0
293
17
    }
294
}
295
296
// Ensure the size of the sort key is the same as a `u64`.
297
assert_eq_size!(AwaitedActionSortKey, u64);
298
299
const_assert_eq!(
300
    AwaitedActionSortKey::new(0x1234_5678, 0x9abc_def0).0,
301
    // Note: Result has 0x12345678 + 0x80000000 = 0x92345678 because we need
302
    // to shift the `i32::MIN` value to be represented by zero.
303
    // Note: `6543210f` are the inverted bits of `9abcdef0`.
304
    // This effectively inverts the priority to now have the highest priority
305
    // be the lowest timestamps.
306
    AwaitedActionSortKey(0x9234_5678_6543_210f).0
307
);
308
// Ensure the priority is used as the sort key first.
309
const_assert!(
310
    AwaitedActionSortKey::new(i32::MAX, 0).0 > AwaitedActionSortKey::new(i32::MAX - 1, 0).0
311
);
312
const_assert!(AwaitedActionSortKey::new(i32::MAX - 1, 0).0 > AwaitedActionSortKey::new(1, 0).0);
313
const_assert!(AwaitedActionSortKey::new(1, 0).0 > AwaitedActionSortKey::new(0, 0).0);
314
const_assert!(AwaitedActionSortKey::new(0, 0).0 > AwaitedActionSortKey::new(-1, 0).0);
315
const_assert!(AwaitedActionSortKey::new(-1, 0).0 > AwaitedActionSortKey::new(i32::MIN + 1, 0).0);
316
const_assert!(
317
    AwaitedActionSortKey::new(i32::MIN + 1, 0).0 > AwaitedActionSortKey::new(i32::MIN, 0).0
318
);
319
320
// Ensure the insert timestamp is used as the sort key second.
321
const_assert!(AwaitedActionSortKey::new(0, u32::MIN).0 > AwaitedActionSortKey::new(0, u32::MAX).0);