Coverage Report

Created: 2026-09-18 20:40

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-scheduler/src/awaited_action_db/awaited_action.rs
Line
Count
Source
1
// Copyright 2024 The NativeLink Authors. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use std::sync::Arc;
16
use std::time::{SystemTime, UNIX_EPOCH};
17
18
use nativelink_error::{Error, ResultExt, make_input_err};
19
use nativelink_metric::{
20
    MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent,
21
};
22
use nativelink_util::action_messages::{
23
    ActionInfo, ActionStage, ActionState, OperationId, WorkerId,
24
};
25
use nativelink_util::origin_event::{
26
    BAZEL_METADATA_KEY, OriginMetadata, request_metadata_from_baggage,
27
};
28
use opentelemetry::baggage::BaggageExt;
29
use opentelemetry::context::Context;
30
use opentelemetry_semantic_conventions::attribute::ENDUSER_ID;
31
use serde::{Deserialize, Serialize};
32
use static_assertions::{assert_eq_size, const_assert, const_assert_eq};
33
34
/// The version of the awaited action.
35
/// This number will always increment by one each time
36
/// the action is updated.
37
#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)]
38
struct AwaitedActionVersion(i64);
39
40
impl MetricsComponent for AwaitedActionVersion {
41
0
    fn publish(
42
0
        &self,
43
0
        _kind: MetricKind,
44
0
        _field_metadata: MetricFieldData,
45
0
    ) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> {
46
0
        Ok(MetricPublishKnownKindData::Counter(u64::from_ne_bytes(
47
0
            self.0.to_ne_bytes(),
48
0
        )))
49
0
    }
50
}
51
52
/// An action that is being awaited on and last known state.
53
#[derive(Debug, Clone, MetricsComponent, Serialize, Deserialize)]
54
pub struct AwaitedAction {
55
    /// The current version of the action.
56
    #[metric(help = "The version of the AwaitedAction")]
57
    version: AwaitedActionVersion,
58
59
    /// The action that is being awaited on.
60
    #[metric(help = "The action info of the AwaitedAction")]
61
    action_info: Arc<ActionInfo>,
62
63
    /// The operation id of the action.
64
    // If you need the client operation id, it may be set in
65
    // ActionState::operation_id.
66
    #[metric(help = "The operation id of the AwaitedAction")]
67
    operation_id: OperationId,
68
69
    /// The currentsort key used to order the actions.
70
    #[metric(help = "The sort key of the AwaitedAction")]
71
    sort_key: AwaitedActionSortKey,
72
73
    /// The time the action was last updated.
74
    #[metric(help = "The last time the worker updated the AwaitedAction")]
75
    last_worker_updated_timestamp: SystemTime,
76
77
    /// The last time the client sent a keepalive message.
78
    #[metric(help = "The last time the client sent a keepalive message")]
79
    last_client_keepalive_timestamp: SystemTime,
80
81
    /// Worker that is currently running this action, None if unassigned.
82
    #[metric(help = "The worker id of the AwaitedAction")]
83
    worker_id: Option<WorkerId>,
84
85
    /// The current state of the action.
86
    #[metric(help = "The state of the AwaitedAction")]
87
    state: Arc<ActionState>,
88
89
    /// The origin metadata of the action.
90
    maybe_origin_metadata: Option<OriginMetadata>,
91
92
    /// Number of attempts the job has been tried.
93
    #[metric(help = "The number of attempts the AwaitedAction has been tried")]
94
    pub attempts: usize,
95
}
96
97
impl AwaitedAction {
98
74
    pub fn new(operation_id: OperationId, action_info: Arc<ActionInfo>, now: SystemTime) -> Self {
99
74
        let sort_key = AwaitedActionSortKey::new_with_unique_key(
100
74
            action_info.priority,
101
74
            &action_info.insert_timestamp,
102
        );
103
74
        let action_state = Arc::new(ActionState {
104
74
            stage: ActionStage::Queued,
105
74
            // Note: We don't use the real client_operation_id here because
106
74
            // the only place AwaitedAction::new should ever be called is
107
74
            // when the action is first created and this struct will be stored
108
74
            // in the database, so we don't want to accidentally leak the
109
74
            // client_operation_id to all clients.
110
74
            client_operation_id: operation_id.clone(),
111
74
            action_digest: action_info.unique_qualifier.digest(),
112
74
            last_transition_timestamp: now,
113
74
        });
114
115
74
        let ctx = Context::current();
116
74
        let baggage = ctx.baggage();
117
118
74
        let maybe_origin_metadata = if baggage.is_empty() {
119
69
            None
120
        } else {
121
5
            let bazel_metadata = baggage
122
5
                .get(BAZEL_METADATA_KEY)
123
5
                .and_then(|value| 
request_metadata_from_baggage1
(
value.as_str()1
).
ok1
());
124
            Some(OriginMetadata {
125
5
                identity: baggage
126
5
                    .get(ENDUSER_ID)
127
5
                    .map(|v| v.as_str().to_string())
128
5
                    .unwrap_or_default(),
129
5
                bazel_metadata,
130
            })
131
        };
132
133
74
        Self {
134
74
            version: AwaitedActionVersion(0),
135
74
            action_info,
136
74
            operation_id,
137
74
            sort_key,
138
74
            attempts: 0,
139
74
            last_worker_updated_timestamp: now,
140
74
            last_client_keepalive_timestamp: now,
141
74
            maybe_origin_metadata,
142
74
            worker_id: None,
143
74
            state: action_state,
144
74
        }
145
74
    }
146
147
149
    pub(crate) const fn version(&self) -> i64 {
148
149
        self.version.0
149
149
    }
150
151
63
    pub(crate) const fn set_version(&mut self, version: i64) {
152
63
        self.version = AwaitedActionVersion(version);
153
63
    }
154
155
63
    pub(crate) const fn increment_version(&mut self) {
156
63
        self.version = AwaitedActionVersion(self.version.0 + 1);
157
63
    }
158
159
584
    pub const fn action_info(&self) -> &Arc<ActionInfo> {
160
584
        &self.action_info
161
584
    }
162
163
219
    pub const fn operation_id(&self) -> &OperationId {
164
219
        &self.operation_id
165
219
    }
166
167
129
    pub(crate) const fn sort_key(&self) -> AwaitedActionSortKey {
168
129
        self.sort_key
169
129
    }
170
171
1.46k
    pub const fn state(&self) -> &Arc<ActionState> {
172
1.46k
        &self.state
173
1.46k
    }
174
175
19
    pub fn is_complete(&self) -> bool {
176
19
        match &self.state.stage {
177
            ActionStage::Unknown
178
            | ActionStage::CacheCheck
179
            | ActionStage::Queued
180
18
            | ActionStage::Executing => false,
181
1
            ActionStage::Completed(_) | ActionStage::CompletedFromCache(_) => true,
182
        }
183
19
    }
184
185
177
    pub(crate) const fn maybe_origin_metadata(&self) -> Option<&OriginMetadata> {
186
177
        self.maybe_origin_metadata.as_ref()
187
177
    }
188
189
189
    pub(crate) const fn worker_id(&self) -> Option<&WorkerId> {
190
189
        self.worker_id.as_ref()
191
189
    }
192
193
590
    pub(crate) const fn last_worker_updated_timestamp(&self) -> SystemTime {
194
590
        self.last_worker_updated_timestamp
195
590
    }
196
197
159
    pub(crate) const fn worker_keep_alive(&mut self, now: SystemTime) {
198
159
        self.last_worker_updated_timestamp = now;
199
159
    }
200
201
628
    pub(crate) const fn last_client_keepalive_timestamp(&self) -> SystemTime {
202
628
        self.last_client_keepalive_timestamp
203
628
    }
204
205
62
    pub(crate) const fn update_client_keep_alive(&mut self, now: SystemTime) {
206
62
        self.last_client_keepalive_timestamp = now;
207
62
    }
208
209
77
    pub(crate) fn set_client_operation_id(&mut self, client_operation_id: OperationId) {
210
77
        Arc::make_mut(&mut self.state).client_operation_id = client_operation_id;
211
77
    }
212
213
    /// Sets the worker id that is currently processing this action.
214
80
    pub fn set_worker_id(&mut self, new_maybe_worker_id: Option<WorkerId>, now: SystemTime) {
215
80
        if self.worker_id != new_maybe_worker_id {
216
71
            self.worker_id = new_maybe_worker_id;
217
71
            self.worker_keep_alive(now);
218
71
        
}9
219
80
    }
220
221
    /// Sets the current state of the action and updates the last worker updated timestamp.
222
88
    pub fn worker_set_state(&mut self, mut state: Arc<ActionState>, now: SystemTime) {
223
88
        core::mem::swap(&mut self.state, &mut state);
224
88
        self.worker_keep_alive(now);
225
88
    }
226
}
227
228
impl TryFrom<&[u8]> for AwaitedAction {
229
    type Error = Error;
230
0
    fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
231
0
        serde_json::from_slice(value)
232
0
            .map_err(|e| make_input_err!("{}", e.to_string()))
233
0
            .err_tip(|| "In AwaitedAction::TryFrom::&[u8]")
234
0
    }
235
}
236
237
/// The key used to sort the awaited actions.
238
///
239
/// The rules for sorting are as follows:
240
/// 1. priority of the action
241
/// 2. insert order of the action (lower = higher priority)
242
/// 3. (mostly random hash based on the action info)
243
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
244
#[repr(transparent)]
245
pub struct AwaitedActionSortKey(u64);
246
247
impl MetricsComponent for AwaitedActionSortKey {
248
0
    fn publish(
249
0
        &self,
250
0
        _kind: MetricKind,
251
0
        _field_metadata: MetricFieldData,
252
0
    ) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> {
253
0
        Ok(MetricPublishKnownKindData::Counter(self.0))
254
0
    }
255
}
256
257
impl AwaitedActionSortKey {
258
74
    const fn new(priority: i32, insert_timestamp: u32) -> Self {
259
        // Shift the signed i32 range [i32::MIN, i32::MAX] to the unsigned u32 range
260
        // [0, u32::MAX] to preserve ordering when we convert to bytes for sorting.
261
74
        let priority_u32 = i32::MIN.unsigned_abs().wrapping_add_signed(priority);
262
74
        let priority = priority_u32.to_be_bytes();
263
264
        // Invert our timestamp so the larger the timestamp the lower the number.
265
        // This makes timestamp descending order instead of ascending.
266
74
        let timestamp = (insert_timestamp ^ u32::MAX).to_be_bytes();
267
268
74
        Self(u64::from_be_bytes([
269
74
            priority[0],
270
74
            priority[1],
271
74
            priority[2],
272
74
            priority[3],
273
74
            timestamp[0],
274
74
            timestamp[1],
275
74
            timestamp[2],
276
74
            timestamp[3],
277
74
        ]))
278
74
    }
279
280
74
    fn new_with_unique_key(priority: i32, insert_timestamp: &SystemTime) -> Self {
281
74
        let timestamp = u32::try_from(
282
74
            insert_timestamp
283
74
                .duration_since(UNIX_EPOCH)
284
74
                .unwrap()
285
74
                .as_secs(),
286
        )
287
74
        .unwrap_or(u32::MAX);
288
74
        Self::new(priority, timestamp)
289
74
    }
290
291
19
    pub(crate) const fn as_u64(self) -> u64 {
292
19
        self.0
293
19
    }
294
}
295
296
// Ensure the size of the sort key is the same as a `u64`.
297
assert_eq_size!(AwaitedActionSortKey, u64);
298
299
const_assert_eq!(
300
    AwaitedActionSortKey::new(0x1234_5678, 0x9abc_def0).0,
301
    // Note: Result has 0x12345678 + 0x80000000 = 0x92345678 because we need
302
    // to shift the `i32::MIN` value to be represented by zero.
303
    // Note: `6543210f` are the inverted bits of `9abcdef0`.
304
    // This effectively inverts the priority to now have the highest priority
305
    // be the lowest timestamps.
306
    AwaitedActionSortKey(0x9234_5678_6543_210f).0
307
);
308
// Ensure the priority is used as the sort key first.
309
const_assert!(
310
    AwaitedActionSortKey::new(i32::MAX, 0).0 > AwaitedActionSortKey::new(i32::MAX - 1, 0).0
311
);
312
const_assert!(AwaitedActionSortKey::new(i32::MAX - 1, 0).0 > AwaitedActionSortKey::new(1, 0).0);
313
const_assert!(AwaitedActionSortKey::new(1, 0).0 > AwaitedActionSortKey::new(0, 0).0);
314
const_assert!(AwaitedActionSortKey::new(0, 0).0 > AwaitedActionSortKey::new(-1, 0).0);
315
const_assert!(AwaitedActionSortKey::new(-1, 0).0 > AwaitedActionSortKey::new(i32::MIN + 1, 0).0);
316
const_assert!(
317
    AwaitedActionSortKey::new(i32::MIN + 1, 0).0 > AwaitedActionSortKey::new(i32::MIN, 0).0
318
);
319
320
// Ensure the insert timestamp is used as the sort key second.
321
const_assert!(AwaitedActionSortKey::new(0, u32::MIN).0 > AwaitedActionSortKey::new(0, u32::MAX).0);