/build/source/nativelink-scheduler/src/awaited_action_db/awaited_action.rs
Line | Count | Source |
1 | | // Copyright 2024 The NativeLink Authors. All rights reserved. |
2 | | // |
3 | | // Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // See LICENSE file for details |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | use std::sync::Arc; |
16 | | use std::time::{SystemTime, UNIX_EPOCH}; |
17 | | |
18 | | use nativelink_error::{Error, ResultExt, make_input_err}; |
19 | | use nativelink_metric::{ |
20 | | MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent, |
21 | | }; |
22 | | use nativelink_util::action_messages::{ |
23 | | ActionInfo, ActionStage, ActionState, OperationId, WorkerId, |
24 | | }; |
25 | | use nativelink_util::origin_event::{ |
26 | | BAZEL_METADATA_KEY, OriginMetadata, request_metadata_from_baggage, |
27 | | }; |
28 | | use opentelemetry::baggage::BaggageExt; |
29 | | use opentelemetry::context::Context; |
30 | | use opentelemetry_semantic_conventions::attribute::ENDUSER_ID; |
31 | | use serde::{Deserialize, Serialize}; |
32 | | use static_assertions::{assert_eq_size, const_assert, const_assert_eq}; |
33 | | |
34 | | /// The version of the awaited action. |
35 | | /// This number will always increment by one each time |
36 | | /// the action is updated. |
37 | | #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] |
38 | | struct AwaitedActionVersion(i64); |
39 | | |
40 | | impl MetricsComponent for AwaitedActionVersion { |
41 | 0 | fn publish( |
42 | 0 | &self, |
43 | 0 | _kind: MetricKind, |
44 | 0 | _field_metadata: MetricFieldData, |
45 | 0 | ) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> { |
46 | 0 | Ok(MetricPublishKnownKindData::Counter(u64::from_ne_bytes( |
47 | 0 | self.0.to_ne_bytes(), |
48 | 0 | ))) |
49 | 0 | } |
50 | | } |
51 | | |
52 | | /// An action that is being awaited on and last known state. |
53 | | #[derive(Debug, Clone, MetricsComponent, Serialize, Deserialize)] |
54 | | pub struct AwaitedAction { |
55 | | /// The current version of the action. |
56 | | #[metric(help = "The version of the AwaitedAction")] |
57 | | version: AwaitedActionVersion, |
58 | | |
59 | | /// The action that is being awaited on. |
60 | | #[metric(help = "The action info of the AwaitedAction")] |
61 | | action_info: Arc<ActionInfo>, |
62 | | |
63 | | /// The operation id of the action. |
64 | | // If you need the client operation id, it may be set in |
65 | | // ActionState::operation_id. |
66 | | #[metric(help = "The operation id of the AwaitedAction")] |
67 | | operation_id: OperationId, |
68 | | |
69 | | /// The currentsort key used to order the actions. |
70 | | #[metric(help = "The sort key of the AwaitedAction")] |
71 | | sort_key: AwaitedActionSortKey, |
72 | | |
73 | | /// The time the action was last updated. |
74 | | #[metric(help = "The last time the worker updated the AwaitedAction")] |
75 | | last_worker_updated_timestamp: SystemTime, |
76 | | |
77 | | /// The last time the client sent a keepalive message. |
78 | | #[metric(help = "The last time the client sent a keepalive message")] |
79 | | last_client_keepalive_timestamp: SystemTime, |
80 | | |
81 | | /// Worker that is currently running this action, None if unassigned. |
82 | | #[metric(help = "The worker id of the AwaitedAction")] |
83 | | worker_id: Option<WorkerId>, |
84 | | |
85 | | /// The current state of the action. |
86 | | #[metric(help = "The state of the AwaitedAction")] |
87 | | state: Arc<ActionState>, |
88 | | |
89 | | /// The origin metadata of the action. |
90 | | maybe_origin_metadata: Option<OriginMetadata>, |
91 | | |
92 | | /// Number of attempts the job has been tried. |
93 | | #[metric(help = "The number of attempts the AwaitedAction has been tried")] |
94 | | pub attempts: usize, |
95 | | } |
96 | | |
97 | | impl AwaitedAction { |
98 | 46 | pub fn new(operation_id: OperationId, action_info: Arc<ActionInfo>, now: SystemTime) -> Self { |
99 | 46 | let sort_key = AwaitedActionSortKey::new_with_unique_key( |
100 | 46 | action_info.priority, |
101 | 46 | &action_info.insert_timestamp, |
102 | | ); |
103 | 46 | let action_state = Arc::new(ActionState { |
104 | 46 | stage: ActionStage::Queued, |
105 | 46 | // Note: We don't use the real client_operation_id here because |
106 | 46 | // the only place AwaitedAction::new should ever be called is |
107 | 46 | // when the action is first created and this struct will be stored |
108 | 46 | // in the database, so we don't want to accidentally leak the |
109 | 46 | // client_operation_id to all clients. |
110 | 46 | client_operation_id: operation_id.clone(), |
111 | 46 | action_digest: action_info.unique_qualifier.digest(), |
112 | 46 | last_transition_timestamp: now, |
113 | 46 | }); |
114 | | |
115 | 46 | let ctx = Context::current(); |
116 | 46 | let baggage = ctx.baggage(); |
117 | | |
118 | 46 | let maybe_origin_metadata = if baggage.is_empty() { |
119 | 41 | None |
120 | | } else { |
121 | 5 | let bazel_metadata = baggage |
122 | 5 | .get(BAZEL_METADATA_KEY) |
123 | 5 | .and_then(|value| request_metadata_from_baggage1 (value.as_str()1 ).ok1 ()); |
124 | | Some(OriginMetadata { |
125 | 5 | identity: baggage |
126 | 5 | .get(ENDUSER_ID) |
127 | 5 | .map(|v| v.as_str().to_string()) |
128 | 5 | .unwrap_or_default(), |
129 | 5 | bazel_metadata, |
130 | | }) |
131 | | }; |
132 | | |
133 | 46 | Self { |
134 | 46 | version: AwaitedActionVersion(0), |
135 | 46 | action_info, |
136 | 46 | operation_id, |
137 | 46 | sort_key, |
138 | 46 | attempts: 0, |
139 | 46 | last_worker_updated_timestamp: now, |
140 | 46 | last_client_keepalive_timestamp: now, |
141 | 46 | maybe_origin_metadata, |
142 | 46 | worker_id: None, |
143 | 46 | state: action_state, |
144 | 46 | } |
145 | 46 | } |
146 | | |
147 | 113 | pub(crate) const fn version(&self) -> i64 { |
148 | 113 | self.version.0 |
149 | 113 | } |
150 | | |
151 | 57 | pub(crate) const fn set_version(&mut self, version: i64) { |
152 | 57 | self.version = AwaitedActionVersion(version); |
153 | 57 | } |
154 | | |
155 | 46 | pub(crate) const fn increment_version(&mut self) { |
156 | 46 | self.version = AwaitedActionVersion(self.version.0 + 1); |
157 | 46 | } |
158 | | |
159 | 444 | pub const fn action_info(&self) -> &Arc<ActionInfo> { |
160 | 444 | &self.action_info |
161 | 444 | } |
162 | | |
163 | 161 | pub const fn operation_id(&self) -> &OperationId { |
164 | 161 | &self.operation_id |
165 | 161 | } |
166 | | |
167 | 92 | pub(crate) const fn sort_key(&self) -> AwaitedActionSortKey { |
168 | 92 | self.sort_key |
169 | 92 | } |
170 | | |
171 | 1.22k | pub const fn state(&self) -> &Arc<ActionState> { |
172 | 1.22k | &self.state |
173 | 1.22k | } |
174 | | |
175 | 17 | pub fn is_complete(&self) -> bool { |
176 | 17 | match &self.state.stage { |
177 | | ActionStage::Unknown |
178 | | | ActionStage::CacheCheck |
179 | | | ActionStage::Queued |
180 | 16 | | ActionStage::Executing => false, |
181 | 1 | ActionStage::Completed(_) | ActionStage::CompletedFromCache(_) => true, |
182 | | } |
183 | 17 | } |
184 | | |
185 | 159 | pub(crate) const fn maybe_origin_metadata(&self) -> Option<&OriginMetadata> { |
186 | 159 | self.maybe_origin_metadata.as_ref() |
187 | 159 | } |
188 | | |
189 | 142 | pub(crate) const fn worker_id(&self) -> Option<&WorkerId> { |
190 | 142 | self.worker_id.as_ref() |
191 | 142 | } |
192 | | |
193 | 567 | pub(crate) const fn last_worker_updated_timestamp(&self) -> SystemTime { |
194 | 567 | self.last_worker_updated_timestamp |
195 | 567 | } |
196 | | |
197 | 111 | pub(crate) const fn worker_keep_alive(&mut self, now: SystemTime) { |
198 | 111 | self.last_worker_updated_timestamp = now; |
199 | 111 | } |
200 | | |
201 | 609 | pub(crate) const fn last_client_keepalive_timestamp(&self) -> SystemTime { |
202 | 609 | self.last_client_keepalive_timestamp |
203 | 609 | } |
204 | | |
205 | 61 | pub(crate) const fn update_client_keep_alive(&mut self, now: SystemTime) { |
206 | 61 | self.last_client_keepalive_timestamp = now; |
207 | 61 | } |
208 | | |
209 | 72 | pub(crate) fn set_client_operation_id(&mut self, client_operation_id: OperationId) { |
210 | 72 | Arc::make_mut(&mut self.state).client_operation_id = client_operation_id; |
211 | 72 | } |
212 | | |
213 | | /// Sets the worker id that is currently processing this action. |
214 | 59 | pub(crate) fn set_worker_id(&mut self, new_maybe_worker_id: Option<WorkerId>, now: SystemTime) { |
215 | 59 | if self.worker_id != new_maybe_worker_id { |
216 | 51 | self.worker_id = new_maybe_worker_id; |
217 | 51 | self.worker_keep_alive(now); |
218 | 51 | }8 |
219 | 59 | } |
220 | | |
221 | | /// Sets the current state of the action and updates the last worker updated timestamp. |
222 | 60 | pub fn worker_set_state(&mut self, mut state: Arc<ActionState>, now: SystemTime) { |
223 | 60 | core::mem::swap(&mut self.state, &mut state); |
224 | 60 | self.worker_keep_alive(now); |
225 | 60 | } |
226 | | } |
227 | | |
228 | | impl TryFrom<&[u8]> for AwaitedAction { |
229 | | type Error = Error; |
230 | 0 | fn try_from(value: &[u8]) -> Result<Self, Self::Error> { |
231 | 0 | serde_json::from_slice(value) |
232 | 0 | .map_err(|e| make_input_err!("{}", e.to_string())) |
233 | 0 | .err_tip(|| "In AwaitedAction::TryFrom::&[u8]") |
234 | 0 | } |
235 | | } |
236 | | |
237 | | /// The key used to sort the awaited actions. |
238 | | /// |
239 | | /// The rules for sorting are as follows: |
240 | | /// 1. priority of the action |
241 | | /// 2. insert order of the action (lower = higher priority) |
242 | | /// 3. (mostly random hash based on the action info) |
243 | | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] |
244 | | #[repr(transparent)] |
245 | | pub struct AwaitedActionSortKey(u64); |
246 | | |
247 | | impl MetricsComponent for AwaitedActionSortKey { |
248 | 0 | fn publish( |
249 | 0 | &self, |
250 | 0 | _kind: MetricKind, |
251 | 0 | _field_metadata: MetricFieldData, |
252 | 0 | ) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> { |
253 | 0 | Ok(MetricPublishKnownKindData::Counter(self.0)) |
254 | 0 | } |
255 | | } |
256 | | |
257 | | impl AwaitedActionSortKey { |
258 | 46 | const fn new(priority: i32, insert_timestamp: u32) -> Self { |
259 | | // Shift the signed i32 range [i32::MIN, i32::MAX] to the unsigned u32 range |
260 | | // [0, u32::MAX] to preserve ordering when we convert to bytes for sorting. |
261 | 46 | let priority_u32 = i32::MIN.unsigned_abs().wrapping_add_signed(priority); |
262 | 46 | let priority = priority_u32.to_be_bytes(); |
263 | | |
264 | | // Invert our timestamp so the larger the timestamp the lower the number. |
265 | | // This makes timestamp descending order instead of ascending. |
266 | 46 | let timestamp = (insert_timestamp ^ u32::MAX).to_be_bytes(); |
267 | | |
268 | 46 | Self(u64::from_be_bytes([ |
269 | 46 | priority[0], |
270 | 46 | priority[1], |
271 | 46 | priority[2], |
272 | 46 | priority[3], |
273 | 46 | timestamp[0], |
274 | 46 | timestamp[1], |
275 | 46 | timestamp[2], |
276 | 46 | timestamp[3], |
277 | 46 | ])) |
278 | 46 | } |
279 | | |
280 | 46 | fn new_with_unique_key(priority: i32, insert_timestamp: &SystemTime) -> Self { |
281 | 46 | let timestamp = u32::try_from( |
282 | 46 | insert_timestamp |
283 | 46 | .duration_since(UNIX_EPOCH) |
284 | 46 | .unwrap() |
285 | 46 | .as_secs(), |
286 | | ) |
287 | 46 | .unwrap_or(u32::MAX); |
288 | 46 | Self::new(priority, timestamp) |
289 | 46 | } |
290 | | |
291 | 17 | pub(crate) const fn as_u64(self) -> u64 { |
292 | 17 | self.0 |
293 | 17 | } |
294 | | } |
295 | | |
296 | | // Ensure the size of the sort key is the same as a `u64`. |
297 | | assert_eq_size!(AwaitedActionSortKey, u64); |
298 | | |
299 | | const_assert_eq!( |
300 | | AwaitedActionSortKey::new(0x1234_5678, 0x9abc_def0).0, |
301 | | // Note: Result has 0x12345678 + 0x80000000 = 0x92345678 because we need |
302 | | // to shift the `i32::MIN` value to be represented by zero. |
303 | | // Note: `6543210f` are the inverted bits of `9abcdef0`. |
304 | | // This effectively inverts the priority to now have the highest priority |
305 | | // be the lowest timestamps. |
306 | | AwaitedActionSortKey(0x9234_5678_6543_210f).0 |
307 | | ); |
308 | | // Ensure the priority is used as the sort key first. |
309 | | const_assert!( |
310 | | AwaitedActionSortKey::new(i32::MAX, 0).0 > AwaitedActionSortKey::new(i32::MAX - 1, 0).0 |
311 | | ); |
312 | | const_assert!(AwaitedActionSortKey::new(i32::MAX - 1, 0).0 > AwaitedActionSortKey::new(1, 0).0); |
313 | | const_assert!(AwaitedActionSortKey::new(1, 0).0 > AwaitedActionSortKey::new(0, 0).0); |
314 | | const_assert!(AwaitedActionSortKey::new(0, 0).0 > AwaitedActionSortKey::new(-1, 0).0); |
315 | | const_assert!(AwaitedActionSortKey::new(-1, 0).0 > AwaitedActionSortKey::new(i32::MIN + 1, 0).0); |
316 | | const_assert!( |
317 | | AwaitedActionSortKey::new(i32::MIN + 1, 0).0 > AwaitedActionSortKey::new(i32::MIN, 0).0 |
318 | | ); |
319 | | |
320 | | // Ensure the insert timestamp is used as the sort key second. |
321 | | const_assert!(AwaitedActionSortKey::new(0, u32::MIN).0 > AwaitedActionSortKey::new(0, u32::MAX).0); |