/build/source/nativelink-util/src/operation_state_manager.rs
Line | Count | Source |
1 | | // Copyright 2024 Trace Machina, Inc. All rights reserved. |
2 | | // |
3 | | // Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // See LICENSE file for details |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | use core::pin::Pin; |
16 | | use std::sync::Arc; |
17 | | use std::time::SystemTime; |
18 | | |
19 | | use async_trait::async_trait; |
20 | | use bitflags::bitflags; |
21 | | use futures::Stream; |
22 | | use nativelink_error::Error; |
23 | | use nativelink_metric::MetricsComponent; |
24 | | |
25 | | use crate::action_messages::{ |
26 | | ActionInfo, ActionStage, ActionState, ActionUniqueKey, OperationId, WorkerId, |
27 | | }; |
28 | | use crate::common::DigestInfo; |
29 | | use crate::origin_event::OriginMetadata; |
30 | | |
31 | | bitflags! { |
32 | | #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] |
33 | | pub struct OperationStageFlags: u32 { |
34 | | const CacheCheck = 1 << 1; |
35 | | const Queued = 1 << 2; |
36 | | const Executing = 1 << 3; |
37 | | const Completed = 1 << 4; |
38 | | const Any = u32::MAX; |
39 | | } |
40 | | } |
41 | | |
42 | | impl Default for OperationStageFlags { |
43 | 1.47k | fn default() -> Self { |
44 | 1.47k | Self::Any |
45 | 1.47k | } |
46 | | } |
47 | | |
48 | | #[async_trait] |
49 | | pub trait ActionStateResult: Send + Sync + 'static { |
50 | | /// Provides the current state of the action. |
51 | | async fn as_state(&self) -> Result<(Arc<ActionState>, Option<OriginMetadata>), Error>; |
52 | | /// Waits for the state of the action to change. |
53 | | async fn changed(&mut self) -> Result<(Arc<ActionState>, Option<OriginMetadata>), Error>; |
54 | | /// Provide result as action info. This behavior will not be supported by all implementations. |
55 | | async fn as_action_info(&self) -> Result<(Arc<ActionInfo>, Option<OriginMetadata>), Error>; |
56 | | /// When a client last checked in on this operation, where the |
57 | | /// implementation tracks it. `None` where it does not. |
58 | 3 | async fn client_last_seen(&self) -> Result<Option<SystemTime>, Error> { |
59 | | Ok(None) |
60 | 3 | } |
61 | | } |
62 | | |
63 | | /// The direction in which the results are ordered. |
64 | | #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] |
65 | | pub enum OrderDirection { |
66 | | Asc, |
67 | | Desc, |
68 | | } |
69 | | |
70 | | /// The filters used to query operations from the state manager. |
71 | | #[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] |
72 | | pub struct OperationFilter { |
73 | | // TODO(palfrey): create rust builder pattern? |
74 | | /// The stage(s) that the operation must be in. |
75 | | pub stages: OperationStageFlags, |
76 | | |
77 | | /// The client operation id. |
78 | | pub client_operation_id: Option<OperationId>, |
79 | | |
80 | | /// The operation id. |
81 | | pub operation_id: Option<OperationId>, |
82 | | |
83 | | /// The worker that the operation must be assigned to. |
84 | | pub worker_id: Option<WorkerId>, |
85 | | |
86 | | /// The digest of the action that the operation must have. |
87 | | pub action_digest: Option<DigestInfo>, |
88 | | |
89 | | /// The operation must have its worker timestamp before this time. |
90 | | pub worker_update_before: Option<SystemTime>, |
91 | | |
92 | | /// The operation must have been completed before this time. |
93 | | pub completed_before: Option<SystemTime>, |
94 | | |
95 | | /// The unique key for filtering specific action results. |
96 | | pub unique_key: Option<ActionUniqueKey>, |
97 | | |
98 | | /// If the results should be ordered by priority and in which direction. |
99 | | pub order_by_priority_direction: Option<OrderDirection>, |
100 | | } |
101 | | |
102 | | pub type ActionStateResultStream<'a> = |
103 | | Pin<Box<dyn Stream<Item = Box<dyn ActionStateResult>> + Send + 'a>>; |
104 | | |
105 | | #[async_trait] |
106 | | pub trait ClientStateManager: Sync + Send + Unpin + MetricsComponent + 'static { |
107 | | /// Add a new action to the queue or joins an existing action. |
108 | | async fn add_action( |
109 | | &self, |
110 | | client_operation_id: OperationId, |
111 | | action_info: Arc<ActionInfo>, |
112 | | ) -> Result<Box<dyn ActionStateResult>, Error>; |
113 | | |
114 | | /// Returns a stream of operations that match the filter. |
115 | | async fn filter_operations( |
116 | | &self, |
117 | | filter: OperationFilter, |
118 | | ) -> Result<ActionStateResultStream, Error>; |
119 | | } |
120 | | |
121 | | /// The type of update to perform on an operation. |
122 | | #[derive(Debug, PartialEq, Clone)] |
123 | | #[allow( |
124 | | clippy::large_enum_variant, |
125 | | reason = "TODO Fix this. Breaks on stable, but not on nightly" |
126 | | )] |
127 | | pub enum UpdateOperationType { |
128 | | /// Notification that the operation is still alive. |
129 | | KeepAlive, |
130 | | |
131 | | /// Notification that the operation has been updated. |
132 | | UpdateWithActionStage(ActionStage), |
133 | | |
134 | | /// Notification that the operation has been completed. |
135 | | UpdateWithError(Error), |
136 | | |
137 | | /// Notification that the worker disconnected. |
138 | | UpdateWithDisconnect, |
139 | | |
140 | | /// Notification that the execution stage has completed and it's just IO happening now. |
141 | | ExecutionComplete, |
142 | | |
143 | | /// The worker never ran the action: it declined the dispatch, the |
144 | | /// dispatch never reached it, or it never acknowledged it. Requeue |
145 | | /// without counting an attempt; nothing was tried. |
146 | | UpdateWithDecline(Decline), |
147 | | |
148 | | /// The worker killed the action for a resource it had declared too |
149 | | /// little of; requeue it with a larger reservation, within the retry cap. |
150 | | UpdateWithEscalation(Escalation), |
151 | | } |
152 | | |
153 | | /// Why a dispatched action is going back to the queue untried. |
154 | | #[derive(Debug, Clone, PartialEq, Eq)] |
155 | | pub struct Decline { |
156 | | pub reason: String, |
157 | | } |
158 | | |
159 | | /// A reservation to raise before an action runs again. |
160 | | #[derive(Debug, Clone, PartialEq, Eq)] |
161 | | pub struct Escalation { |
162 | | /// The platform property carrying the reservation. |
163 | | pub property: String, |
164 | | /// The new value. |
165 | | pub value: u64, |
166 | | /// Why: the error the kill produced, kept for the client if the cap |
167 | | /// ends the escalation. |
168 | | pub reason: Error, |
169 | | /// Set on the last step: the CPU property and the largest worker's |
170 | | /// whole CPU, so the action runs alone there. |
171 | | pub cpu: Option<(String, u64)>, |
172 | | } |
173 | | |
174 | | #[async_trait] |
175 | | pub trait WorkerStateManager: Sync + Send + MetricsComponent { |
176 | | /// Update that state of an operation. |
177 | | /// The worker must also send periodic updates even if the state |
178 | | /// did not change with a modified timestamp in order to prevent |
179 | | /// the operation from being considered stale and being rescheduled. |
180 | | async fn update_operation( |
181 | | &self, |
182 | | operation_id: &OperationId, |
183 | | worker_id: &WorkerId, |
184 | | update: UpdateOperationType, |
185 | | ) -> Result<(), Error>; |
186 | | |
187 | | /// Whether the operation is still executing on this worker. False once |
188 | | /// it has finished, been requeued or reassigned, or no longer exists. |
189 | | async fn is_executing_on_worker( |
190 | | &self, |
191 | | operation_id: &OperationId, |
192 | | worker_id: &WorkerId, |
193 | | ) -> Result<bool, Error>; |
194 | | } |
195 | | |
196 | | #[async_trait] |
197 | | pub trait MatchingEngineStateManager: Sync + Send + MetricsComponent { |
198 | | /// Returns a stream of operations that match the filter. |
199 | | async fn filter_operations<'a>( |
200 | | &'a self, |
201 | | filter: OperationFilter, |
202 | | ) -> Result<ActionStateResultStream<'a>, Error>; |
203 | | |
204 | | /// Assign an operation to a worker or unassign it. |
205 | | async fn assign_operation( |
206 | | &self, |
207 | | operation_id: &OperationId, |
208 | | worker_id_or_reason_for_unassign: Result<&WorkerId, Error>, |
209 | | ) -> Result<(), Error>; |
210 | | |
211 | | /// Completes a queued operation with `err`, without retrying it. Does |
212 | | /// nothing if the operation is no longer queued. Returns whether the |
213 | | /// operation was completed by this call. |
214 | | async fn fail_queued_operation( |
215 | | &self, |
216 | | operation_id: &OperationId, |
217 | | err: Error, |
218 | | ) -> Result<bool, Error>; |
219 | | } |