/build/source/nativelink-service/src/worker_api_server.rs
Line | Count | Source |
1 | | // Copyright 2024 Trace Machina, Inc. All rights reserved. |
2 | | // |
3 | | // Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // See LICENSE file for details |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | use core::convert::Into; |
16 | | use core::pin::Pin; |
17 | | use core::time::Duration; |
18 | | use std::collections::HashMap; |
19 | | use std::sync::Arc; |
20 | | use std::time::{SystemTime, UNIX_EPOCH}; |
21 | | |
22 | | use futures::stream::unfold; |
23 | | use futures::{Stream, StreamExt}; |
24 | | use nativelink_config::cas_server::WorkerApiConfig; |
25 | | use nativelink_error::{make_err, Code, Error, ResultExt}; |
26 | | use nativelink_proto::com::github::trace_machina::nativelink::remote_execution::update_for_scheduler::Update; |
27 | | use nativelink_proto::com::github::trace_machina::nativelink::remote_execution::worker_api_server::{ |
28 | | WorkerApi, WorkerApiServer as Server, |
29 | | }; |
30 | | use nativelink_proto::com::github::trace_machina::nativelink::remote_execution::{ |
31 | | execute_declined, execute_result, ExecuteAccepted, ExecuteComplete, ExecuteDeclined, ExecuteResult, GoingAwayRequest, KeepAliveRequest, UpdateForScheduler, UpdateForWorker |
32 | | }; |
33 | | use nativelink_scheduler::worker::Worker; |
34 | | use nativelink_scheduler::worker_scheduler::WorkerScheduler; |
35 | | use nativelink_util::background_spawn; |
36 | | use nativelink_util::action_messages::{OperationId, WorkerId}; |
37 | | use nativelink_util::operation_state_manager::UpdateOperationType; |
38 | | use nativelink_util::platform_properties::PlatformProperties; |
39 | | use rand::RngCore; |
40 | | use tokio::sync::mpsc; |
41 | | use tokio::time::interval; |
42 | | use tonic::{Response, Status}; |
43 | | use tracing::{debug, error, warn, instrument, Level}; |
44 | | use uuid::Uuid; |
45 | | |
46 | | pub type ConnectWorkerStream = |
47 | | Pin<Box<dyn Stream<Item = Result<UpdateForWorker, Status>> + Send + Sync + 'static>>; |
48 | | |
49 | | pub type NowFn = Box<dyn Fn() -> Result<Duration, Error> + Send + Sync>; |
50 | | |
51 | | /// Makes the unique part of a new worker's id. |
52 | | pub type WorkerIdFn = Box<dyn Fn() -> String + Send + Sync>; |
53 | | |
54 | | /// How often workers are told to kill operations the scheduler no longer |
55 | | /// has executing on them, unless overridden by |
56 | | /// `kill_revoked_operations_interval_s` in the worker API config. |
57 | | const DEFAULT_KILL_REVOKED_OPERATIONS_INTERVAL_S: u64 = 5; |
58 | | |
59 | | pub struct WorkerApiServer { |
60 | | scheduler: Arc<dyn WorkerScheduler>, |
61 | | now_fn: Arc<NowFn>, |
62 | | node_id: [u8; 6], |
63 | | worker_id_fn: Option<WorkerIdFn>, |
64 | | } |
65 | | |
66 | | impl core::fmt::Debug for WorkerApiServer { |
67 | 0 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { |
68 | 0 | f.debug_struct("WorkerApiServer") |
69 | 0 | .field("node_id", &self.node_id) |
70 | 0 | .finish_non_exhaustive() |
71 | 0 | } |
72 | | } |
73 | | |
74 | | impl WorkerApiServer { |
75 | 0 | pub fn new( |
76 | 0 | config: &WorkerApiConfig, |
77 | 0 | schedulers: &HashMap<String, Arc<dyn WorkerScheduler>>, |
78 | 0 | ) -> Result<Self, Error> { |
79 | 0 | let node_id = { |
80 | 0 | let mut out = [0; 6]; |
81 | 0 | rand::rng().fill_bytes(&mut out); |
82 | 0 | out |
83 | | }; |
84 | 0 | let now_fn: Arc<NowFn> = Arc::new(Box::new(move || { |
85 | 0 | SystemTime::now().duration_since(UNIX_EPOCH).map_err(|err| { |
86 | 0 | Error::from_std_err(Code::Internal, &err) |
87 | 0 | .append("System time is now behind unix epoch") |
88 | 0 | }) |
89 | 0 | })); |
90 | 0 | Self::spawn_timers(config, schedulers, &now_fn); |
91 | 0 | Self::with_shared_now_fn(config, schedulers, now_fn, node_id) |
92 | 0 | } |
93 | | |
94 | | /// Starts the worker timeout and revoked-operation timers for every |
95 | | /// scheduler, reading the time from `now_fn`. `new()` calls it; a server |
96 | | /// built with `new_with_now_fn()` has no timers until this is called. |
97 | 204 | pub fn spawn_timers( |
98 | 204 | config: &WorkerApiConfig, |
99 | 204 | schedulers: &HashMap<String, Arc<dyn WorkerScheduler>>, |
100 | 204 | now_fn: &Arc<NowFn>, |
101 | 204 | ) { |
102 | 204 | let kill_revoked_enabled = !config.disable_kill_revoked_operations; |
103 | 204 | let kill_revoked_interval_s = if config.kill_revoked_operations_interval_s == 0 { |
104 | 204 | DEFAULT_KILL_REVOKED_OPERATIONS_INTERVAL_S |
105 | | } else { |
106 | 0 | config.kill_revoked_operations_interval_s |
107 | | }; |
108 | 204 | for scheduler in schedulers.values() { |
109 | | // This will protect us from holding a reference to the scheduler forever in the |
110 | | // event our ExecutionServer dies. Our scheduler is a weak ref, so the spawn will |
111 | | // eventually see the Arc went away and return. |
112 | 204 | let weak_scheduler = Arc::downgrade(scheduler); |
113 | 204 | let now_fn = now_fn.clone(); |
114 | 204 | background_spawn!("worker_api_server", async move { |
115 | 204 | let mut timeout_ticker = interval(Duration::from_secs(1)); |
116 | 204 | let mut kill_ticker = interval(Duration::from_secs(kill_revoked_interval_s)); |
117 | | loop { |
118 | 14.4k | tokio::select! { |
119 | 14.4k | _ = timeout_ticker.tick() => { |
120 | 11.8k | let timestamp = match now_fn() { |
121 | 11.8k | Ok(timestamp) => timestamp, |
122 | 0 | Err(err) => { |
123 | 0 | error!(?err, "Failed to read the time for remove_timedout_workers"); |
124 | 0 | continue; |
125 | | } |
126 | | }; |
127 | 11.8k | match weak_scheduler.upgrade() { |
128 | 11.8k | Some(scheduler) => { |
129 | 0 | if let Err(err) = |
130 | 11.8k | scheduler.remove_timedout_workers(timestamp.as_secs()).await |
131 | | { |
132 | 0 | error!(?err, "Failed to remove_timedout_workers",); |
133 | 11.8k | } |
134 | | } |
135 | | // If we fail to upgrade, our service is probably destroyed, so return. |
136 | 5 | None => return, |
137 | | } |
138 | | } |
139 | 14.4k | _ = kill_ticker.tick(), if kill_revoked_enabled => { |
140 | 2.47k | match weak_scheduler.upgrade() { |
141 | 2.47k | Some(scheduler) => { |
142 | 2.47k | if let Err(err0 ) = scheduler.kill_revoked_operations().await { |
143 | 0 | error!(?err, "Failed to kill_revoked_operations"); |
144 | 2.47k | } |
145 | | } |
146 | 0 | None => return, |
147 | | } |
148 | | } |
149 | | } |
150 | | } |
151 | 5 | }); |
152 | | } |
153 | 204 | } |
154 | | |
155 | | /// Same as `new()`, but you can pass a custom `now_fn`, that returns a Duration since `UNIX_EPOCH` |
156 | | /// representing the current time. Used mostly in unit tests. |
157 | 222 | pub fn new_with_now_fn( |
158 | 222 | config: &WorkerApiConfig, |
159 | 222 | schedulers: &HashMap<String, Arc<dyn WorkerScheduler>>, |
160 | 222 | now_fn: NowFn, |
161 | 222 | node_id: [u8; 6], |
162 | 222 | ) -> Result<Self, Error> { |
163 | 222 | Self::with_shared_now_fn(config, schedulers, Arc::new(now_fn), node_id) |
164 | 222 | } |
165 | | |
166 | 222 | fn with_shared_now_fn( |
167 | 222 | config: &WorkerApiConfig, |
168 | 222 | schedulers: &HashMap<String, Arc<dyn WorkerScheduler>>, |
169 | 222 | now_fn: Arc<NowFn>, |
170 | 222 | node_id: [u8; 6], |
171 | 222 | ) -> Result<Self, Error> { |
172 | 222 | let scheduler = schedulers |
173 | 222 | .get(&config.scheduler) |
174 | 222 | .err_tip(|| {0 |
175 | 0 | format!( |
176 | | "Scheduler needs config for '{}' because it exists in worker_api", |
177 | | config.scheduler |
178 | | ) |
179 | 0 | })? |
180 | 222 | .clone(); |
181 | 222 | Ok(Self { |
182 | 222 | scheduler, |
183 | 222 | now_fn, |
184 | 222 | node_id, |
185 | 222 | worker_id_fn: None, |
186 | 222 | }) |
187 | 222 | } |
188 | | |
189 | | /// Takes the unique part of each worker id from `worker_id_fn` instead |
190 | | /// of a time-based UUID, for tests that need the same ids every run. |
191 | | #[must_use] |
192 | 204 | pub fn with_worker_id_fn(mut self, worker_id_fn: WorkerIdFn) -> Self { |
193 | 204 | self.worker_id_fn = Some(worker_id_fn); |
194 | 204 | self |
195 | 204 | } |
196 | | |
197 | 0 | pub fn into_service(self) -> Server<Self> { |
198 | 0 | Server::new(self) |
199 | 0 | } |
200 | | |
201 | 1.92k | async fn inner_connect_worker( |
202 | 1.92k | &self, |
203 | 1.92k | mut update_stream: impl Stream<Item = Result<UpdateForScheduler, Status>> |
204 | 1.92k | + Unpin |
205 | 1.92k | + Send |
206 | 1.92k | + 'static, |
207 | 1.92k | ) -> Result<Response<ConnectWorkerStream>, Error> { |
208 | 1.92k | let first_message = update_stream |
209 | 1.92k | .next() |
210 | 1.92k | .await |
211 | 1.92k | .err_tip(|| "Missing first message for connect_worker")?0 |
212 | 1.92k | .err_tip(|| "Error reading first message for connect_worker")?0 ; |
213 | 1.92k | let Some(Update::ConnectWorkerRequest(connect_worker_request)) = first_message.update |
214 | | else { |
215 | 0 | return Err(make_err!( |
216 | 0 | Code::Internal, |
217 | 0 | "First message was not a ConnectWorkerRequest" |
218 | 0 | )); |
219 | | }; |
220 | | |
221 | 1.92k | let (tx, rx) = mpsc::channel(nativelink_scheduler::worker::channel_capacity( |
222 | 1.92k | connect_worker_request.max_inflight_tasks, |
223 | 1.92k | )); |
224 | | |
225 | | // First convert our proto platform properties into one our scheduler understands. |
226 | 1.92k | let platform_properties = { |
227 | 1.92k | let mut platform_properties = PlatformProperties::default(); |
228 | 1.92k | for property7 in connect_worker_request.properties { |
229 | 7 | let platform_property_value = self |
230 | 7 | .scheduler |
231 | 7 | .get_platform_property_manager() |
232 | 7 | .make_prop_value(&property.name, &property.value) |
233 | 7 | .err_tip(|| "Bad Property during connect_worker()")?0 ; |
234 | 7 | platform_properties |
235 | 7 | .properties |
236 | 7 | .insert(property.name.clone(), platform_property_value); |
237 | | } |
238 | 1.92k | platform_properties |
239 | | }; |
240 | | |
241 | | // Now register the worker with the scheduler. |
242 | 1.92k | let worker_id = { |
243 | 1.92k | let unique = self.worker_id_fn.as_ref().map_or_else( |
244 | 18 | || Uuid::now_v6(&self.node_id).hyphenated().to_string(), |
245 | 1.90k | |worker_id_fn| worker_id_fn(), |
246 | | ); |
247 | 1.92k | let worker_id = WorkerId(format!( |
248 | 1.92k | "{}{unique}", |
249 | 1.92k | connect_worker_request.worker_id_prefix, |
250 | 1.92k | )); |
251 | 1.92k | let mut worker = Worker::new( |
252 | 1.92k | worker_id.clone(), |
253 | 1.92k | platform_properties, |
254 | 1.92k | tx, |
255 | 1.92k | (self.now_fn)()?0 .as_secs(), |
256 | 1.92k | connect_worker_request.max_inflight_tasks, |
257 | | ); |
258 | 1.92k | worker.admits_when_idle = connect_worker_request.admits_when_idle; |
259 | 1.92k | self.scheduler |
260 | 1.92k | .add_worker(worker) |
261 | 1.92k | .await |
262 | 1.92k | .err_tip(|| "Failed to add worker in inner_connect_worker()")?0 ; |
263 | 1.92k | worker_id |
264 | | }; |
265 | | |
266 | 1.92k | WorkerConnection::start( |
267 | 1.92k | self.scheduler.clone(), |
268 | 1.92k | self.now_fn.clone(), |
269 | 1.92k | worker_id.clone(), |
270 | 1.92k | update_stream, |
271 | | ); |
272 | | |
273 | 1.92k | Ok(Response::new(Box::pin(unfold( |
274 | 1.92k | (rx, worker_id), |
275 | 15.0k | move |state| async move { |
276 | 15.0k | let (mut rx, worker_id) = state; |
277 | 15.0k | if let Some(update_for_worker13.1k ) = rx.recv().await { |
278 | 13.1k | return Some((Ok(update_for_worker), (rx, worker_id))); |
279 | 187 | } |
280 | 187 | warn!( |
281 | | ?worker_id, |
282 | | "UpdateForWorker channel was closed, thus closing connection to worker node", |
283 | | ); |
284 | | |
285 | 187 | None |
286 | 28.2k | }, |
287 | | )))) |
288 | 1.92k | } |
289 | | |
290 | 1.92k | pub async fn inner_connect_worker_for_testing( |
291 | 1.92k | &self, |
292 | 1.92k | update_stream: impl Stream<Item = Result<UpdateForScheduler, Status>> + Unpin + Send + 'static, |
293 | 1.92k | ) -> Result<Response<ConnectWorkerStream>, Error> { |
294 | 1.92k | self.inner_connect_worker(update_stream).await |
295 | 1.92k | } |
296 | | } |
297 | | |
298 | | #[tonic::async_trait] |
299 | | impl WorkerApi for WorkerApiServer { |
300 | | type ConnectWorkerStream = ConnectWorkerStream; |
301 | | |
302 | | #[instrument( |
303 | | err, |
304 | | level = Level::ERROR, |
305 | | skip_all, |
306 | | fields(request = ?grpc_request.get_ref()) |
307 | | )] |
308 | | async fn connect_worker( |
309 | | &self, |
310 | | grpc_request: tonic::Request<tonic::Streaming<UpdateForScheduler>>, |
311 | | ) -> Result<Response<Self::ConnectWorkerStream>, Status> { |
312 | | let resp = self |
313 | | .inner_connect_worker(grpc_request.into_inner()) |
314 | | .await |
315 | | .map_err(Into::into); |
316 | | if resp.is_ok() { |
317 | | debug!(return = "Ok(<stream>)"); |
318 | | } |
319 | | resp |
320 | | } |
321 | | } |
322 | | |
323 | | struct WorkerConnection { |
324 | | scheduler: Arc<dyn WorkerScheduler>, |
325 | | now_fn: Arc<NowFn>, |
326 | | worker_id: WorkerId, |
327 | | } |
328 | | |
329 | | impl WorkerConnection { |
330 | 1.92k | fn start( |
331 | 1.92k | scheduler: Arc<dyn WorkerScheduler>, |
332 | 1.92k | now_fn: Arc<NowFn>, |
333 | 1.92k | worker_id: WorkerId, |
334 | 1.92k | mut connection: impl Stream<Item = Result<UpdateForScheduler, Status>> + Unpin + Send + 'static, |
335 | 1.92k | ) { |
336 | 1.92k | let instance = Self { |
337 | 1.92k | scheduler, |
338 | 1.92k | now_fn, |
339 | 1.92k | worker_id, |
340 | 1.92k | }; |
341 | | |
342 | 1.92k | background_spawn!("worker_api", async move {1.91k |
343 | | // The stream is read by a task of its own that only hands each |
344 | | // message on; they are processed here, in order. Processing one |
345 | | // inline in the reader (a result's store write, say) stopped the |
346 | | // reads for its duration, and a worker that filled the stream's |
347 | | // flow-control window in the meantime had nothing it was allowed |
348 | | // to put on the wire: its keepalives waited behind the stall |
349 | | // until this scheduler evicted it. The queue is unbounded on |
350 | | // purpose: a worker's messages are bounded by its in-flight |
351 | | // actions and its keepalive interval, and backpressure here is |
352 | | // exactly the stall. |
353 | 1.91k | let (tx, mut rx) = mpsc::unbounded_channel(); |
354 | 1.91k | let reader_worker_id = instance.worker_id.clone(); |
355 | 1.91k | let reader_scheduler = instance.scheduler.clone(); |
356 | 1.91k | let reader_now_fn = instance.now_fn.clone(); |
357 | 1.91k | background_spawn!("worker_api_reader", async move { |
358 | 104k | while let Some(maybe_update102k ) = connection.next().await { |
359 | 102k | let update = match maybe_update.map(|u| u.update) { |
360 | 102k | Ok(Some(update)) => update, |
361 | | Ok(None) => { |
362 | 0 | tracing::warn!(worker_id=?reader_worker_id, "Empty update"); |
363 | 0 | continue; |
364 | | } |
365 | 0 | Err(err) => { |
366 | 0 | tracing::warn!(worker_id=?reader_worker_id, ?err, "Error from worker"); |
367 | 0 | break; |
368 | | } |
369 | | }; |
370 | | // The worker is heard from now, whenever this message |
371 | | // gets processed: one queued behind a slow result must |
372 | | // not let the worker time out in the meantime. |
373 | 102k | if let Ok(now) = reader_now_fn() { |
374 | 102k | reader_scheduler |
375 | 102k | .worker_message_received(&reader_worker_id, now.as_secs()) |
376 | 102k | .await; |
377 | 0 | } |
378 | 102k | if tx.send(update).is_err() { |
379 | 0 | break; |
380 | 102k | } |
381 | | } |
382 | 733 | }); |
383 | 1.91k | let mut had_going_away = false; |
384 | 100k | while let Some(update99.1k ) = rx.recv().await { |
385 | 99.1k | let result99.0k = match update { |
386 | 0 | Update::ConnectWorkerRequest(_connect_worker_request) => Err(make_err!( |
387 | 0 | Code::Internal, |
388 | 0 | "Got ConnectWorkerRequest after initial message for {}", |
389 | 0 | instance.worker_id |
390 | 0 | )), |
391 | 67.2k | Update::KeepAliveRequest(keep_alive_request) => { |
392 | 67.2k | instance.inner_keep_alive(keep_alive_request).await |
393 | | } |
394 | 1 | Update::GoingAwayRequest(going_away_request) => { |
395 | | // A drain keeps the worker until its stream closes, |
396 | | // so that close still has to remove it. |
397 | 1 | had_going_away = !going_away_request.drain; |
398 | 1 | instance.inner_going_away(going_away_request).await |
399 | | } |
400 | 10.5k | Update::ExecuteResult(execute_result) => { |
401 | 10.5k | instance.inner_execution_response(execute_result).await |
402 | | } |
403 | 10.3k | Update::ExecuteComplete(execute_complete) => { |
404 | 10.3k | instance.execution_complete(execute_complete).await |
405 | | } |
406 | 10.9k | Update::ExecuteAccepted(execute_accepted) => { |
407 | 10.9k | instance.dispatch_accepted(execute_accepted).await |
408 | | } |
409 | 1 | Update::ExecuteDeclined(execute_declined) => { |
410 | 1 | instance.dispatch_declined(execute_declined).await |
411 | | } |
412 | | }; |
413 | 99.0k | if let Err(err8 ) = result { |
414 | 8 | tracing::warn!(worker_id=?instance.worker_id, ?err, "Error processing worker message"); |
415 | 99.0k | } |
416 | | } |
417 | 733 | tracing::debug!(worker_id=?instance.worker_id, "Update for scheduler dropped"); |
418 | 733 | if !had_going_away { |
419 | 732 | drop( |
420 | 733 | instance |
421 | 733 | .scheduler |
422 | 733 | .worker_disconnected(&instance.worker_id) |
423 | 733 | .await, |
424 | | ); |
425 | 0 | } |
426 | 732 | }); |
427 | 1.92k | } |
428 | | |
429 | 67.2k | async fn inner_keep_alive(&self, keep_alive_request: KeepAliveRequest) -> Result<(), Error> { |
430 | 67.2k | self.scheduler |
431 | 67.2k | .worker_keep_alive_received( |
432 | 67.2k | &self.worker_id, |
433 | 67.2k | (self.now_fn)()?0 .as_secs(), |
434 | 67.2k | keep_alive_request.load, |
435 | | ) |
436 | 67.2k | .await |
437 | 67.2k | .err_tip(|| "Could not process keep_alive from worker in inner_keep_alive()")?0 ; |
438 | 67.2k | if let Some(running67.2k ) = keep_alive_request.running_operations { |
439 | | // The memory first: the running report can release an operation |
440 | | // the worker left out, and a peak for it is then dropped anyway. |
441 | 67.2k | let peaks: Vec<(OperationId, u64)> = running |
442 | 67.2k | .memory |
443 | 67.2k | .into_iter() |
444 | 67.2k | .map(|memory| {1 |
445 | 1 | ( |
446 | 1 | OperationId::from(memory.operation_id), |
447 | 1 | memory.peak_memory_kb, |
448 | 1 | ) |
449 | 1 | }) |
450 | 67.2k | .collect(); |
451 | 67.2k | if !peaks.is_empty() { |
452 | 1 | self.scheduler |
453 | 1 | .worker_operation_memory_reported(&self.worker_id, peaks) |
454 | 1 | .await |
455 | 1 | .err_tip(|| "Could not process the memory report in inner_keep_alive()")?0 ; |
456 | 67.2k | } |
457 | 67.2k | self.scheduler |
458 | 67.2k | .worker_running_operations_reported( |
459 | 67.2k | &self.worker_id, |
460 | 67.2k | running |
461 | 67.2k | .operation_ids |
462 | 67.2k | .into_iter() |
463 | 67.2k | .map(OperationId::from) |
464 | 67.2k | .collect(), |
465 | 67.2k | ) |
466 | 67.2k | .await |
467 | 67.2k | .err_tip( |
468 | | || "Could not process the running operations report in inner_keep_alive()", |
469 | 0 | )?; |
470 | 11 | } |
471 | 67.2k | Ok(()) |
472 | 67.2k | } |
473 | | |
474 | 1 | async fn inner_going_away(&self, going_away_request: GoingAwayRequest) -> Result<(), Error> { |
475 | 1 | if going_away_request.drain { |
476 | 1 | self.scheduler |
477 | 1 | .set_drain_worker(&self.worker_id, true) |
478 | 1 | .await |
479 | 1 | .err_tip(|| "While draining worker in WorkerApiServer::inner_going_away")?0 ; |
480 | 1 | return Ok(()); |
481 | 0 | } |
482 | 0 | self.scheduler |
483 | 0 | .remove_worker(&self.worker_id) |
484 | 0 | .await |
485 | 0 | .err_tip(|| "While calling WorkerApiServer::inner_going_away")?; |
486 | 0 | Ok(()) |
487 | 1 | } |
488 | | |
489 | | /// Any message from the worker proves it is alive, not only keepalives. |
490 | | /// Only a keepalive carries a load report and lifts a pause; this |
491 | | /// refreshes the timestamp alone, so a decline cannot undo the pause |
492 | | /// it just took. |
493 | | /// |
494 | | /// The refresh is best-effort: it is a side effect of the message, |
495 | | /// never the point of it. A result or completion the worker already |
496 | | /// produced must reach `update_action` even if the refresh fails, or |
497 | | /// the operation is stranded in `Executing` with nothing left to |
498 | | /// finish it. |
499 | 31.8k | async fn touch_liveness_best_effort(&self) { |
500 | 31.8k | let result = async { |
501 | 31.8k | self.scheduler |
502 | 31.8k | .worker_liveness_refreshed(&self.worker_id, (self.now_fn)()?0 .as_secs()) |
503 | 31.8k | .await |
504 | 31.8k | } |
505 | 31.8k | .await; |
506 | 31.8k | if let Err(err1 ) = result { |
507 | 1 | warn!( |
508 | | worker_id = %self.worker_id, |
509 | | ?err, |
510 | | "Could not refresh worker liveness; continuing to process the message", |
511 | | ); |
512 | 31.8k | } |
513 | 31.8k | } |
514 | | |
515 | 10.5k | async fn inner_execution_response(&self, execute_result: ExecuteResult) -> Result<(), Error> { |
516 | 10.5k | self.touch_liveness_best_effort().await; |
517 | 10.5k | let operation_id = OperationId::from(execute_result.operation_id.clone()); |
518 | | |
519 | 10.5k | if let Some(resource_usage0 ) = execute_result.resource_usage { |
520 | 0 | self.scheduler |
521 | 0 | .record_action_resource_usage(&self.worker_id, &operation_id, resource_usage) |
522 | 0 | .await |
523 | 0 | .err_tip(|| { |
524 | 0 | format!("Failed to record resource usage for operation {operation_id}") |
525 | 0 | })?; |
526 | 10.5k | } |
527 | | |
528 | 10.5k | match execute_result |
529 | 10.5k | .result |
530 | 10.5k | .err_tip(|| "Expected result to exist in ExecuteResult")?0 |
531 | | { |
532 | 10.3k | execute_result::Result::ExecuteResponse(finished_result) => { |
533 | 10.3k | let action_stage = finished_result |
534 | 10.3k | .try_into() |
535 | 10.3k | .err_tip(|| "Failed to convert ExecuteResponse into an ActionStage")?0 ; |
536 | 10.3k | self.scheduler |
537 | 10.3k | .update_action( |
538 | 10.3k | &self.worker_id, |
539 | 10.3k | &operation_id, |
540 | 10.3k | UpdateOperationType::UpdateWithActionStage(action_stage), |
541 | 10.3k | ) |
542 | 10.3k | .await |
543 | 10.3k | .err_tip(|| format!7 ("Failed to operation {operation_id}"))?7 ; |
544 | | } |
545 | 260 | execute_result::Result::InternalError(e) => { |
546 | 260 | self.scheduler |
547 | 260 | .update_action( |
548 | 260 | &self.worker_id, |
549 | 260 | &operation_id, |
550 | 260 | UpdateOperationType::UpdateWithError(e.into()), |
551 | 260 | ) |
552 | 260 | .await |
553 | 260 | .err_tip(|| format!1 ("Failed to operation {operation_id}"))?1 ; |
554 | | } |
555 | | } |
556 | 10.5k | Ok(()) |
557 | 10.5k | } |
558 | | |
559 | | /// The answer is recorded before the liveness refresh: the refresh runs |
560 | | /// the unacknowledged sweep, which would otherwise requeue an |
561 | | /// acknowledgement that arrives right at the timeout. |
562 | 10.9k | async fn dispatch_accepted(&self, execute_accepted: ExecuteAccepted) -> Result<(), Error> { |
563 | 10.9k | let operation_id = OperationId::from(execute_accepted.operation_id); |
564 | 10.9k | self.scheduler |
565 | 10.9k | .worker_dispatch_accepted(&self.worker_id, &operation_id) |
566 | 10.9k | .await |
567 | 10.9k | .err_tip(|| format!0 ("Failed to record acceptance of operation {operation_id}"))?0 ; |
568 | 10.9k | self.touch_liveness_best_effort().await; |
569 | 10.9k | Ok(()) |
570 | 10.9k | } |
571 | | |
572 | 1 | async fn dispatch_declined(&self, execute_declined: ExecuteDeclined) -> Result<(), Error> { |
573 | 1 | let operation_id = OperationId::from(execute_declined.operation_id); |
574 | 1 | let reason = execute_declined::Reason::try_from(execute_declined.reason) |
575 | 1 | .unwrap_or(execute_declined::Reason::Unspecified); |
576 | 1 | let needs_kb = (reason == execute_declined::Reason::Load && execute_declined.needed_kb > 0) |
577 | 1 | .then_some(execute_declined.needed_kb); |
578 | 1 | let mut why = reason.as_str_name().to_ascii_lowercase(); |
579 | 1 | if !execute_declined.detail.is_empty() { |
580 | 0 | why.push_str(": "); |
581 | 0 | why.push_str(&execute_declined.detail); |
582 | 1 | } |
583 | 1 | self.scheduler |
584 | 1 | .worker_dispatch_declined(&self.worker_id, &operation_id, why, needs_kb) |
585 | 1 | .await |
586 | 1 | .err_tip(|| format!0 ("Failed to record decline of operation {operation_id}"))?0 ; |
587 | 1 | self.touch_liveness_best_effort().await; |
588 | 1 | Ok(()) |
589 | 1 | } |
590 | | |
591 | 10.3k | async fn execution_complete(&self, execute_complete: ExecuteComplete) -> Result<(), Error> { |
592 | 10.3k | self.touch_liveness_best_effort().await; |
593 | 10.3k | let operation_id = OperationId::from(execute_complete.operation_id); |
594 | 10.3k | self.scheduler |
595 | 10.3k | .update_action( |
596 | 10.3k | &self.worker_id, |
597 | 10.3k | &operation_id, |
598 | 10.3k | UpdateOperationType::ExecutionComplete, |
599 | 10.3k | ) |
600 | 10.3k | .await |
601 | 10.3k | .err_tip(|| format!0 ("Failed to operation {operation_id}"))?0 ; |
602 | 10.3k | Ok(()) |
603 | 10.3k | } |
604 | | } |