Coverage Report

Created: 2026-11-21 11:48

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-service/src/worker_api_server.rs
Line
Count
Source
1
// Copyright 2024 Trace Machina, Inc. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::convert::Into;
16
use core::pin::Pin;
17
use core::time::Duration;
18
use std::collections::HashMap;
19
use std::sync::Arc;
20
use std::time::{SystemTime, UNIX_EPOCH};
21
22
use futures::stream::unfold;
23
use futures::{Stream, StreamExt};
24
use nativelink_config::cas_server::WorkerApiConfig;
25
use nativelink_error::{make_err, Code, Error, ResultExt};
26
use nativelink_proto::com::github::trace_machina::nativelink::remote_execution::update_for_scheduler::Update;
27
use nativelink_proto::com::github::trace_machina::nativelink::remote_execution::worker_api_server::{
28
    WorkerApi, WorkerApiServer as Server,
29
};
30
use nativelink_proto::com::github::trace_machina::nativelink::remote_execution::{
31
    execute_declined, execute_result, ExecuteAccepted, ExecuteComplete, ExecuteDeclined, ExecuteResult, GoingAwayRequest, KeepAliveRequest, UpdateForScheduler, UpdateForWorker
32
};
33
use nativelink_scheduler::worker::Worker;
34
use nativelink_scheduler::worker_scheduler::WorkerScheduler;
35
use nativelink_util::background_spawn;
36
use nativelink_util::action_messages::{OperationId, WorkerId};
37
use nativelink_util::operation_state_manager::UpdateOperationType;
38
use nativelink_util::platform_properties::PlatformProperties;
39
use rand::RngCore;
40
use tokio::sync::mpsc;
41
use tokio::time::interval;
42
use tonic::{Response, Status};
43
use tracing::{debug, error, warn, instrument, Level};
44
use uuid::Uuid;
45
46
pub type ConnectWorkerStream =
47
    Pin<Box<dyn Stream<Item = Result<UpdateForWorker, Status>> + Send + Sync + 'static>>;
48
49
pub type NowFn = Box<dyn Fn() -> Result<Duration, Error> + Send + Sync>;
50
51
/// Makes the unique part of a new worker's id.
52
pub type WorkerIdFn = Box<dyn Fn() -> String + Send + Sync>;
53
54
/// How often workers are told to kill operations the scheduler no longer
55
/// has executing on them, unless overridden by
56
/// `kill_revoked_operations_interval_s` in the worker API config.
57
const DEFAULT_KILL_REVOKED_OPERATIONS_INTERVAL_S: u64 = 5;
58
59
pub struct WorkerApiServer {
60
    scheduler: Arc<dyn WorkerScheduler>,
61
    now_fn: Arc<NowFn>,
62
    node_id: [u8; 6],
63
    worker_id_fn: Option<WorkerIdFn>,
64
}
65
66
impl core::fmt::Debug for WorkerApiServer {
67
0
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
68
0
        f.debug_struct("WorkerApiServer")
69
0
            .field("node_id", &self.node_id)
70
0
            .finish_non_exhaustive()
71
0
    }
72
}
73
74
impl WorkerApiServer {
75
0
    pub fn new(
76
0
        config: &WorkerApiConfig,
77
0
        schedulers: &HashMap<String, Arc<dyn WorkerScheduler>>,
78
0
    ) -> Result<Self, Error> {
79
0
        let node_id = {
80
0
            let mut out = [0; 6];
81
0
            rand::rng().fill_bytes(&mut out);
82
0
            out
83
        };
84
0
        let now_fn: Arc<NowFn> = Arc::new(Box::new(move || {
85
0
            SystemTime::now().duration_since(UNIX_EPOCH).map_err(|err| {
86
0
                Error::from_std_err(Code::Internal, &err)
87
0
                    .append("System time is now behind unix epoch")
88
0
            })
89
0
        }));
90
0
        Self::spawn_timers(config, schedulers, &now_fn);
91
0
        Self::with_shared_now_fn(config, schedulers, now_fn, node_id)
92
0
    }
93
94
    /// Starts the worker timeout and revoked-operation timers for every
95
    /// scheduler, reading the time from `now_fn`. `new()` calls it; a server
96
    /// built with `new_with_now_fn()` has no timers until this is called.
97
204
    pub fn spawn_timers(
98
204
        config: &WorkerApiConfig,
99
204
        schedulers: &HashMap<String, Arc<dyn WorkerScheduler>>,
100
204
        now_fn: &Arc<NowFn>,
101
204
    ) {
102
204
        let kill_revoked_enabled = !config.disable_kill_revoked_operations;
103
204
        let kill_revoked_interval_s = if config.kill_revoked_operations_interval_s == 0 {
104
204
            DEFAULT_KILL_REVOKED_OPERATIONS_INTERVAL_S
105
        } else {
106
0
            config.kill_revoked_operations_interval_s
107
        };
108
204
        for scheduler in schedulers.values() {
109
            // This will protect us from holding a reference to the scheduler forever in the
110
            // event our ExecutionServer dies. Our scheduler is a weak ref, so the spawn will
111
            // eventually see the Arc went away and return.
112
204
            let weak_scheduler = Arc::downgrade(scheduler);
113
204
            let now_fn = now_fn.clone();
114
204
            background_spawn!("worker_api_server", async move {
115
204
                let mut timeout_ticker = interval(Duration::from_secs(1));
116
204
                let mut kill_ticker = interval(Duration::from_secs(kill_revoked_interval_s));
117
                loop {
118
14.4k
                    tokio::select! {
119
14.4k
                        _ = timeout_ticker.tick() => {
120
11.8k
                            let timestamp = match now_fn() {
121
11.8k
                                Ok(timestamp) => timestamp,
122
0
                                Err(err) => {
123
0
                                    error!(?err, "Failed to read the time for remove_timedout_workers");
124
0
                                    continue;
125
                                }
126
                            };
127
11.8k
                            match weak_scheduler.upgrade() {
128
11.8k
                                Some(scheduler) => {
129
0
                                    if let Err(err) =
130
11.8k
                                        scheduler.remove_timedout_workers(timestamp.as_secs()).await
131
                                    {
132
0
                                        error!(?err, "Failed to remove_timedout_workers",);
133
11.8k
                                    }
134
                                }
135
                                // If we fail to upgrade, our service is probably destroyed, so return.
136
5
                                None => return,
137
                            }
138
                        }
139
14.4k
                        _ = kill_ticker.tick(), if kill_revoked_enabled => {
140
2.47k
                            match weak_scheduler.upgrade() {
141
2.47k
                                Some(scheduler) => {
142
2.47k
                                    if let Err(
err0
) = scheduler.kill_revoked_operations().await {
143
0
                                        error!(?err, "Failed to kill_revoked_operations");
144
2.47k
                                    }
145
                                }
146
0
                                None => return,
147
                            }
148
                        }
149
                    }
150
                }
151
5
            });
152
        }
153
204
    }
154
155
    /// Same as `new()`, but you can pass a custom `now_fn`, that returns a Duration since `UNIX_EPOCH`
156
    /// representing the current time. Used mostly in  unit tests.
157
222
    pub fn new_with_now_fn(
158
222
        config: &WorkerApiConfig,
159
222
        schedulers: &HashMap<String, Arc<dyn WorkerScheduler>>,
160
222
        now_fn: NowFn,
161
222
        node_id: [u8; 6],
162
222
    ) -> Result<Self, Error> {
163
222
        Self::with_shared_now_fn(config, schedulers, Arc::new(now_fn), node_id)
164
222
    }
165
166
222
    fn with_shared_now_fn(
167
222
        config: &WorkerApiConfig,
168
222
        schedulers: &HashMap<String, Arc<dyn WorkerScheduler>>,
169
222
        now_fn: Arc<NowFn>,
170
222
        node_id: [u8; 6],
171
222
    ) -> Result<Self, Error> {
172
222
        let scheduler = schedulers
173
222
            .get(&config.scheduler)
174
222
            .err_tip(|| 
{0
175
0
                format!(
176
                    "Scheduler needs config for '{}' because it exists in worker_api",
177
                    config.scheduler
178
                )
179
0
            })?
180
222
            .clone();
181
222
        Ok(Self {
182
222
            scheduler,
183
222
            now_fn,
184
222
            node_id,
185
222
            worker_id_fn: None,
186
222
        })
187
222
    }
188
189
    /// Takes the unique part of each worker id from `worker_id_fn` instead
190
    /// of a time-based UUID, for tests that need the same ids every run.
191
    #[must_use]
192
204
    pub fn with_worker_id_fn(mut self, worker_id_fn: WorkerIdFn) -> Self {
193
204
        self.worker_id_fn = Some(worker_id_fn);
194
204
        self
195
204
    }
196
197
0
    pub fn into_service(self) -> Server<Self> {
198
0
        Server::new(self)
199
0
    }
200
201
1.92k
    async fn inner_connect_worker(
202
1.92k
        &self,
203
1.92k
        mut update_stream: impl Stream<Item = Result<UpdateForScheduler, Status>>
204
1.92k
        + Unpin
205
1.92k
        + Send
206
1.92k
        + 'static,
207
1.92k
    ) -> Result<Response<ConnectWorkerStream>, Error> {
208
1.92k
        let first_message = update_stream
209
1.92k
            .next()
210
1.92k
            .await
211
1.92k
            .err_tip(|| "Missing first message for connect_worker")
?0
212
1.92k
            .err_tip(|| "Error reading first message for connect_worker")
?0
;
213
1.92k
        let Some(Update::ConnectWorkerRequest(connect_worker_request)) = first_message.update
214
        else {
215
0
            return Err(make_err!(
216
0
                Code::Internal,
217
0
                "First message was not a ConnectWorkerRequest"
218
0
            ));
219
        };
220
221
1.92k
        let (tx, rx) = mpsc::channel(nativelink_scheduler::worker::channel_capacity(
222
1.92k
            connect_worker_request.max_inflight_tasks,
223
1.92k
        ));
224
225
        // First convert our proto platform properties into one our scheduler understands.
226
1.92k
        let platform_properties = {
227
1.92k
            let mut platform_properties = PlatformProperties::default();
228
1.92k
            for 
property7
in connect_worker_request.properties {
229
7
                let platform_property_value = self
230
7
                    .scheduler
231
7
                    .get_platform_property_manager()
232
7
                    .make_prop_value(&property.name, &property.value)
233
7
                    .err_tip(|| "Bad Property during connect_worker()")
?0
;
234
7
                platform_properties
235
7
                    .properties
236
7
                    .insert(property.name.clone(), platform_property_value);
237
            }
238
1.92k
            platform_properties
239
        };
240
241
        // Now register the worker with the scheduler.
242
1.92k
        let worker_id = {
243
1.92k
            let unique = self.worker_id_fn.as_ref().map_or_else(
244
18
                || Uuid::now_v6(&self.node_id).hyphenated().to_string(),
245
1.90k
                |worker_id_fn| worker_id_fn(),
246
            );
247
1.92k
            let worker_id = WorkerId(format!(
248
1.92k
                "{}{unique}",
249
1.92k
                connect_worker_request.worker_id_prefix,
250
1.92k
            ));
251
1.92k
            let mut worker = Worker::new(
252
1.92k
                worker_id.clone(),
253
1.92k
                platform_properties,
254
1.92k
                tx,
255
1.92k
                (self.now_fn)()
?0
.as_secs(),
256
1.92k
                connect_worker_request.max_inflight_tasks,
257
            );
258
1.92k
            worker.admits_when_idle = connect_worker_request.admits_when_idle;
259
1.92k
            self.scheduler
260
1.92k
                .add_worker(worker)
261
1.92k
                .await
262
1.92k
                .err_tip(|| "Failed to add worker in inner_connect_worker()")
?0
;
263
1.92k
            worker_id
264
        };
265
266
1.92k
        WorkerConnection::start(
267
1.92k
            self.scheduler.clone(),
268
1.92k
            self.now_fn.clone(),
269
1.92k
            worker_id.clone(),
270
1.92k
            update_stream,
271
        );
272
273
1.92k
        Ok(Response::new(Box::pin(unfold(
274
1.92k
            (rx, worker_id),
275
15.0k
            move |state| async move {
276
15.0k
                let (mut rx, worker_id) = state;
277
15.0k
                if let Some(
update_for_worker13.1k
) = rx.recv().await {
278
13.1k
                    return Some((Ok(update_for_worker), (rx, worker_id)));
279
187
                }
280
187
                warn!(
281
                    ?worker_id,
282
                    "UpdateForWorker channel was closed, thus closing connection to worker node",
283
                );
284
285
187
                None
286
28.2k
            },
287
        ))))
288
1.92k
    }
289
290
1.92k
    pub async fn inner_connect_worker_for_testing(
291
1.92k
        &self,
292
1.92k
        update_stream: impl Stream<Item = Result<UpdateForScheduler, Status>> + Unpin + Send + 'static,
293
1.92k
    ) -> Result<Response<ConnectWorkerStream>, Error> {
294
1.92k
        self.inner_connect_worker(update_stream).await
295
1.92k
    }
296
}
297
298
#[tonic::async_trait]
299
impl WorkerApi for WorkerApiServer {
300
    type ConnectWorkerStream = ConnectWorkerStream;
301
302
    #[instrument(
303
        err,
304
        level = Level::ERROR,
305
        skip_all,
306
        fields(request = ?grpc_request.get_ref())
307
    )]
308
    async fn connect_worker(
309
        &self,
310
        grpc_request: tonic::Request<tonic::Streaming<UpdateForScheduler>>,
311
    ) -> Result<Response<Self::ConnectWorkerStream>, Status> {
312
        let resp = self
313
            .inner_connect_worker(grpc_request.into_inner())
314
            .await
315
            .map_err(Into::into);
316
        if resp.is_ok() {
317
            debug!(return = "Ok(<stream>)");
318
        }
319
        resp
320
    }
321
}
322
323
struct WorkerConnection {
324
    scheduler: Arc<dyn WorkerScheduler>,
325
    now_fn: Arc<NowFn>,
326
    worker_id: WorkerId,
327
}
328
329
impl WorkerConnection {
330
1.92k
    fn start(
331
1.92k
        scheduler: Arc<dyn WorkerScheduler>,
332
1.92k
        now_fn: Arc<NowFn>,
333
1.92k
        worker_id: WorkerId,
334
1.92k
        mut connection: impl Stream<Item = Result<UpdateForScheduler, Status>> + Unpin + Send + 'static,
335
1.92k
    ) {
336
1.92k
        let instance = Self {
337
1.92k
            scheduler,
338
1.92k
            now_fn,
339
1.92k
            worker_id,
340
1.92k
        };
341
342
1.92k
        background_spawn!("worker_api", async move 
{1.91k
343
            // The stream is read by a task of its own that only hands each
344
            // message on; they are processed here, in order. Processing one
345
            // inline in the reader (a result's store write, say) stopped the
346
            // reads for its duration, and a worker that filled the stream's
347
            // flow-control window in the meantime had nothing it was allowed
348
            // to put on the wire: its keepalives waited behind the stall
349
            // until this scheduler evicted it. The queue is unbounded on
350
            // purpose: a worker's messages are bounded by its in-flight
351
            // actions and its keepalive interval, and backpressure here is
352
            // exactly the stall.
353
1.91k
            let (tx, mut rx) = mpsc::unbounded_channel();
354
1.91k
            let reader_worker_id = instance.worker_id.clone();
355
1.91k
            let reader_scheduler = instance.scheduler.clone();
356
1.91k
            let reader_now_fn = instance.now_fn.clone();
357
1.91k
            background_spawn!("worker_api_reader", async move {
358
104k
                while let Some(
maybe_update102k
) = connection.next().await {
359
102k
                    let update = match maybe_update.map(|u| u.update) {
360
102k
                        Ok(Some(update)) => update,
361
                        Ok(None) => {
362
0
                            tracing::warn!(worker_id=?reader_worker_id, "Empty update");
363
0
                            continue;
364
                        }
365
0
                        Err(err) => {
366
0
                            tracing::warn!(worker_id=?reader_worker_id, ?err, "Error from worker");
367
0
                            break;
368
                        }
369
                    };
370
                    // The worker is heard from now, whenever this message
371
                    // gets processed: one queued behind a slow result must
372
                    // not let the worker time out in the meantime.
373
102k
                    if let Ok(now) = reader_now_fn() {
374
102k
                        reader_scheduler
375
102k
                            .worker_message_received(&reader_worker_id, now.as_secs())
376
102k
                            .await;
377
0
                    }
378
102k
                    if tx.send(update).is_err() {
379
0
                        break;
380
102k
                    }
381
                }
382
733
            });
383
1.91k
            let mut had_going_away = false;
384
100k
            while let Some(
update99.1k
) = rx.recv().await {
385
99.1k
                let 
result99.0k
= match update {
386
0
                    Update::ConnectWorkerRequest(_connect_worker_request) => Err(make_err!(
387
0
                        Code::Internal,
388
0
                        "Got ConnectWorkerRequest after initial message for {}",
389
0
                        instance.worker_id
390
0
                    )),
391
67.2k
                    Update::KeepAliveRequest(keep_alive_request) => {
392
67.2k
                        instance.inner_keep_alive(keep_alive_request).await
393
                    }
394
1
                    Update::GoingAwayRequest(going_away_request) => {
395
                        // A drain keeps the worker until its stream closes,
396
                        // so that close still has to remove it.
397
1
                        had_going_away = !going_away_request.drain;
398
1
                        instance.inner_going_away(going_away_request).await
399
                    }
400
10.5k
                    Update::ExecuteResult(execute_result) => {
401
10.5k
                        instance.inner_execution_response(execute_result).await
402
                    }
403
10.3k
                    Update::ExecuteComplete(execute_complete) => {
404
10.3k
                        instance.execution_complete(execute_complete).await
405
                    }
406
10.9k
                    Update::ExecuteAccepted(execute_accepted) => {
407
10.9k
                        instance.dispatch_accepted(execute_accepted).await
408
                    }
409
1
                    Update::ExecuteDeclined(execute_declined) => {
410
1
                        instance.dispatch_declined(execute_declined).await
411
                    }
412
                };
413
99.0k
                if let Err(
err8
) = result {
414
8
                    tracing::warn!(worker_id=?instance.worker_id, ?err, "Error processing worker message");
415
99.0k
                }
416
            }
417
733
            tracing::debug!(worker_id=?instance.worker_id, "Update for scheduler dropped");
418
733
            if !had_going_away {
419
732
                drop(
420
733
                    instance
421
733
                        .scheduler
422
733
                        .worker_disconnected(&instance.worker_id)
423
733
                        .await,
424
                );
425
0
            }
426
732
        });
427
1.92k
    }
428
429
67.2k
    async fn inner_keep_alive(&self, keep_alive_request: KeepAliveRequest) -> Result<(), Error> {
430
67.2k
        self.scheduler
431
67.2k
            .worker_keep_alive_received(
432
67.2k
                &self.worker_id,
433
67.2k
                (self.now_fn)()
?0
.as_secs(),
434
67.2k
                keep_alive_request.load,
435
            )
436
67.2k
            .await
437
67.2k
            .err_tip(|| "Could not process keep_alive from worker in inner_keep_alive()")
?0
;
438
67.2k
        if let Some(
running67.2k
) = keep_alive_request.running_operations {
439
            // The memory first: the running report can release an operation
440
            // the worker left out, and a peak for it is then dropped anyway.
441
67.2k
            let peaks: Vec<(OperationId, u64)> = running
442
67.2k
                .memory
443
67.2k
                .into_iter()
444
67.2k
                .map(|memory| 
{1
445
1
                    (
446
1
                        OperationId::from(memory.operation_id),
447
1
                        memory.peak_memory_kb,
448
1
                    )
449
1
                })
450
67.2k
                .collect();
451
67.2k
            if !peaks.is_empty() {
452
1
                self.scheduler
453
1
                    .worker_operation_memory_reported(&self.worker_id, peaks)
454
1
                    .await
455
1
                    .err_tip(|| "Could not process the memory report in inner_keep_alive()")
?0
;
456
67.2k
            }
457
67.2k
            self.scheduler
458
67.2k
                .worker_running_operations_reported(
459
67.2k
                    &self.worker_id,
460
67.2k
                    running
461
67.2k
                        .operation_ids
462
67.2k
                        .into_iter()
463
67.2k
                        .map(OperationId::from)
464
67.2k
                        .collect(),
465
67.2k
                )
466
67.2k
                .await
467
67.2k
                .err_tip(
468
                    || "Could not process the running operations report in inner_keep_alive()",
469
0
                )?;
470
11
        }
471
67.2k
        Ok(())
472
67.2k
    }
473
474
1
    async fn inner_going_away(&self, going_away_request: GoingAwayRequest) -> Result<(), Error> {
475
1
        if going_away_request.drain {
476
1
            self.scheduler
477
1
                .set_drain_worker(&self.worker_id, true)
478
1
                .await
479
1
                .err_tip(|| "While draining worker in WorkerApiServer::inner_going_away")
?0
;
480
1
            return Ok(());
481
0
        }
482
0
        self.scheduler
483
0
            .remove_worker(&self.worker_id)
484
0
            .await
485
0
            .err_tip(|| "While calling WorkerApiServer::inner_going_away")?;
486
0
        Ok(())
487
1
    }
488
489
    /// Any message from the worker proves it is alive, not only keepalives.
490
    /// Only a keepalive carries a load report and lifts a pause; this
491
    /// refreshes the timestamp alone, so a decline cannot undo the pause
492
    /// it just took.
493
    ///
494
    /// The refresh is best-effort: it is a side effect of the message,
495
    /// never the point of it. A result or completion the worker already
496
    /// produced must reach `update_action` even if the refresh fails, or
497
    /// the operation is stranded in `Executing` with nothing left to
498
    /// finish it.
499
31.8k
    async fn touch_liveness_best_effort(&self) {
500
31.8k
        let result = async {
501
31.8k
            self.scheduler
502
31.8k
                .worker_liveness_refreshed(&self.worker_id, (self.now_fn)()
?0
.as_secs())
503
31.8k
                .await
504
31.8k
        }
505
31.8k
        .await;
506
31.8k
        if let Err(
err1
) = result {
507
1
            warn!(
508
                worker_id = %self.worker_id,
509
                ?err,
510
                "Could not refresh worker liveness; continuing to process the message",
511
            );
512
31.8k
        }
513
31.8k
    }
514
515
10.5k
    async fn inner_execution_response(&self, execute_result: ExecuteResult) -> Result<(), Error> {
516
10.5k
        self.touch_liveness_best_effort().await;
517
10.5k
        let operation_id = OperationId::from(execute_result.operation_id.clone());
518
519
10.5k
        if let Some(
resource_usage0
) = execute_result.resource_usage {
520
0
            self.scheduler
521
0
                .record_action_resource_usage(&self.worker_id, &operation_id, resource_usage)
522
0
                .await
523
0
                .err_tip(|| {
524
0
                    format!("Failed to record resource usage for operation {operation_id}")
525
0
                })?;
526
10.5k
        }
527
528
10.5k
        match execute_result
529
10.5k
            .result
530
10.5k
            .err_tip(|| "Expected result to exist in ExecuteResult")
?0
531
        {
532
10.3k
            execute_result::Result::ExecuteResponse(finished_result) => {
533
10.3k
                let action_stage = finished_result
534
10.3k
                    .try_into()
535
10.3k
                    .err_tip(|| "Failed to convert ExecuteResponse into an ActionStage")
?0
;
536
10.3k
                self.scheduler
537
10.3k
                    .update_action(
538
10.3k
                        &self.worker_id,
539
10.3k
                        &operation_id,
540
10.3k
                        UpdateOperationType::UpdateWithActionStage(action_stage),
541
10.3k
                    )
542
10.3k
                    .await
543
10.3k
                    .err_tip(|| 
format!7
("Failed to operation {operation_id}"))
?7
;
544
            }
545
260
            execute_result::Result::InternalError(e) => {
546
260
                self.scheduler
547
260
                    .update_action(
548
260
                        &self.worker_id,
549
260
                        &operation_id,
550
260
                        UpdateOperationType::UpdateWithError(e.into()),
551
260
                    )
552
260
                    .await
553
260
                    .err_tip(|| 
format!1
("Failed to operation {operation_id}"))
?1
;
554
            }
555
        }
556
10.5k
        Ok(())
557
10.5k
    }
558
559
    /// The answer is recorded before the liveness refresh: the refresh runs
560
    /// the unacknowledged sweep, which would otherwise requeue an
561
    /// acknowledgement that arrives right at the timeout.
562
10.9k
    async fn dispatch_accepted(&self, execute_accepted: ExecuteAccepted) -> Result<(), Error> {
563
10.9k
        let operation_id = OperationId::from(execute_accepted.operation_id);
564
10.9k
        self.scheduler
565
10.9k
            .worker_dispatch_accepted(&self.worker_id, &operation_id)
566
10.9k
            .await
567
10.9k
            .err_tip(|| 
format!0
("Failed to record acceptance of operation {operation_id}"))
?0
;
568
10.9k
        self.touch_liveness_best_effort().await;
569
10.9k
        Ok(())
570
10.9k
    }
571
572
1
    async fn dispatch_declined(&self, execute_declined: ExecuteDeclined) -> Result<(), Error> {
573
1
        let operation_id = OperationId::from(execute_declined.operation_id);
574
1
        let reason = execute_declined::Reason::try_from(execute_declined.reason)
575
1
            .unwrap_or(execute_declined::Reason::Unspecified);
576
1
        let needs_kb = (reason == execute_declined::Reason::Load && execute_declined.needed_kb > 0)
577
1
            .then_some(execute_declined.needed_kb);
578
1
        let mut why = reason.as_str_name().to_ascii_lowercase();
579
1
        if !execute_declined.detail.is_empty() {
580
0
            why.push_str(": ");
581
0
            why.push_str(&execute_declined.detail);
582
1
        }
583
1
        self.scheduler
584
1
            .worker_dispatch_declined(&self.worker_id, &operation_id, why, needs_kb)
585
1
            .await
586
1
            .err_tip(|| 
format!0
("Failed to record decline of operation {operation_id}"))
?0
;
587
1
        self.touch_liveness_best_effort().await;
588
1
        Ok(())
589
1
    }
590
591
10.3k
    async fn execution_complete(&self, execute_complete: ExecuteComplete) -> Result<(), Error> {
592
10.3k
        self.touch_liveness_best_effort().await;
593
10.3k
        let operation_id = OperationId::from(execute_complete.operation_id);
594
10.3k
        self.scheduler
595
10.3k
            .update_action(
596
10.3k
                &self.worker_id,
597
10.3k
                &operation_id,
598
10.3k
                UpdateOperationType::ExecutionComplete,
599
10.3k
            )
600
10.3k
            .await
601
10.3k
            .err_tip(|| 
format!0
("Failed to operation {operation_id}"))
?0
;
602
10.3k
        Ok(())
603
10.3k
    }
604
}