Coverage Report

Created: 2026-11-03 13:44

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-util/src/operation_state_manager.rs
Line
Count
Source
1
// Copyright 2024 Trace Machina, Inc. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::pin::Pin;
16
use std::sync::Arc;
17
use std::time::SystemTime;
18
19
use async_trait::async_trait;
20
use bitflags::bitflags;
21
use futures::Stream;
22
use nativelink_error::Error;
23
use nativelink_metric::MetricsComponent;
24
25
use crate::action_messages::{
26
    ActionInfo, ActionStage, ActionState, ActionUniqueKey, OperationId, WorkerId,
27
};
28
use crate::common::DigestInfo;
29
use crate::origin_event::OriginMetadata;
30
31
bitflags! {
32
    #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
33
    pub struct OperationStageFlags: u32 {
34
        const CacheCheck = 1 << 1;
35
        const Queued     = 1 << 2;
36
        const Executing  = 1 << 3;
37
        const Completed  = 1 << 4;
38
        const Any        = u32::MAX;
39
    }
40
}
41
42
impl Default for OperationStageFlags {
43
1.47k
    fn default() -> Self {
44
1.47k
        Self::Any
45
1.47k
    }
46
}
47
48
#[async_trait]
49
pub trait ActionStateResult: Send + Sync + 'static {
50
    /// Provides the current state of the action.
51
    async fn as_state(&self) -> Result<(Arc<ActionState>, Option<OriginMetadata>), Error>;
52
    /// Waits for the state of the action to change.
53
    async fn changed(&mut self) -> Result<(Arc<ActionState>, Option<OriginMetadata>), Error>;
54
    /// Provide result as action info. This behavior will not be supported by all implementations.
55
    async fn as_action_info(&self) -> Result<(Arc<ActionInfo>, Option<OriginMetadata>), Error>;
56
    /// When a client last checked in on this operation, where the
57
    /// implementation tracks it. `None` where it does not.
58
3
    async fn client_last_seen(&self) -> Result<Option<SystemTime>, Error> {
59
        Ok(None)
60
3
    }
61
}
62
63
/// The direction in which the results are ordered.
64
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
65
pub enum OrderDirection {
66
    Asc,
67
    Desc,
68
}
69
70
/// The filters used to query operations from the state manager.
71
#[derive(Default, Debug, Clone, PartialEq, Eq, Hash)]
72
pub struct OperationFilter {
73
    // TODO(palfrey): create rust builder pattern?
74
    /// The stage(s) that the operation must be in.
75
    pub stages: OperationStageFlags,
76
77
    /// The client operation id.
78
    pub client_operation_id: Option<OperationId>,
79
80
    /// The operation id.
81
    pub operation_id: Option<OperationId>,
82
83
    /// The worker that the operation must be assigned to.
84
    pub worker_id: Option<WorkerId>,
85
86
    /// The digest of the action that the operation must have.
87
    pub action_digest: Option<DigestInfo>,
88
89
    /// The operation must have its worker timestamp before this time.
90
    pub worker_update_before: Option<SystemTime>,
91
92
    /// The operation must have been completed before this time.
93
    pub completed_before: Option<SystemTime>,
94
95
    /// The unique key for filtering specific action results.
96
    pub unique_key: Option<ActionUniqueKey>,
97
98
    /// If the results should be ordered by priority and in which direction.
99
    pub order_by_priority_direction: Option<OrderDirection>,
100
}
101
102
pub type ActionStateResultStream<'a> =
103
    Pin<Box<dyn Stream<Item = Box<dyn ActionStateResult>> + Send + 'a>>;
104
105
#[async_trait]
106
pub trait ClientStateManager: Sync + Send + Unpin + MetricsComponent + 'static {
107
    /// Add a new action to the queue or joins an existing action.
108
    async fn add_action(
109
        &self,
110
        client_operation_id: OperationId,
111
        action_info: Arc<ActionInfo>,
112
    ) -> Result<Box<dyn ActionStateResult>, Error>;
113
114
    /// Returns a stream of operations that match the filter.
115
    async fn filter_operations(
116
        &self,
117
        filter: OperationFilter,
118
    ) -> Result<ActionStateResultStream, Error>;
119
}
120
121
/// The type of update to perform on an operation.
122
#[derive(Debug, PartialEq, Clone)]
123
#[allow(
124
    clippy::large_enum_variant,
125
    reason = "TODO Fix this. Breaks on stable, but not on nightly"
126
)]
127
pub enum UpdateOperationType {
128
    /// Notification that the operation is still alive.
129
    KeepAlive,
130
131
    /// Notification that the operation has been updated.
132
    UpdateWithActionStage(ActionStage),
133
134
    /// Notification that the operation has been completed.
135
    UpdateWithError(Error),
136
137
    /// Notification that the worker disconnected.
138
    UpdateWithDisconnect,
139
140
    /// Notification that the execution stage has completed and it's just IO happening now.
141
    ExecutionComplete,
142
143
    /// The worker never ran the action: it declined the dispatch, the
144
    /// dispatch never reached it, or it never acknowledged it. Requeue
145
    /// without counting an attempt; nothing was tried.
146
    UpdateWithDecline(Decline),
147
148
    /// The worker killed the action for a resource it had declared too
149
    /// little of; requeue it with a larger reservation, within the retry cap.
150
    UpdateWithEscalation(Escalation),
151
}
152
153
/// Why a dispatched action is going back to the queue untried.
154
#[derive(Debug, Clone, PartialEq, Eq)]
155
pub struct Decline {
156
    pub reason: String,
157
}
158
159
/// A reservation to raise before an action runs again.
160
#[derive(Debug, Clone, PartialEq, Eq)]
161
pub struct Escalation {
162
    /// The platform property carrying the reservation.
163
    pub property: String,
164
    /// The new value.
165
    pub value: u64,
166
    /// Why: the error the kill produced, kept for the client if the cap
167
    /// ends the escalation.
168
    pub reason: Error,
169
    /// Set on the last step: the CPU property and the largest worker's
170
    /// whole CPU, so the action runs alone there.
171
    pub cpu: Option<(String, u64)>,
172
}
173
174
#[async_trait]
175
pub trait WorkerStateManager: Sync + Send + MetricsComponent {
176
    /// Update that state of an operation.
177
    /// The worker must also send periodic updates even if the state
178
    /// did not change with a modified timestamp in order to prevent
179
    /// the operation from being considered stale and being rescheduled.
180
    async fn update_operation(
181
        &self,
182
        operation_id: &OperationId,
183
        worker_id: &WorkerId,
184
        update: UpdateOperationType,
185
    ) -> Result<(), Error>;
186
187
    /// Whether the operation is still executing on this worker. False once
188
    /// it has finished, been requeued or reassigned, or no longer exists.
189
    async fn is_executing_on_worker(
190
        &self,
191
        operation_id: &OperationId,
192
        worker_id: &WorkerId,
193
    ) -> Result<bool, Error>;
194
}
195
196
#[async_trait]
197
pub trait MatchingEngineStateManager: Sync + Send + MetricsComponent {
198
    /// Returns a stream of operations that match the filter.
199
    async fn filter_operations<'a>(
200
        &'a self,
201
        filter: OperationFilter,
202
    ) -> Result<ActionStateResultStream<'a>, Error>;
203
204
    /// Assign an operation to a worker or unassign it.
205
    async fn assign_operation(
206
        &self,
207
        operation_id: &OperationId,
208
        worker_id_or_reason_for_unassign: Result<&WorkerId, Error>,
209
    ) -> Result<(), Error>;
210
211
    /// Completes a queued operation with `err`, without retrying it. Does
212
    /// nothing if the operation is no longer queued. Returns whether the
213
    /// operation was completed by this call.
214
    async fn fail_queued_operation(
215
        &self,
216
        operation_id: &OperationId,
217
        err: Error,
218
    ) -> Result<bool, Error>;
219
}