Coverage Report

Created: 2026-07-21 15:28

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-scheduler/src/historical_resource_scheduler.rs
Line
Count
Source
1
// Copyright 2026 The NativeLink Authors. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::time::Duration;
16
use std::collections::{HashMap, HashSet};
17
use std::fs;
18
use std::sync::Arc;
19
use std::time::Instant;
20
21
use async_trait::async_trait;
22
use nativelink_config::schedulers::HistoricalResourceSpec;
23
use nativelink_error::{Error, ResultExt};
24
use nativelink_metric::{
25
    MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent, RootMetricsComponent,
26
};
27
use nativelink_util::action_messages::{ActionInfo, OperationId};
28
use nativelink_util::operation_state_manager::{
29
    ActionStateResult, ActionStateResultStream, ClientStateManager, OperationFilter,
30
};
31
use nativelink_util::origin_event::{BAZEL_METADATA_KEY, request_metadata_from_baggage};
32
use opentelemetry::baggage::BaggageExt;
33
use opentelemetry::context::Context;
34
use parking_lot::Mutex;
35
use serde::Deserialize;
36
use tracing::{debug, warn};
37
38
use crate::known_platform_property_provider::KnownPlatformPropertyProvider;
39
40
#[derive(Debug, Clone, Hash, Eq, PartialEq)]
41
struct HintKey {
42
    target_id: Option<String>,
43
    action_mnemonic: Option<String>,
44
}
45
46
#[derive(Debug, Clone, Deserialize)]
47
struct HistoricalResourceHint {
48
    #[serde(default)]
49
    target_id: Option<String>,
50
    #[serde(default)]
51
    action_mnemonic: Option<String>,
52
    #[serde(default)]
53
    cpu_count: Option<u64>,
54
    #[serde(default)]
55
    memory_kb: Option<u64>,
56
    #[serde(default)]
57
    memory_mib: Option<u64>,
58
}
59
60
impl HistoricalResourceHint {
61
1
    fn key(&self) -> Option<HintKey> {
62
1
        if self.target_id.is_none() && 
self.action_mnemonic0
.
is_none0
() {
63
0
            return None;
64
1
        }
65
1
        Some(HintKey {
66
1
            target_id: self.target_id.clone(),
67
1
            action_mnemonic: self.action_mnemonic.clone(),
68
1
        })
69
1
    }
70
71
1
    fn memory_kb(&self) -> Option<u64> {
72
1
        self.memory_kb.or_else(|| 
{0
73
0
            self.memory_mib
74
0
                .map(|memory_mib| memory_mib.saturating_mul(1024))
75
0
        })
76
1
    }
77
}
78
79
#[derive(Debug, Deserialize)]
80
#[serde(untagged)]
81
enum HistoricalResourceHintFile {
82
    List(Vec<HistoricalResourceHint>),
83
    Object { hints: Vec<HistoricalResourceHint> },
84
}
85
86
impl HistoricalResourceHintFile {
87
1
    fn into_hints(self) -> Vec<HistoricalResourceHint> {
88
1
        match self {
89
1
            Self::List(
hints0
) | Self::Object { hints } => hints,
90
        }
91
1
    }
92
}
93
94
#[derive(Debug, Default)]
95
struct HintState {
96
    hints: HashMap<HintKey, HistoricalResourceHint>,
97
    last_loaded: Option<Instant>,
98
}
99
100
pub struct HistoricalResourceScheduler {
101
    hints_file: String,
102
    refresh_interval: Duration,
103
    cpu_property_name: String,
104
    memory_property_name: String,
105
    scheduler: Option<Arc<dyn KnownPlatformPropertyProvider>>,
106
    known_properties: Mutex<HashMap<String, Vec<String>>>,
107
    hint_state: Mutex<HintState>,
108
}
109
110
impl core::fmt::Debug for HistoricalResourceScheduler {
111
0
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
112
0
        f.debug_struct("HistoricalResourceScheduler")
113
0
            .field("hints_file", &self.hints_file)
114
0
            .field("refresh_interval", &self.refresh_interval)
115
0
            .field("cpu_property_name", &self.cpu_property_name)
116
0
            .field("memory_property_name", &self.memory_property_name)
117
0
            .field("known_properties", &self.known_properties)
118
0
            .finish_non_exhaustive()
119
0
    }
120
}
121
122
impl HistoricalResourceScheduler {
123
    #[must_use]
124
1
    pub fn new(
125
1
        spec: &HistoricalResourceSpec,
126
1
        scheduler: Arc<dyn KnownPlatformPropertyProvider>,
127
1
    ) -> Self {
128
1
        Self {
129
1
            hints_file: spec.hints_file.clone(),
130
1
            refresh_interval: Duration::from_secs(spec.refresh_interval_s),
131
1
            cpu_property_name: spec.cpu_property_name.clone(),
132
1
            memory_property_name: spec.memory_property_name.clone(),
133
1
            scheduler: Some(scheduler),
134
1
            known_properties: Mutex::new(HashMap::new()),
135
1
            hint_state: Mutex::default(),
136
1
        }
137
1
    }
138
139
1
    fn refresh_hints(&self) {
140
1
        let now = Instant::now();
141
        {
142
1
            let hint_state = self.hint_state.lock();
143
1
            if let Some(
last_loaded0
) = hint_state.last_loaded
144
0
                && (self.refresh_interval.is_zero()
145
0
                    || now.duration_since(last_loaded) < self.refresh_interval)
146
            {
147
0
                return;
148
1
            }
149
        }
150
151
1
        let hints_file_contents = match fs::read_to_string(&self.hints_file) {
152
1
            Ok(contents) => contents,
153
0
            Err(err) => {
154
0
                warn!(?err, hints_file = %self.hints_file, "Failed to read historical resource hints");
155
0
                self.hint_state.lock().last_loaded = Some(now);
156
0
                return;
157
            }
158
        };
159
1
        let hints = match serde_json::from_str::<HistoricalResourceHintFile>(&hints_file_contents) {
160
1
            Ok(hints_file) => hints_file.into_hints(),
161
0
            Err(err) => {
162
0
                warn!(?err, hints_file = %self.hints_file, "Failed to parse historical resource hints");
163
0
                self.hint_state.lock().last_loaded = Some(now);
164
0
                return;
165
            }
166
        };
167
1
        let hints = hints
168
1
            .into_iter()
169
1
            .filter_map(|hint| hint.key().map(|key| (key, hint)))
170
1
            .collect::<HashMap<_, _>>();
171
1
        debug!(hints_file = %self.hints_file, hints = hints.len(), "Loaded historical resource hints");
172
1
        let mut hint_state = self.hint_state.lock();
173
1
        hint_state.hints = hints;
174
1
        hint_state.last_loaded = Some(now);
175
1
    }
176
177
1
    fn hint_for_current_action(&self) -> Option<HistoricalResourceHint> {
178
1
        self.refresh_hints();
179
1
        let ctx = Context::current();
180
1
        let metadata = ctx
181
1
            .baggage()
182
1
            .get(BAZEL_METADATA_KEY)
183
1
            .and_then(|value| request_metadata_from_baggage(value.as_str()).ok())
?0
;
184
1
        let target_id = non_empty_string(metadata.target_id);
185
1
        let action_mnemonic = non_empty_string(metadata.action_mnemonic);
186
1
        let hint_state = self.hint_state.lock();
187
1
        let exact_key = HintKey {
188
1
            target_id: target_id.clone(),
189
1
            action_mnemonic: action_mnemonic.clone(),
190
1
        };
191
1
        hint_state
192
1
            .hints
193
1
            .get(&exact_key)
194
1
            .or_else(|| 
{0
195
0
                hint_state.hints.get(&HintKey {
196
0
                    target_id,
197
0
                    action_mnemonic: None,
198
0
                })
199
0
            })
200
1
            .or_else(|| 
{0
201
0
                hint_state.hints.get(&HintKey {
202
0
                    target_id: None,
203
0
                    action_mnemonic,
204
0
                })
205
0
            })
206
1
            .cloned()
207
1
    }
208
209
1
    fn apply_hint(&self, action_info: &mut ActionInfo, hint: &HistoricalResourceHint) {
210
1
        if let Some(cpu_count) = hint.cpu_count {
211
1
            apply_minimum_platform_property(
212
1
                &mut action_info.platform_properties,
213
1
                &self.cpu_property_name,
214
1
                cpu_count,
215
1
            );
216
1
        
}0
217
1
        if let Some(memory_kb) = hint.memory_kb() {
218
1
            apply_minimum_platform_property(
219
1
                &mut action_info.platform_properties,
220
1
                &self.memory_property_name,
221
1
                memory_kb,
222
1
            );
223
1
        
}0
224
1
    }
225
226
0
    async fn inner_get_known_properties(&self, instance_name: &str) -> Result<Vec<String>, Error> {
227
        {
228
0
            let known_properties = self.known_properties.lock();
229
0
            if let Some(property_manager) = known_properties.get(instance_name) {
230
0
                return Ok(property_manager.clone());
231
0
            }
232
        }
233
0
        let known_platform_property_provider = self
234
0
            .scheduler
235
0
            .as_ref()
236
0
            .err_tip(|| "Inner scheduler does not implement KnownPlatformPropertyProvider for HistoricalResourceScheduler")?;
237
0
        let mut known_properties = HashSet::<String>::from_iter(
238
0
            known_platform_property_provider
239
0
                .get_known_properties(instance_name)
240
0
                .await?,
241
        );
242
0
        known_properties.insert(self.cpu_property_name.clone());
243
0
        known_properties.insert(self.memory_property_name.clone());
244
0
        let final_known_properties: Vec<String> = known_properties.into_iter().collect();
245
0
        self.known_properties
246
0
            .lock()
247
0
            .insert(instance_name.to_string(), final_known_properties.clone());
248
0
        Ok(final_known_properties)
249
0
    }
250
}
251
252
2
fn non_empty_string(value: String) -> Option<String> {
253
2
    if value.is_empty() { 
None0
} else { Some(value) }
254
2
}
255
256
2
fn apply_minimum_platform_property(
257
2
    platform_properties: &mut HashMap<String, String>,
258
2
    property_name: &str,
259
2
    minimum_value: u64,
260
2
) {
261
2
    if minimum_value == 0 {
262
0
        return;
263
2
    }
264
2
    let should_update = platform_properties
265
2
        .get(property_name)
266
2
        .and_then(|value| value.parse::<u64>().ok())
267
2
        .is_none_or(|current_value| current_value < minimum_value);
268
2
    if should_update {
269
1
        platform_properties.insert(property_name.to_string(), minimum_value.to_string());
270
1
    }
271
2
}
272
273
#[async_trait]
274
impl KnownPlatformPropertyProvider for HistoricalResourceScheduler {
275
0
    async fn get_known_properties(&self, instance_name: &str) -> Result<Vec<String>, Error> {
276
        self.inner_get_known_properties(instance_name).await
277
0
    }
278
}
279
280
#[async_trait]
281
impl ClientStateManager for HistoricalResourceScheduler {
282
    async fn add_action(
283
        &self,
284
        client_operation_id: OperationId,
285
        mut action_info: Arc<ActionInfo>,
286
1
    ) -> Result<Box<dyn ActionStateResult>, Error> {
287
        if let Some(hint) = self.hint_for_current_action() {
288
            self.apply_hint(Arc::make_mut(&mut action_info), &hint);
289
        }
290
        self.scheduler
291
            .as_ref()
292
            .err_tip(|| "Inner scheduler not available for HistoricalResourceScheduler")?
293
            .add_action(client_operation_id, action_info)
294
            .await
295
1
    }
296
297
    async fn filter_operations<'a>(
298
        &'a self,
299
        filter: OperationFilter,
300
0
    ) -> Result<ActionStateResultStream<'a>, Error> {
301
        self.scheduler
302
            .as_ref()
303
            .err_tip(|| "Inner scheduler not available for HistoricalResourceScheduler")?
304
            .filter_operations(filter)
305
            .await
306
0
    }
307
}
308
309
impl MetricsComponent for HistoricalResourceScheduler {
310
0
    fn publish(
311
0
        &self,
312
0
        kind: MetricKind,
313
0
        field_metadata: MetricFieldData,
314
0
    ) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> {
315
0
        match &self.scheduler {
316
0
            Some(scheduler) => scheduler.publish(kind, field_metadata),
317
0
            None => Ok(MetricPublishKnownKindData::Component),
318
        }
319
0
    }
320
}
321
322
impl RootMetricsComponent for HistoricalResourceScheduler {}