/build/source/nativelink-scheduler/src/historical_resource_scheduler.rs
Line | Count | Source |
1 | | // Copyright 2026 The NativeLink Authors. All rights reserved. |
2 | | // |
3 | | // Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // See LICENSE file for details |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | use core::time::Duration; |
16 | | use std::collections::{HashMap, HashSet}; |
17 | | use std::fs; |
18 | | use std::sync::Arc; |
19 | | use std::time::Instant; |
20 | | |
21 | | use async_trait::async_trait; |
22 | | use nativelink_config::schedulers::HistoricalResourceSpec; |
23 | | use nativelink_error::{Error, ResultExt}; |
24 | | use nativelink_metric::{ |
25 | | MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent, RootMetricsComponent, |
26 | | }; |
27 | | use nativelink_util::action_messages::{ActionInfo, OperationId}; |
28 | | use nativelink_util::operation_state_manager::{ |
29 | | ActionStateResult, ActionStateResultStream, ClientStateManager, OperationFilter, |
30 | | }; |
31 | | use nativelink_util::origin_event::{BAZEL_METADATA_KEY, request_metadata_from_baggage}; |
32 | | use opentelemetry::baggage::BaggageExt; |
33 | | use opentelemetry::context::Context; |
34 | | use parking_lot::Mutex; |
35 | | use serde::Deserialize; |
36 | | use tracing::{debug, warn}; |
37 | | |
38 | | use crate::known_platform_property_provider::KnownPlatformPropertyProvider; |
39 | | |
40 | | #[derive(Debug, Clone, Hash, Eq, PartialEq)] |
41 | | struct HintKey { |
42 | | target_id: Option<String>, |
43 | | action_mnemonic: Option<String>, |
44 | | } |
45 | | |
46 | | #[derive(Debug, Clone, Deserialize)] |
47 | | struct HistoricalResourceHint { |
48 | | #[serde(default)] |
49 | | target_id: Option<String>, |
50 | | #[serde(default)] |
51 | | action_mnemonic: Option<String>, |
52 | | #[serde(default)] |
53 | | cpu_count: Option<u64>, |
54 | | #[serde(default)] |
55 | | memory_kb: Option<u64>, |
56 | | #[serde(default)] |
57 | | memory_mib: Option<u64>, |
58 | | } |
59 | | |
60 | | impl HistoricalResourceHint { |
61 | 1 | fn key(&self) -> Option<HintKey> { |
62 | 1 | if self.target_id.is_none() && self.action_mnemonic0 .is_none0 () { |
63 | 0 | return None; |
64 | 1 | } |
65 | 1 | Some(HintKey { |
66 | 1 | target_id: self.target_id.clone(), |
67 | 1 | action_mnemonic: self.action_mnemonic.clone(), |
68 | 1 | }) |
69 | 1 | } |
70 | | |
71 | 1 | fn memory_kb(&self) -> Option<u64> { |
72 | 1 | self.memory_kb.or_else(|| {0 |
73 | 0 | self.memory_mib |
74 | 0 | .map(|memory_mib| memory_mib.saturating_mul(1024)) |
75 | 0 | }) |
76 | 1 | } |
77 | | } |
78 | | |
79 | | #[derive(Debug, Deserialize)] |
80 | | #[serde(untagged)] |
81 | | enum HistoricalResourceHintFile { |
82 | | List(Vec<HistoricalResourceHint>), |
83 | | Object { hints: Vec<HistoricalResourceHint> }, |
84 | | } |
85 | | |
86 | | impl HistoricalResourceHintFile { |
87 | 1 | fn into_hints(self) -> Vec<HistoricalResourceHint> { |
88 | 1 | match self { |
89 | 1 | Self::List(hints0 ) | Self::Object { hints } => hints, |
90 | | } |
91 | 1 | } |
92 | | } |
93 | | |
94 | | #[derive(Debug, Default)] |
95 | | struct HintState { |
96 | | hints: HashMap<HintKey, HistoricalResourceHint>, |
97 | | last_loaded: Option<Instant>, |
98 | | } |
99 | | |
100 | | pub struct HistoricalResourceScheduler { |
101 | | hints_file: String, |
102 | | refresh_interval: Duration, |
103 | | cpu_property_name: String, |
104 | | memory_property_name: String, |
105 | | scheduler: Option<Arc<dyn KnownPlatformPropertyProvider>>, |
106 | | known_properties: Mutex<HashMap<String, Vec<String>>>, |
107 | | hint_state: Mutex<HintState>, |
108 | | } |
109 | | |
110 | | impl core::fmt::Debug for HistoricalResourceScheduler { |
111 | 0 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { |
112 | 0 | f.debug_struct("HistoricalResourceScheduler") |
113 | 0 | .field("hints_file", &self.hints_file) |
114 | 0 | .field("refresh_interval", &self.refresh_interval) |
115 | 0 | .field("cpu_property_name", &self.cpu_property_name) |
116 | 0 | .field("memory_property_name", &self.memory_property_name) |
117 | 0 | .field("known_properties", &self.known_properties) |
118 | 0 | .finish_non_exhaustive() |
119 | 0 | } |
120 | | } |
121 | | |
122 | | impl HistoricalResourceScheduler { |
123 | | #[must_use] |
124 | 1 | pub fn new( |
125 | 1 | spec: &HistoricalResourceSpec, |
126 | 1 | scheduler: Arc<dyn KnownPlatformPropertyProvider>, |
127 | 1 | ) -> Self { |
128 | 1 | Self { |
129 | 1 | hints_file: spec.hints_file.clone(), |
130 | 1 | refresh_interval: Duration::from_secs(spec.refresh_interval_s), |
131 | 1 | cpu_property_name: spec.cpu_property_name.clone(), |
132 | 1 | memory_property_name: spec.memory_property_name.clone(), |
133 | 1 | scheduler: Some(scheduler), |
134 | 1 | known_properties: Mutex::new(HashMap::new()), |
135 | 1 | hint_state: Mutex::default(), |
136 | 1 | } |
137 | 1 | } |
138 | | |
139 | 1 | fn refresh_hints(&self) { |
140 | 1 | let now = Instant::now(); |
141 | | { |
142 | 1 | let hint_state = self.hint_state.lock(); |
143 | 1 | if let Some(last_loaded0 ) = hint_state.last_loaded |
144 | 0 | && (self.refresh_interval.is_zero() |
145 | 0 | || now.duration_since(last_loaded) < self.refresh_interval) |
146 | | { |
147 | 0 | return; |
148 | 1 | } |
149 | | } |
150 | | |
151 | 1 | let hints_file_contents = match fs::read_to_string(&self.hints_file) { |
152 | 1 | Ok(contents) => contents, |
153 | 0 | Err(err) => { |
154 | 0 | warn!(?err, hints_file = %self.hints_file, "Failed to read historical resource hints"); |
155 | 0 | self.hint_state.lock().last_loaded = Some(now); |
156 | 0 | return; |
157 | | } |
158 | | }; |
159 | 1 | let hints = match serde_json::from_str::<HistoricalResourceHintFile>(&hints_file_contents) { |
160 | 1 | Ok(hints_file) => hints_file.into_hints(), |
161 | 0 | Err(err) => { |
162 | 0 | warn!(?err, hints_file = %self.hints_file, "Failed to parse historical resource hints"); |
163 | 0 | self.hint_state.lock().last_loaded = Some(now); |
164 | 0 | return; |
165 | | } |
166 | | }; |
167 | 1 | let hints = hints |
168 | 1 | .into_iter() |
169 | 1 | .filter_map(|hint| hint.key().map(|key| (key, hint))) |
170 | 1 | .collect::<HashMap<_, _>>(); |
171 | 1 | debug!(hints_file = %self.hints_file, hints = hints.len(), "Loaded historical resource hints"); |
172 | 1 | let mut hint_state = self.hint_state.lock(); |
173 | 1 | hint_state.hints = hints; |
174 | 1 | hint_state.last_loaded = Some(now); |
175 | 1 | } |
176 | | |
177 | 1 | fn hint_for_current_action(&self) -> Option<HistoricalResourceHint> { |
178 | 1 | self.refresh_hints(); |
179 | 1 | let ctx = Context::current(); |
180 | 1 | let metadata = ctx |
181 | 1 | .baggage() |
182 | 1 | .get(BAZEL_METADATA_KEY) |
183 | 1 | .and_then(|value| request_metadata_from_baggage(value.as_str()).ok())?0 ; |
184 | 1 | let target_id = non_empty_string(metadata.target_id); |
185 | 1 | let action_mnemonic = non_empty_string(metadata.action_mnemonic); |
186 | 1 | let hint_state = self.hint_state.lock(); |
187 | 1 | let exact_key = HintKey { |
188 | 1 | target_id: target_id.clone(), |
189 | 1 | action_mnemonic: action_mnemonic.clone(), |
190 | 1 | }; |
191 | 1 | hint_state |
192 | 1 | .hints |
193 | 1 | .get(&exact_key) |
194 | 1 | .or_else(|| {0 |
195 | 0 | hint_state.hints.get(&HintKey { |
196 | 0 | target_id, |
197 | 0 | action_mnemonic: None, |
198 | 0 | }) |
199 | 0 | }) |
200 | 1 | .or_else(|| {0 |
201 | 0 | hint_state.hints.get(&HintKey { |
202 | 0 | target_id: None, |
203 | 0 | action_mnemonic, |
204 | 0 | }) |
205 | 0 | }) |
206 | 1 | .cloned() |
207 | 1 | } |
208 | | |
209 | 1 | fn apply_hint(&self, action_info: &mut ActionInfo, hint: &HistoricalResourceHint) { |
210 | 1 | if let Some(cpu_count) = hint.cpu_count { |
211 | 1 | apply_minimum_platform_property( |
212 | 1 | &mut action_info.platform_properties, |
213 | 1 | &self.cpu_property_name, |
214 | 1 | cpu_count, |
215 | 1 | ); |
216 | 1 | }0 |
217 | 1 | if let Some(memory_kb) = hint.memory_kb() { |
218 | 1 | apply_minimum_platform_property( |
219 | 1 | &mut action_info.platform_properties, |
220 | 1 | &self.memory_property_name, |
221 | 1 | memory_kb, |
222 | 1 | ); |
223 | 1 | }0 |
224 | 1 | } |
225 | | |
226 | 0 | async fn inner_get_known_properties(&self, instance_name: &str) -> Result<Vec<String>, Error> { |
227 | | { |
228 | 0 | let known_properties = self.known_properties.lock(); |
229 | 0 | if let Some(property_manager) = known_properties.get(instance_name) { |
230 | 0 | return Ok(property_manager.clone()); |
231 | 0 | } |
232 | | } |
233 | 0 | let known_platform_property_provider = self |
234 | 0 | .scheduler |
235 | 0 | .as_ref() |
236 | 0 | .err_tip(|| "Inner scheduler does not implement KnownPlatformPropertyProvider for HistoricalResourceScheduler")?; |
237 | 0 | let mut known_properties = HashSet::<String>::from_iter( |
238 | 0 | known_platform_property_provider |
239 | 0 | .get_known_properties(instance_name) |
240 | 0 | .await?, |
241 | | ); |
242 | 0 | known_properties.insert(self.cpu_property_name.clone()); |
243 | 0 | known_properties.insert(self.memory_property_name.clone()); |
244 | 0 | let final_known_properties: Vec<String> = known_properties.into_iter().collect(); |
245 | 0 | self.known_properties |
246 | 0 | .lock() |
247 | 0 | .insert(instance_name.to_string(), final_known_properties.clone()); |
248 | 0 | Ok(final_known_properties) |
249 | 0 | } |
250 | | } |
251 | | |
252 | 2 | fn non_empty_string(value: String) -> Option<String> { |
253 | 2 | if value.is_empty() { None0 } else { Some(value) } |
254 | 2 | } |
255 | | |
256 | 2 | fn apply_minimum_platform_property( |
257 | 2 | platform_properties: &mut HashMap<String, String>, |
258 | 2 | property_name: &str, |
259 | 2 | minimum_value: u64, |
260 | 2 | ) { |
261 | 2 | if minimum_value == 0 { |
262 | 0 | return; |
263 | 2 | } |
264 | 2 | let should_update = platform_properties |
265 | 2 | .get(property_name) |
266 | 2 | .and_then(|value| value.parse::<u64>().ok()) |
267 | 2 | .is_none_or(|current_value| current_value < minimum_value); |
268 | 2 | if should_update { |
269 | 1 | platform_properties.insert(property_name.to_string(), minimum_value.to_string()); |
270 | 1 | } |
271 | 2 | } |
272 | | |
273 | | #[async_trait] |
274 | | impl KnownPlatformPropertyProvider for HistoricalResourceScheduler { |
275 | 0 | async fn get_known_properties(&self, instance_name: &str) -> Result<Vec<String>, Error> { |
276 | | self.inner_get_known_properties(instance_name).await |
277 | 0 | } |
278 | | } |
279 | | |
280 | | #[async_trait] |
281 | | impl ClientStateManager for HistoricalResourceScheduler { |
282 | | async fn add_action( |
283 | | &self, |
284 | | client_operation_id: OperationId, |
285 | | mut action_info: Arc<ActionInfo>, |
286 | 1 | ) -> Result<Box<dyn ActionStateResult>, Error> { |
287 | | if let Some(hint) = self.hint_for_current_action() { |
288 | | self.apply_hint(Arc::make_mut(&mut action_info), &hint); |
289 | | } |
290 | | self.scheduler |
291 | | .as_ref() |
292 | | .err_tip(|| "Inner scheduler not available for HistoricalResourceScheduler")? |
293 | | .add_action(client_operation_id, action_info) |
294 | | .await |
295 | 1 | } |
296 | | |
297 | | async fn filter_operations<'a>( |
298 | | &'a self, |
299 | | filter: OperationFilter, |
300 | 0 | ) -> Result<ActionStateResultStream<'a>, Error> { |
301 | | self.scheduler |
302 | | .as_ref() |
303 | | .err_tip(|| "Inner scheduler not available for HistoricalResourceScheduler")? |
304 | | .filter_operations(filter) |
305 | | .await |
306 | 0 | } |
307 | | } |
308 | | |
309 | | impl MetricsComponent for HistoricalResourceScheduler { |
310 | 0 | fn publish( |
311 | 0 | &self, |
312 | 0 | kind: MetricKind, |
313 | 0 | field_metadata: MetricFieldData, |
314 | 0 | ) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> { |
315 | 0 | match &self.scheduler { |
316 | 0 | Some(scheduler) => scheduler.publish(kind, field_metadata), |
317 | 0 | None => Ok(MetricPublishKnownKindData::Component), |
318 | | } |
319 | 0 | } |
320 | | } |
321 | | |
322 | | impl RootMetricsComponent for HistoricalResourceScheduler {} |