/build/source/nativelink-sim/src/property.rs
Line | Count | Source |
1 | | // Copyright 2026 Trace Machina, Inc. All rights reserved. |
2 | | // |
3 | | // Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // See LICENSE file for details |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | //! Property mode: scenarios generated from a seed, and a shrinker that |
16 | | //! cuts a failing one down to the steps that still fail and prints it as |
17 | | //! builder code to paste into the catalogue. |
18 | | |
19 | | use core::fmt::Write as _; |
20 | | use core::time::Duration; |
21 | | |
22 | | use nativelink_error::Error; |
23 | | |
24 | | use crate::actions::{Action, Run}; |
25 | | use crate::checker::{Invariant, verdict}; |
26 | | use crate::net::Link; |
27 | | use crate::scenario::{Scenario, Step}; |
28 | | use crate::world::{Backend, WorkerSpec}; |
29 | | |
30 | | /// `SplitMix64`, so a seed gives the same scenario on every machine. |
31 | | #[derive(Debug, Clone, Copy)] |
32 | | pub struct Rng(u64); |
33 | | |
34 | | impl Rng { |
35 | 12.2k | pub const fn new(seed: u64) -> Self { |
36 | 12.2k | Self(seed) |
37 | 12.2k | } |
38 | | |
39 | 561k | pub const fn next_u64(&mut self) -> u64 { |
40 | 561k | self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); |
41 | 561k | let mut z = self.0; |
42 | 561k | z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); |
43 | 561k | z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); |
44 | 561k | z ^ (z >> 31) |
45 | 561k | } |
46 | | |
47 | | /// Uniform in `low..=high`. |
48 | 33.5k | pub const fn range(&mut self, low: u64, high: u64) -> u64 { |
49 | 33.5k | low + self.next_u64() % (high - low + 1) |
50 | 33.5k | } |
51 | | |
52 | | /// Uniform in `0..len`. |
53 | 414 | pub fn index(&mut self, len: usize) -> usize { |
54 | 414 | usize::try_from(self.next_u64() % len as u64).unwrap_or(0) |
55 | 414 | } |
56 | | |
57 | 15.6k | pub const fn chance(&mut self, percent: u64) -> bool { |
58 | 15.6k | self.range(1, 100) <= percent |
59 | 15.6k | } |
60 | | } |
61 | | |
62 | | /// What a generated scenario may do to its workers and clients. |
63 | | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
64 | | #[allow(clippy::struct_excessive_bools, reason = "one switch per fault kind")] |
65 | | pub struct Faults { |
66 | | pub kill: bool, |
67 | | pub cut: bool, |
68 | | pub silence: bool, |
69 | | pub client_gone: bool, |
70 | | } |
71 | | |
72 | | impl Faults { |
73 | | pub const ALL: Self = Self { |
74 | | kill: true, |
75 | | cut: true, |
76 | | silence: true, |
77 | | client_gone: true, |
78 | | }; |
79 | | } |
80 | | |
81 | | /// One to eight workers, one to forty actions over twenty seconds, and up |
82 | | /// to four faults from `faults`. A killed worker is replaced two seconds |
83 | | /// later; a fleet that only shrinks proves nothing. |
84 | 80 | pub fn generate(seed: u64, backends: &[Backend], faults: Faults) -> Scenario { |
85 | 80 | let mut rng = Rng::new(seed); |
86 | 80 | let workers = 1 + rng.index(8); |
87 | 80 | let spec = WorkerSpec::default().max_inflight(rng.range(1, 3)); |
88 | 80 | let mut scenario = Scenario::new(&format!("property-{seed}"), seed) |
89 | 80 | .backends(backends) |
90 | 80 | .settle(Duration::from_mins(10)) |
91 | 80 | .workers(workers, spec.clone()); |
92 | 80 | let actions = u32::try_from(rng.range(1, 40)).unwrap_or(1); |
93 | 1.73k | for i in 0..actions80 { |
94 | 1.73k | let mut action = Action::new(i) |
95 | 1.73k | .run_ms(rng.range(100, 5_000)) |
96 | 1.73k | .priority(i32::try_from(rng.range(0, 3)).unwrap_or(0)); |
97 | 1.73k | if rng.chance(20) { |
98 | 338 | action = action.uncacheable(); |
99 | 1.39k | } |
100 | 1.73k | if rng.chance(15) { |
101 | 260 | action = action.attempts(&[Run::Error(nativelink_error::Code::Internal), Run::Succeed]); |
102 | 1.47k | } |
103 | 1.73k | scenario = scenario.at( |
104 | 1.73k | rng.range(0, 20_000), |
105 | 1.73k | Step::execute(&format!("c{i}"), action), |
106 | | ); |
107 | | } |
108 | 80 | let mut menu = Vec::new(); |
109 | 80 | if faults.kill { |
110 | 80 | menu.push(0); |
111 | 80 | }0 |
112 | 80 | if faults.cut { |
113 | 80 | menu.push(1); |
114 | 80 | }0 |
115 | 80 | if faults.silence { |
116 | 64 | menu.push(2); |
117 | 64 | }16 |
118 | 80 | if faults.client_gone { |
119 | 80 | menu.push(3); |
120 | 80 | }0 |
121 | 80 | if menu.is_empty() { |
122 | 0 | return scenario; |
123 | 80 | } |
124 | 80 | for _ in 0..rng.range(0, 4) { |
125 | 167 | let at = rng.range(0, 25_000); |
126 | 167 | let worker = rng.index(workers); |
127 | 167 | scenario = match menu[rng.index(menu.len())] { |
128 | | // A killed worker's pod comes back, as it would in a cluster. |
129 | 44 | 0 => scenario |
130 | 44 | .at(at, Step::KillWorker { worker }) |
131 | 44 | .at(at + 2_000, Step::add_workers(1, spec.clone())), |
132 | 42 | 1 => scenario.at(at, Step::CutWorker { worker }), |
133 | 32 | 2 => scenario.silence(worker, at, if rng.chance(50) { 1_00013 } else { 15_00019 }), |
134 | 49 | _ => scenario.at( |
135 | 49 | at, |
136 | 49 | Step::client_gone(&format!("c{}", rng.range(0, u64::from(actions) - 1))), |
137 | | ), |
138 | | }; |
139 | | } |
140 | 80 | scenario |
141 | 80 | } |
142 | | |
143 | | /// The invariants `scenario` breaks on `backend`. |
144 | 80 | pub fn failures(scenario: &Scenario, backend: Backend) -> Result<Vec<Invariant>, Error> { |
145 | 80 | let report = scenario.run_on(backend)?0 ; |
146 | 80 | Ok(verdict(&report.violations).into_iter().collect()) |
147 | 80 | } |
148 | | |
149 | | /// Removes steps one at a time while the scenario still breaks `target` |
150 | | /// on `backend`, until no single removal keeps it broken. Worker additions |
151 | | /// at time zero stay, so the fleet does not vanish. |
152 | 0 | pub fn shrink(scenario: &Scenario, backend: Backend, target: Invariant) -> Result<Scenario, Error> { |
153 | 0 | let mut current = scenario.clone(); |
154 | | loop { |
155 | 0 | let steps = current.steps().to_vec(); |
156 | 0 | let mut smaller = None; |
157 | 0 | for i in 0..steps.len() { |
158 | 0 | if matches!(steps[i], (0, Step::AddWorkers { .. })) { |
159 | 0 | continue; |
160 | 0 | } |
161 | 0 | let mut candidate_steps = steps.clone(); |
162 | 0 | candidate_steps.remove(i); |
163 | | // A silence and its end go together. |
164 | | if let Step::SetLink { |
165 | 0 | worker, |
166 | | link: Link::Silent, |
167 | 0 | } = steps[i].1 |
168 | 0 | && let Some(j) = candidate_steps.iter().position(|(at, s)| { |
169 | 0 | *at > steps[i].0 |
170 | 0 | && matches!(s, Step::SetLink { worker: w, link: Link::Up } if *w == worker) |
171 | 0 | }) |
172 | 0 | { |
173 | 0 | candidate_steps.remove(j); |
174 | 0 | } |
175 | 0 | let candidate = current.clone().with_steps(candidate_steps); |
176 | 0 | if failures(&candidate, backend)?.contains(&target) { |
177 | 0 | smaller = Some(candidate); |
178 | 0 | break; |
179 | 0 | } |
180 | | } |
181 | 0 | match smaller { |
182 | 0 | Some(next) => current = next, |
183 | 0 | None => return Ok(current), |
184 | | } |
185 | | } |
186 | 0 | } |
187 | | |
188 | | /// `scenario` as the builder code that makes it. |
189 | 0 | pub fn to_code(scenario: &Scenario) -> String { |
190 | 0 | let mut code = format!("Scenario::new({:?}, 0)\n", scenario.name()); |
191 | 0 | for (at, step) in scenario.steps() { |
192 | 0 | writeln!(code, " .at({at}, {})", step.to_code()).unwrap(); |
193 | 0 | } |
194 | 0 | code.push_str(" .run()"); |
195 | 0 | code |
196 | 0 | } |