Coverage Report

Created: 2026-11-13 23:29

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-sim/src/property.rs
Line
Count
Source
1
// Copyright 2026 Trace Machina, Inc. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
//! Property mode: scenarios generated from a seed, and a shrinker that
16
//! cuts a failing one down to the steps that still fail and prints it as
17
//! builder code to paste into the catalogue.
18
19
use core::fmt::Write as _;
20
use core::time::Duration;
21
22
use nativelink_error::Error;
23
24
use crate::actions::{Action, Run};
25
use crate::checker::{Invariant, verdict};
26
use crate::net::Link;
27
use crate::scenario::{Scenario, Step};
28
use crate::world::{Backend, WorkerSpec};
29
30
/// `SplitMix64`, so a seed gives the same scenario on every machine.
31
#[derive(Debug, Clone, Copy)]
32
pub struct Rng(u64);
33
34
impl Rng {
35
12.2k
    pub const fn new(seed: u64) -> Self {
36
12.2k
        Self(seed)
37
12.2k
    }
38
39
561k
    pub const fn next_u64(&mut self) -> u64 {
40
561k
        self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15);
41
561k
        let mut z = self.0;
42
561k
        z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
43
561k
        z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
44
561k
        z ^ (z >> 31)
45
561k
    }
46
47
    /// Uniform in `low..=high`.
48
33.5k
    pub const fn range(&mut self, low: u64, high: u64) -> u64 {
49
33.5k
        low + self.next_u64() % (high - low + 1)
50
33.5k
    }
51
52
    /// Uniform in `0..len`.
53
414
    pub fn index(&mut self, len: usize) -> usize {
54
414
        usize::try_from(self.next_u64() % len as u64).unwrap_or(0)
55
414
    }
56
57
15.6k
    pub const fn chance(&mut self, percent: u64) -> bool {
58
15.6k
        self.range(1, 100) <= percent
59
15.6k
    }
60
}
61
62
/// What a generated scenario may do to its workers and clients.
63
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
64
#[allow(clippy::struct_excessive_bools, reason = "one switch per fault kind")]
65
pub struct Faults {
66
    pub kill: bool,
67
    pub cut: bool,
68
    pub silence: bool,
69
    pub client_gone: bool,
70
}
71
72
impl Faults {
73
    pub const ALL: Self = Self {
74
        kill: true,
75
        cut: true,
76
        silence: true,
77
        client_gone: true,
78
    };
79
}
80
81
/// One to eight workers, one to forty actions over twenty seconds, and up
82
/// to four faults from `faults`. A killed worker is replaced two seconds
83
/// later; a fleet that only shrinks proves nothing.
84
80
pub fn generate(seed: u64, backends: &[Backend], faults: Faults) -> Scenario {
85
80
    let mut rng = Rng::new(seed);
86
80
    let workers = 1 + rng.index(8);
87
80
    let spec = WorkerSpec::default().max_inflight(rng.range(1, 3));
88
80
    let mut scenario = Scenario::new(&format!("property-{seed}"), seed)
89
80
        .backends(backends)
90
80
        .settle(Duration::from_mins(10))
91
80
        .workers(workers, spec.clone());
92
80
    let actions = u32::try_from(rng.range(1, 40)).unwrap_or(1);
93
1.73k
    for i in 
0..actions80
{
94
1.73k
        let mut action = Action::new(i)
95
1.73k
            .run_ms(rng.range(100, 5_000))
96
1.73k
            .priority(i32::try_from(rng.range(0, 3)).unwrap_or(0));
97
1.73k
        if rng.chance(20) {
98
338
            action = action.uncacheable();
99
1.39k
        }
100
1.73k
        if rng.chance(15) {
101
260
            action = action.attempts(&[Run::Error(nativelink_error::Code::Internal), Run::Succeed]);
102
1.47k
        }
103
1.73k
        scenario = scenario.at(
104
1.73k
            rng.range(0, 20_000),
105
1.73k
            Step::execute(&format!("c{i}"), action),
106
        );
107
    }
108
80
    let mut menu = Vec::new();
109
80
    if faults.kill {
110
80
        menu.push(0);
111
80
    
}0
112
80
    if faults.cut {
113
80
        menu.push(1);
114
80
    
}0
115
80
    if faults.silence {
116
64
        menu.push(2);
117
64
    
}16
118
80
    if faults.client_gone {
119
80
        menu.push(3);
120
80
    
}0
121
80
    if menu.is_empty() {
122
0
        return scenario;
123
80
    }
124
80
    for _ in 0..rng.range(0, 4) {
125
167
        let at = rng.range(0, 25_000);
126
167
        let worker = rng.index(workers);
127
167
        scenario = match menu[rng.index(menu.len())] {
128
            // A killed worker's pod comes back, as it would in a cluster.
129
44
            0 => scenario
130
44
                .at(at, Step::KillWorker { worker })
131
44
                .at(at + 2_000, Step::add_workers(1, spec.clone())),
132
42
            1 => scenario.at(at, Step::CutWorker { worker }),
133
32
            2 => scenario.silence(worker, at, if rng.chance(50) { 
1_00013
} else {
15_00019
}),
134
49
            _ => scenario.at(
135
49
                at,
136
49
                Step::client_gone(&format!("c{}", rng.range(0, u64::from(actions) - 1))),
137
            ),
138
        };
139
    }
140
80
    scenario
141
80
}
142
143
/// The invariants `scenario` breaks on `backend`.
144
80
pub fn failures(scenario: &Scenario, backend: Backend) -> Result<Vec<Invariant>, Error> {
145
80
    let report = scenario.run_on(backend)
?0
;
146
80
    Ok(verdict(&report.violations).into_iter().collect())
147
80
}
148
149
/// Removes steps one at a time while the scenario still breaks `target`
150
/// on `backend`, until no single removal keeps it broken. Worker additions
151
/// at time zero stay, so the fleet does not vanish.
152
0
pub fn shrink(scenario: &Scenario, backend: Backend, target: Invariant) -> Result<Scenario, Error> {
153
0
    let mut current = scenario.clone();
154
    loop {
155
0
        let steps = current.steps().to_vec();
156
0
        let mut smaller = None;
157
0
        for i in 0..steps.len() {
158
0
            if matches!(steps[i], (0, Step::AddWorkers { .. })) {
159
0
                continue;
160
0
            }
161
0
            let mut candidate_steps = steps.clone();
162
0
            candidate_steps.remove(i);
163
            // A silence and its end go together.
164
            if let Step::SetLink {
165
0
                worker,
166
                link: Link::Silent,
167
0
            } = steps[i].1
168
0
                && let Some(j) = candidate_steps.iter().position(|(at, s)| {
169
0
                    *at > steps[i].0
170
0
                        && matches!(s, Step::SetLink { worker: w, link: Link::Up } if *w == worker)
171
0
                })
172
0
            {
173
0
                candidate_steps.remove(j);
174
0
            }
175
0
            let candidate = current.clone().with_steps(candidate_steps);
176
0
            if failures(&candidate, backend)?.contains(&target) {
177
0
                smaller = Some(candidate);
178
0
                break;
179
0
            }
180
        }
181
0
        match smaller {
182
0
            Some(next) => current = next,
183
0
            None => return Ok(current),
184
        }
185
    }
186
0
}
187
188
/// `scenario` as the builder code that makes it.
189
0
pub fn to_code(scenario: &Scenario) -> String {
190
0
    let mut code = format!("Scenario::new({:?}, 0)\n", scenario.name());
191
0
    for (at, step) in scenario.steps() {
192
0
        writeln!(code, "    .at({at}, {})", step.to_code()).unwrap();
193
0
    }
194
0
    code.push_str("    .run()");
195
0
    code
196
0
}