Coverage Report

Created: 2026-11-13 23:29

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/src/bin/redis_store_tester.rs
Line
Count
Source
1
// Copyright 2026 Trace Machina, Inc. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::pin::Pin;
16
use core::sync::atomic::{AtomicUsize, Ordering};
17
use core::time::Duration;
18
use std::borrow::Cow;
19
use std::env;
20
use std::sync::{Arc, RwLock};
21
22
use bytes::Bytes;
23
use clap::{Parser, ValueEnum};
24
use futures::TryStreamExt;
25
use nativelink_config::stores::{RedisMode, RedisSpec};
26
use nativelink_error::{Code, Error, ResultExt};
27
use nativelink_store::redis_store::RedisStore;
28
use nativelink_util::buf_channel::make_buf_channel_pair;
29
use nativelink_util::store_trait::{
30
    RemoveItemCallback, SchedulerCurrentVersionProvider, SchedulerIndexProvider, SchedulerStore,
31
    SchedulerStoreDataProvider, SchedulerStoreDecodeTo, SchedulerStoreKeyProvider, StoreDriver,
32
    StoreKey, StoreLike, TrueValue, UploadSizeInfo,
33
};
34
use nativelink_util::telemetry::init_tracing;
35
use nativelink_util::{background_spawn, spawn};
36
use rand::Rng;
37
use tokio::time::sleep;
38
use tracing::{error, info};
39
40
// Define test structures that implement the scheduler traits
41
#[derive(Debug, Clone, PartialEq)]
42
struct TestSchedulerData {
43
    key: String,
44
    content: String,
45
    version: i64,
46
}
47
48
#[derive(Debug)]
49
struct TestSchedulerReturn {
50
    version: i64,
51
}
52
53
impl SchedulerStoreKeyProvider for TestSchedulerData {
54
    type Versioned = TrueValue; // Using versioned storage
55
56
0
    fn get_key(&self) -> StoreKey<'static> {
57
0
        StoreKey::Str(Cow::Owned(self.key.clone()))
58
0
    }
59
}
60
61
impl SchedulerStoreDataProvider for TestSchedulerData {
62
0
    fn try_into_bytes(self) -> Result<Bytes, Error> {
63
0
        Ok(Bytes::from(self.content))
64
0
    }
65
66
0
    fn get_indexes(&self) -> Result<Vec<(&'static str, Bytes)>, Error> {
67
        // Add some test indexes - need to use 'static strings
68
0
        Ok(vec![
69
0
            ("test_index", Bytes::from("test_value")),
70
0
            (
71
0
                "content_prefix",
72
0
                Bytes::from(self.content.chars().take(10).collect::<String>()),
73
0
            ),
74
0
        ])
75
0
    }
76
}
77
78
impl SchedulerStoreDecodeTo for TestSchedulerData {
79
    type DecodeOutput = TestSchedulerReturn;
80
81
0
    fn decode(version: i64, _data: Bytes) -> Result<Self::DecodeOutput, Error> {
82
0
        Ok(TestSchedulerReturn { version })
83
0
    }
84
}
85
86
impl SchedulerCurrentVersionProvider for TestSchedulerData {
87
0
    fn current_version(&self) -> i64 {
88
0
        self.version
89
0
    }
90
}
91
92
struct SearchByContentPrefix {
93
    prefix: String,
94
}
95
96
impl SchedulerIndexProvider for SearchByContentPrefix {
97
    const KEY_PREFIX: &'static str = "test:";
98
    const INDEX_NAME: &'static str = "content_prefix";
99
    type Versioned = TrueValue;
100
101
0
    fn index_value(&self) -> Cow<'_, str> {
102
0
        Cow::Borrowed(&self.prefix)
103
0
    }
104
}
105
106
impl SchedulerStoreKeyProvider for SearchByContentPrefix {
107
    type Versioned = TrueValue;
108
109
0
    fn get_key(&self) -> StoreKey<'static> {
110
0
        StoreKey::Str(Cow::Owned("dummy_key".to_string()))
111
0
    }
112
}
113
114
impl SchedulerStoreDecodeTo for SearchByContentPrefix {
115
    type DecodeOutput = TestSchedulerReturn;
116
117
0
    fn decode(version: i64, data: Bytes) -> Result<Self::DecodeOutput, Error> {
118
0
        TestSchedulerData::decode(version, data)
119
0
    }
120
}
121
122
const MAX_KEY: u16 = 1024;
123
124
/// Wrapper type for CLI parsing since we can't implement foreign traits on foreign types.
125
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, ValueEnum)]
126
enum RedisModeArg {
127
    Cluster,
128
    Sentinel,
129
    #[default]
130
    Standard,
131
}
132
133
impl From<RedisModeArg> for RedisMode {
134
0
    fn from(arg: RedisModeArg) -> Self {
135
0
        match arg {
136
0
            RedisModeArg::Standard => Self::Standard,
137
0
            RedisModeArg::Sentinel => Self::Sentinel,
138
0
            RedisModeArg::Cluster => Self::Cluster,
139
        }
140
0
    }
141
}
142
143
0
fn random_key() -> StoreKey<'static> {
144
0
    let key = rand::rng().random_range(0..MAX_KEY);
145
0
    StoreKey::new_str(&key.to_string()).into_owned()
146
0
}
147
148
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, ValueEnum)]
149
enum TestMode {
150
    #[default]
151
    Random,
152
    Sequential,
153
}
154
155
#[derive(Parser, Debug)]
156
#[command(version, about)]
157
struct Args {
158
    #[arg(value_enum, short, long, default_value_t)]
159
    redis_mode: RedisModeArg,
160
161
    #[arg(value_enum, short, long, default_value_t)]
162
    mode: TestMode,
163
}
164
165
0
async fn run<S: StoreDriver + SchedulerStore>(
166
0
    store: Arc<S>,
167
0
    max_loops: usize,
168
0
    failed: Arc<RwLock<bool>>,
169
0
    mode: TestMode,
170
0
) -> Result<(), Error> {
171
0
    let mut count = 0;
172
0
    let in_flight = Arc::new(AtomicUsize::new(0));
173
0
    let fixed_action_value = if let Ok(str_action_value) = env::var("ACTION_VALUE") {
174
0
        Some(str::parse::<usize>(&str_action_value).unwrap())
175
    } else {
176
0
        None
177
    };
178
179
    loop {
180
0
        if count % 1000 == 0 {
181
0
            info!(
182
                "Loop count {count}. In flight: {}",
183
0
                in_flight.load(Ordering::Relaxed)
184
            );
185
0
            if *failed.read().unwrap() {
186
0
                return Err(Error::new(
187
0
                    Code::Internal,
188
0
                    "Failed in redis_store_tester".to_string(),
189
0
                ));
190
0
            }
191
0
        }
192
0
        if count == max_loops {
193
            loop {
194
0
                let remaining = in_flight.load(Ordering::Relaxed);
195
0
                if remaining == 0 {
196
0
                    return Ok(());
197
0
                }
198
0
                info!(remaining, "Remaining");
199
0
                sleep(Duration::from_secs(1)).await;
200
            }
201
0
        }
202
0
        count += 1;
203
0
        in_flight.fetch_add(1, Ordering::Relaxed);
204
205
0
        let store_clone = store.clone();
206
0
        let local_fail = failed.clone();
207
0
        let local_in_flight = in_flight.clone();
208
209
0
        let max_action_value = 7;
210
0
        let action_value = if let Some(av) = fixed_action_value {
211
0
            av
212
        } else {
213
0
            match mode {
214
0
                TestMode::Random => rand::rng().random_range(0..max_action_value),
215
0
                TestMode::Sequential => count % max_action_value,
216
            }
217
        };
218
219
0
        background_spawn!("action", async move {
220
0
            async fn run_action<S: StoreDriver + SchedulerStore>(
221
0
                action_value: usize,
222
0
                store_clone: Arc<S>,
223
0
            ) -> Result<(), Error> {
224
0
                match action_value {
225
                    0 => {
226
0
                        store_clone.has(random_key()).await?;
227
                    }
228
                    1 => {
229
0
                        let (mut tx, rx) = make_buf_channel_pair();
230
0
                        tx.send(Bytes::from_static(b"12345")).await?;
231
0
                        tx.send_eof()?;
232
0
                        store_clone
233
0
                            .update(random_key(), rx, UploadSizeInfo::ExactSize(5))
234
0
                            .await?;
235
                    }
236
                    2 => {
237
0
                        let mut results = (0..MAX_KEY).map(|_| None).collect::<Vec<_>>();
238
239
0
                        store_clone
240
0
                            .has_with_results(
241
0
                                &(0..MAX_KEY)
242
0
                                    .map(|i| StoreKey::Str(Cow::Owned(i.to_string())))
243
0
                                    .collect::<Vec<_>>(),
244
0
                                &mut results,
245
                            )
246
0
                            .await?;
247
                    }
248
                    3 => {
249
0
                        let key = random_key();
250
0
                        store_clone
251
0
                            .update_oneshot(key.borrow(), Bytes::from_static(b"1234"))
252
0
                            .await?;
253
0
                        info!(?key, "Updated");
254
                    }
255
                    4 => {
256
0
                        let res = store_clone
257
0
                            .list(.., |_key| true)
258
0
                            .await
259
0
                            .err_tip(|| "In list")?;
260
0
                        info!(%res, "end list");
261
                    }
262
                    5 => {
263
0
                        let search_provider = SearchByContentPrefix {
264
0
                            prefix: "Searchable".to_string(),
265
0
                        };
266
0
                        for i in 0..5 {
267
0
                            let data = TestSchedulerData {
268
0
                                key: format!("test:search_key_{i}"),
269
0
                                content: format!("Searchable content #{i}"),
270
0
                                version: 0,
271
0
                            };
272
273
0
                            store_clone.update_data(data, None).await?;
274
                        }
275
0
                        let search_results: Vec<_> = store_clone
276
0
                            .search_by_index_prefix(search_provider)
277
0
                            .await?
278
0
                            .try_collect()
279
0
                            .await?;
280
0
                        info!(?search_results, "search results");
281
                    }
282
                    _ => {
283
0
                        let mut data = TestSchedulerData {
284
0
                            key: "test:scheduler_key_1".to_string(),
285
0
                            content: "Test scheduler data #1".to_string(),
286
0
                            version: 0,
287
0
                        };
288
289
0
                        let res = store_clone.get_and_decode(data.clone()).await?;
290
0
                        if let Some(existing_data) = res {
291
0
                            data.version = existing_data.version + 1;
292
0
                        }
293
294
0
                        store_clone
295
0
                            .update_data(data, Some(Duration::from_mins(1)))
296
0
                            .await?;
297
                    }
298
                }
299
0
                Ok(())
300
0
            }
301
0
            match run_action(action_value, store_clone).await {
302
0
                Ok(()) => {}
303
0
                Err(e) => {
304
0
                    error!(?e, "Error!");
305
0
                    *local_fail.write().unwrap() = true;
306
                }
307
            }
308
0
            local_in_flight.fetch_sub(1, Ordering::Relaxed);
309
0
        });
310
    }
311
0
}
312
313
#[derive(Debug)]
314
struct LoggingRemoveCallback {}
315
316
impl RemoveItemCallback for LoggingRemoveCallback {
317
0
    fn callback<'a>(
318
0
        &'a self,
319
0
        store_key: StoreKey<'a>,
320
0
    ) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
321
0
        info!(?store_key, "Callback for removed item");
322
0
        Box::pin(async {})
323
0
    }
324
}
325
326
0
fn main() -> Result<(), Box<dyn core::error::Error>> {
327
0
    let args = Args::parse();
328
0
    let redis_mode: RedisMode = args.redis_mode.into();
329
330
0
    let failed = Arc::new(RwLock::new(false));
331
0
    let redis_host = env::var("REDIS_HOST").unwrap_or_else(|_| "127.0.0.1".to_string());
332
0
    let max_client_permits = env::var("MAX_REDIS_PERMITS")
333
0
        .unwrap_or_else(|_| "100".to_string())
334
0
        .parse()?;
335
0
    let max_loops: usize = env::var("MAX_LOOPS")
336
0
        .unwrap_or_else(|_| "2000000".to_string())
337
0
        .parse()?;
338
339
    #[expect(
340
        clippy::disallowed_methods,
341
        reason = "`We need `tokio::runtime::Runtime::block_on` so we can get errors _after_ threads finished"
342
    )]
343
0
    tokio::runtime::Builder::new_multi_thread()
344
0
        .enable_all()
345
0
        .build()
346
0
        .unwrap()
347
0
        .block_on(async {
348
            // The OTLP exporters need to run in a Tokio context.
349
0
            spawn!("init tracing", async { init_tracing().await })
350
0
                .await?
351
0
                .expect("Init tracing should work");
352
353
0
            let redis_port = match redis_mode {
354
0
                RedisMode::Standard => 6379,
355
0
                RedisMode::Sentinel => 26379,
356
0
                RedisMode::Cluster => 7000,
357
            };
358
0
            let addr = match redis_mode {
359
0
                RedisMode::Sentinel => format!("redis+sentinel://{redis_host}:{redis_port}/"),
360
0
                _ => format!("redis://{redis_host}:{redis_port}/"),
361
            };
362
0
            let spec = RedisSpec {
363
0
                addresses: vec![addr],
364
0
                connection_timeout_ms: 1000,
365
0
                max_client_permits,
366
0
                mode: redis_mode,
367
0
                ..Default::default()
368
0
            };
369
0
            match spec.mode {
370
                RedisMode::Standard | RedisMode::Sentinel => {
371
0
                    let store = RedisStore::new_standard(spec).await?;
372
0
                    store
373
0
                        .clone()
374
0
                        .register_remove_callback(Arc::new(LoggingRemoveCallback {}))?;
375
0
                    run(store, max_loops, failed.clone(), args.mode).await
376
                }
377
                RedisMode::Cluster => {
378
0
                    let store = RedisStore::new_cluster(spec).await?;
379
0
                    store
380
0
                        .clone()
381
0
                        .register_remove_callback(Arc::new(LoggingRemoveCallback {}))?;
382
0
                    run(store, max_loops, failed.clone(), args.mode).await
383
                }
384
            }
385
0
        })
386
0
        .unwrap();
387
0
    if *failed.read().unwrap() {
388
0
        return Err(Error::new(Code::Internal, "Failed in redis_store_tester".to_string()).into());
389
0
    }
390
0
    Ok(())
391
0
}