Coverage Report

Created: 2026-10-01 05:28

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-store/src/redis_utils/ft_aggregate.rs
Line
Count
Source
1
// Copyright 2024-2025 The NativeLink Authors. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use core::fmt::Debug;
16
17
use futures::Stream;
18
use redis::aio::ConnectionLike;
19
use redis::{Arg, ErrorKind, RedisError, Value};
20
use tracing::error;
21
22
use crate::redis_utils::aggregate_types::RedisCursorData;
23
use crate::redis_utils::ft_cursor_read::ft_cursor_read;
24
25
#[derive(Debug)]
26
pub(crate) struct FtAggregateCursor {
27
    pub count: u64,
28
    pub max_idle: u64,
29
}
30
31
#[derive(Debug)]
32
pub(crate) struct FtAggregateOptions {
33
    pub load: Vec<String>,
34
    pub cursor: FtAggregateCursor,
35
    pub sort_by: Vec<String>,
36
    /// Sort direction for every key in `sort_by`.
37
    pub sort_desc: bool,
38
}
39
40
/// Per-query `FT.AGGREGATE` timeout in milliseconds.
41
///
42
/// `RediSearch`'s module default (≈500 ms) is far too tight for the
43
/// scheduler's awaited-action index under any meaningful load: queries
44
/// time out, `NativeLink` surfaces them as parse errors, and the dedup
45
/// lookup fails. When dedup fails the scheduler creates a duplicate
46
/// operation for an action that is already in flight — observed as
47
/// "two same actions running on different PRs" with each running the
48
/// full `max_action_executing_timeout_s` window before completing. Pass an
49
/// explicit value generous enough to absorb 1M+ document scans on a
50
/// busy `RediSearch` instance.
51
const FT_AGGREGATE_TIMEOUT_MS: u64 = 10_000;
52
53
/// The bound on a sorted aggregate. `RediSearch` sorts only this many rows
54
/// and returns no more, and without it the bound is ten.
55
const FT_AGGREGATE_SORT_MAX: u64 = 1_000_000;
56
57
/// Calls `FT.AGGREGATE` in redis. redis-rs does not properly support this command
58
/// so we have to manually handle it.
59
141
pub(crate) async fn ft_aggregate<C>(
60
141
    mut connection_manager: C,
61
141
    index: String,
62
141
    query: String,
63
141
    options: FtAggregateOptions,
64
141
) -> Result<impl Stream<Item = Result<Value, RedisError>> + Send, RedisError>
65
141
where
66
141
    C: ConnectionLike + Send,
67
141
{
68
    struct State<C: ConnectionLike> {
69
        connection_manager: C,
70
        index: String,
71
        data: RedisCursorData,
72
    }
73
74
141
    let mut cmd = redis::cmd("FT.AGGREGATE");
75
141
    let mut ft_aggregate_cmd = cmd
76
141
        .arg(&index)
77
141
        .arg(&query)
78
141
        .arg("TIMEOUT")
79
141
        .arg(FT_AGGREGATE_TIMEOUT_MS)
80
141
        .arg("LOAD")
81
141
        .arg(options.load.len())
82
141
        .arg(&options.load)
83
141
        .arg("WITHCURSOR")
84
141
        .arg("COUNT")
85
141
        .arg(options.cursor.count)
86
141
        .arg("MAXIDLE")
87
141
        .arg(options.cursor.max_idle)
88
141
        .arg("SORTBY")
89
141
        .arg(options.sort_by.len() * 2);
90
141
    let direction = if options.sort_desc { 
"DESC"42
} else {
"ASC"99
};
91
141
    for 
key56
in &options.sort_by {
92
56
        ft_aggregate_cmd = ft_aggregate_cmd.arg(key).arg(direction);
93
56
    }
94
141
    if !options.sort_by.is_empty() {
95
56
        // A SORTBY without MAX makes RediSearch return ten rows, whatever
96
56
        // the cursor's COUNT says: with 300 actions queued the scheduler
97
56
        // listed 10, the matching pass took 10 per pass, the abandoned
98
56
        // sweep retired 10 a minute, and the provisioner read demand as
99
56
        // 10. MAX is the sort's bound, so it has to cover the whole set.
100
56
        ft_aggregate_cmd = ft_aggregate_cmd.arg("MAX").arg(FT_AGGREGATE_SORT_MAX);
101
85
    }
102
141
    let res = ft_aggregate_cmd
103
141
        .query_async::<Value>(&mut connection_manager)
104
141
        .await;
105
141
    let 
data134
= match res {
106
134
        Ok(d) => d,
107
7
        Err(e) => {
108
7
            let all_args: Vec<_> = ft_aggregate_cmd
109
7
                .args_iter()
110
140
                .
map7
(|a| match a {
111
140
                    Arg::Simple(bytes) => match str::from_utf8(bytes) {
112
140
                        Ok(s) => s.to_string(),
113
0
                        Err(_) => format!("{bytes:?}"),
114
                    },
115
0
                    other => {
116
0
                        format!("{other:?}")
117
                    }
118
140
                })
119
7
                .collect();
120
7
            error!(
121
                ?e,
122
                index,
123
                ?query,
124
                ?options,
125
                ?all_args,
126
                "Error calling ft.aggregate"
127
            );
128
7
            return Err(e);
129
        }
130
    };
131
132
134
    let state = State {
133
134
        connection_manager,
134
134
        index,
135
134
        data: data.try_into()
?0
,
136
    };
137
138
134
    Ok(futures::stream::unfold(
139
134
        Some(state),
140
276
        move |maybe_state| async move {
141
276
            let mut state = maybe_state
?0
;
142
            loop {
143
278
                if let Some(
map147
) = state.data.data.pop_front() {
144
147
                    return Some((Ok(map), Some(state)));
145
131
                }
146
131
                if state.data.cursor == 0 {
147
129
                    return None;
148
2
                }
149
2
                let data_res = ft_cursor_read(
150
2
                    &mut state.connection_manager,
151
2
                    state.index.clone(),
152
2
                    state.data.cursor,
153
2
                )
154
2
                .await;
155
2
                state.data = match data_res {
156
2
                    Ok(data) => data,
157
0
                    Err(err) => return Some((Err(err), None)),
158
                };
159
            }
160
552
        },
161
    ))
162
141
}
163
164
133
pub(crate) fn resp2_data_parse(
165
133
    output: &mut RedisCursorData,
166
133
    results_array: &[Value],
167
133
) -> Result<(), RedisError> {
168
133
    let mut results_iter = results_array.iter();
169
133
    match results_iter.next() {
170
133
        Some(Value::Int(t)) => {
171
133
            output.total = *t;
172
133
        }
173
0
        Some(other) => {
174
0
            error!(?other, "Non-int for first value in ft.aggregate");
175
0
            return Err(RedisError::from((
176
0
                ErrorKind::Parse,
177
0
                "Non int for aggregate total",
178
0
                format!("{other:?}"),
179
0
            )));
180
        }
181
        None => {
182
0
            error!("No items in results array for ft.aggregate!");
183
0
            return Err(RedisError::from((
184
0
                ErrorKind::Parse,
185
0
                "No items in results array for ft.aggregate",
186
0
            )));
187
        }
188
    }
189
190
143
    for item in 
results_iter133
{
191
143
        match item {
192
143
            Value::Array(items) if items.len() % 2 == 0 => {}
193
0
            other => {
194
0
                error!(
195
                    ?other,
196
                    "Expected an array with an even number of items, didn't get it for aggregate value"
197
                );
198
0
                return Err(RedisError::from((
199
0
                    ErrorKind::Parse,
200
0
                    "Expected an array with an even number of items, didn't get it for aggregate value",
201
0
                    format!("{other:?}"),
202
0
                )));
203
            }
204
        }
205
206
143
        output.data.push_back(item.clone());
207
    }
208
133
    Ok(())
209
133
}
210
211
3
pub(crate) fn resp3_data_parse(
212
3
    output: &mut RedisCursorData,
213
3
    results_map: &Vec<(Value, Value)>,
214
3
) -> Result<(), RedisError> {
215
11
    for (raw_key, value) in 
results_map3
{
216
11
        let Value::SimpleString(key) = raw_key else {
217
0
            return Err(RedisError::from((
218
0
                ErrorKind::Parse,
219
0
                "Expected SimpleString keys",
220
0
                format!("{raw_key:?}"),
221
0
            )));
222
        };
223
11
        match key.as_str() {
224
11
            "attributes" => {
225
3
                let Value::Array(attributes) = value else {
226
0
                    return Err(RedisError::from((
227
0
                        ErrorKind::Parse,
228
0
                        "Expected array for attributes",
229
0
                        format!("{value:?}"),
230
0
                    )));
231
                };
232
3
                if !attributes.is_empty() {
233
0
                    return Err(RedisError::from((
234
0
                        ErrorKind::Parse,
235
0
                        "Expected empty attributes",
236
0
                        format!("{attributes:?}"),
237
0
                    )));
238
3
                }
239
            }
240
8
            "format" => {
241
3
                let Value::SimpleString(format) = value else {
242
0
                    return Err(RedisError::from((
243
0
                        ErrorKind::Parse,
244
0
                        "Expected SimpleString for format",
245
0
                        format!("{value:?}"),
246
0
                    )));
247
                };
248
3
                if format.as_str() != "STRING" {
249
0
                    return Err(RedisError::from((
250
0
                        ErrorKind::Parse,
251
0
                        "Expected STRING format",
252
0
                        format.clone(),
253
0
                    )));
254
3
                }
255
            }
256
5
            "results" => {
257
3
                let Value::Array(values) = value else {
258
0
                    return Err(RedisError::from((
259
0
                        ErrorKind::Parse,
260
0
                        "Expected Array for results",
261
0
                        format!("{value:?}"),
262
0
                    )));
263
                };
264
5
                for raw_value in 
values3
{
265
5
                    let Value::Map(value) = raw_value else {
266
0
                        return Err(RedisError::from((
267
0
                            ErrorKind::Parse,
268
0
                            "Expected list of maps in result",
269
0
                            format!("{raw_value:?}"),
270
0
                        )));
271
                    };
272
10
                    for (raw_map_key, raw_map_value) in 
value5
{
273
10
                        let Value::SimpleString(map_key) = raw_map_key else {
274
0
                            return Err(RedisError::from((
275
0
                                ErrorKind::Parse,
276
0
                                "Expected SimpleString keys for result maps",
277
0
                                format!("{raw_key:?}"),
278
0
                            )));
279
                        };
280
10
                        match map_key.as_str() {
281
10
                            "extra_attributes" => {
282
5
                                let 
extra_attributes_values4
= match raw_map_value {
283
4
                                    Value::Map(extra_attributes_values) => extra_attributes_values,
284
                                    // A document that expired or was deleted between
285
                                    // the search phase and the load phase comes back
286
                                    // as a row with Nil attributes. Under load this is
287
                                    // routine — completed awaited-action records expire
288
                                    // constantly — so drop the row instead of failing
289
                                    // the whole aggregate. Failing here surfaced to
290
                                    // clients as `INVALID_ARGUMENT`, which Bazel treats
291
                                    // as permanent, so a single expiry race killed the
292
                                    // build.
293
1
                                    Value::Nil => continue,
294
0
                                    other => {
295
0
                                        return Err(RedisError::from((
296
0
                                            ErrorKind::Parse,
297
0
                                            "Expected Map for extra_attributes",
298
0
                                            format!("{other:?}"),
299
0
                                        )));
300
                                    }
301
                                };
302
4
                                let mut output_array = vec![];
303
8
                                for (e_key, e_value) in 
extra_attributes_values4
{
304
8
                                    output_array.push(e_key.clone());
305
8
                                    output_array.push(e_value.clone());
306
8
                                }
307
4
                                output.data.push_back(Value::Array(output_array));
308
                            }
309
5
                            "values" => {
310
5
                                let Value::Array(values_values) = raw_map_value else {
311
0
                                    return Err(RedisError::from((
312
0
                                        ErrorKind::Parse,
313
0
                                        "Expected Array for values",
314
0
                                        format!("{raw_map_value:?}"),
315
0
                                    )));
316
                                };
317
5
                                if !values_values.is_empty() {
318
0
                                    return Err(RedisError::from((
319
0
                                        ErrorKind::Parse,
320
0
                                        "Expected empty values (all in extra_attributes)",
321
0
                                        format!("{values_values:?}"),
322
0
                                    )));
323
5
                                }
324
                            }
325
                            _ => {
326
0
                                return Err(RedisError::from((
327
0
                                    ErrorKind::Parse,
328
0
                                    "Unknown result map key",
329
0
                                    format!("{map_key:?}"),
330
0
                                )));
331
                            }
332
                        }
333
                    }
334
                }
335
            }
336
2
            "total_results" => {
337
1
                let Value::Int(total) = value else {
338
0
                    return Err(RedisError::from((
339
0
                        ErrorKind::Parse,
340
0
                        "Expected int for total_results",
341
0
                        format!("{value:?}"),
342
0
                    )));
343
                };
344
1
                output.total = *total;
345
            }
346
1
            "warning" => {
347
1
                let Value::Array(warnings) = value else {
348
0
                    return Err(RedisError::from((
349
0
                        ErrorKind::Parse,
350
0
                        "Expected Array for warning",
351
0
                        format!("{value:?}"),
352
0
                    )));
353
                };
354
1
                if !warnings.is_empty() {
355
0
                    return Err(RedisError::from((
356
0
                        ErrorKind::Parse,
357
0
                        "Expected empty warnings",
358
0
                        format!("{warnings:?}"),
359
0
                    )));
360
1
                }
361
            }
362
            _ => {
363
0
                return Err(RedisError::from((
364
0
                    ErrorKind::Parse,
365
0
                    "Unexpected key in ft.aggregate",
366
0
                    format!("{key} => {value:?}"),
367
0
                )));
368
            }
369
        }
370
    }
371
3
    Ok(())
372
3
}
373
374
impl TryFrom<Value> for RedisCursorData {
375
    type Error = RedisError;
376
134
    fn try_from(raw_value: Value) -> Result<Self, RedisError> {
377
134
        let Value::Array(value) = raw_value else {
378
0
            error!(
379
                ?raw_value,
380
                "Bad data in ft.aggregate, expected array at top-level"
381
            );
382
0
            return Err(RedisError::from((ErrorKind::Parse, "Expected array")));
383
        };
384
134
        if value.len() < 2 {
385
0
            return Err(RedisError::from((
386
0
                ErrorKind::Parse,
387
0
                "Expected at least 2 elements",
388
0
            )));
389
134
        }
390
134
        let mut output = Self::default();
391
134
        let mut value = value.into_iter();
392
134
        match value.next().unwrap() {
393
132
            Value::Array(d) => resp2_data_parse(&mut output, &d)
?0
,
394
2
            Value::Map(d) => resp3_data_parse(&mut output, &d)
?0
,
395
0
            other => {
396
0
                error!(
397
                    ?other,
398
                    "Bad data in ft.aggregate, expected array for results"
399
                );
400
0
                return Err(RedisError::from((
401
0
                    ErrorKind::Parse,
402
0
                    "Non map item",
403
0
                    format!("{other:?}"),
404
0
                )));
405
            }
406
        }
407
134
        let Value::Int(cursor) = value.next().unwrap() else {
408
0
            return Err(RedisError::from((
409
0
                ErrorKind::Parse,
410
0
                "Expected integer as last element",
411
0
            )));
412
        };
413
134
        output.cursor = cursor as u64;
414
134
        Ok(output)
415
134
    }
416
}