Coverage Report

Created: 2026-11-13 23:29

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/build/source/nativelink-config/src/backcompat.rs
Line
Count
Source
1
// Copyright 2026 Trace Machina, Inc. All rights reserved.
2
//
3
// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//    See LICENSE file for details
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
use std::collections::HashMap;
16
17
use serde::{Deserialize, Deserializer, Serialize};
18
use tracing::warn;
19
20
use crate::cas_server::{ByteStreamConfig, OldByteStreamConfig, WithInstanceName};
21
22
#[derive(Debug, Deserialize)]
23
#[serde(untagged)]
24
enum WithInstanceNameBackCompat<T> {
25
    Map(HashMap<String, T>),
26
    Vec(Vec<WithInstanceName<T>>),
27
}
28
29
14
fn deprecated(old_map: &String, new_map: &String) {
30
14
    warn!(
31
        r"WARNING: Using deprecated map format for services. Please migrate to the new array format:
32
// Old:
33
{}
34
// New:
35
{}
36
",
37
        old_map, new_map
38
    );
39
14
}
40
41
/// Use `#[serde(default, deserialize_with = "backcompat::opt_vec_with_instance_name")]` for backwards
42
/// compatibility with map-based access. A deprecation warning will be written to stderr if the
43
/// old format is used.
44
71
pub(crate) fn opt_vec_with_instance_name<'de, D, T>(
45
71
    deserializer: D,
46
71
) -> Result<Option<Vec<WithInstanceName<T>>>, D::Error>
47
71
where
48
71
    D: Deserializer<'de>,
49
71
    T: Deserialize<'de> + Serialize,
50
{
51
71
    let Some(back_compat) = Option::deserialize(deserializer)
?0
else {
52
0
        return Ok(None);
53
    };
54
55
71
    match back_compat {
56
5
        WithInstanceNameBackCompat::Map(map) => {
57
            // TODO(palfrey): ideally this would be serde_json5::to_string_pretty but that doesn't exist
58
            // JSON is close enough to be workable for now
59
5
            let serde_map = serde_json::to_string_pretty(&map).expect("valid map");
60
5
            let vec: Vec<WithInstanceName<T>> = map
61
5
                .into_iter()
62
5
                .map(|(instance_name, config)| WithInstanceName {
63
6
                    instance_name,
64
6
                    config,
65
6
                })
66
5
                .collect();
67
5
            deprecated(
68
5
                &serde_map,
69
                // TODO(palfrey): ideally this would be serde_json5::to_string_pretty but that doesn't exist
70
                // JSON is close enough to be workable for now
71
5
                &serde_json::to_string_pretty(&vec).expect("valid new map"),
72
            );
73
5
            Ok(Some(vec))
74
        }
75
66
        WithInstanceNameBackCompat::Vec(vec) => Ok(Some(vec)),
76
    }
77
71
}
78
79
#[derive(Debug, Deserialize)]
80
#[serde(untagged)]
81
enum ByteStreamKind {
82
    New(Vec<WithInstanceName<ByteStreamConfig>>),
83
    Old(OldByteStreamConfig),
84
}
85
86
/// Use `#[serde(default, deserialize_with = "backcompat::opt_bytestream")]` for backwards
87
/// compatibility with older bytestream config . A deprecation warning will be written to stderr if the
88
/// old format is used.
89
20
pub(crate) fn opt_bytestream<'de, D>(
90
20
    deserializer: D,
91
20
) -> Result<Option<Vec<WithInstanceName<ByteStreamConfig>>>, D::Error>
92
20
where
93
20
    D: Deserializer<'de>,
94
{
95
20
    let Some(back_compat) = Option::deserialize(deserializer)
?0
else {
96
0
        return Ok(None);
97
    };
98
99
20
    match back_compat {
100
9
        ByteStreamKind::Old(old_config) => {
101
9
            if old_config.max_decoding_message_size != 0 {
102
0
                warn!(
103
                    "WARNING: max_decoding_message_size is ignored on Bytestream now. Please set on the HTTP listener instead"
104
                );
105
9
            }
106
            // TODO(palfrey): ideally this would be serde_json5::to_string_pretty but that doesn't exist
107
            // JSON is close enough to be workable for now
108
9
            let serde_map = serde_json::to_string_pretty(&old_config).expect("valid map");
109
9
            let names = old_config.cas_stores;
110
9
            let vec: Vec<WithInstanceName<_>> = names
111
9
                .iter()
112
9
                .map(|(instance_name, cas_store)| WithInstanceName {
113
9
                    instance_name: instance_name.clone(),
114
9
                    config: ByteStreamConfig {
115
9
                        cas_store: cas_store.clone(),
116
9
                        max_bytes_per_stream: old_config.max_bytes_per_stream,
117
9
                        persist_stream_on_disconnect_timeout_s: old_config
118
9
                            .persist_stream_on_disconnect_timeout_s,
119
9
                        recover_unknown_uuid_resumes: true,
120
9
                    },
121
9
                })
122
9
                .collect();
123
9
            deprecated(
124
9
                &serde_map,
125
                // TODO(palfrey): ideally this would be serde_json5::to_string_pretty but that doesn't exist
126
                // JSON is close enough to be workable for now
127
9
                &serde_json::to_string_pretty(&vec).expect("valid new map"),
128
            );
129
9
            Ok(Some(vec))
130
        }
131
11
        ByteStreamKind::New(vec) => Ok(Some(vec)),
132
    }
133
20
}
134
135
#[cfg(test)]
136
mod tests {
137
    use serde_json::json;
138
    use tracing_test::traced_test;
139
140
    use super::*;
141
142
    #[derive(Debug, Deserialize, Serialize, PartialEq)]
143
    struct PartialConfig {
144
        store: String,
145
    }
146
147
    #[derive(Debug, Deserialize, Serialize, PartialEq)]
148
    struct FullConfig {
149
        #[serde(default, deserialize_with = "opt_vec_with_instance_name")]
150
        cas: Option<Vec<WithInstanceName<PartialConfig>>>,
151
    }
152
153
    #[test]
154
    #[traced_test]
155
1
    fn test_configs_deserialization() {
156
1
        let old_format = json!({
157
1
            "cas": {
158
1
                "foo": { "store": "foo_store" },
159
1
                "bar": { "store": "bar_store" }
160
            }
161
        });
162
163
1
        let new_format = json!({
164
1
            "cas": [
165
                {
166
1
                    "instance_name": "foo",
167
1
                    "store": "foo_store"
168
                },
169
                {
170
1
                    "instance_name": "bar",
171
1
                    "store": "bar_store"
172
                }
173
            ]
174
        });
175
176
1
        let mut old_format: FullConfig = serde_json::from_value(old_format).unwrap();
177
1
        let mut new_format: FullConfig = serde_json::from_value(new_format).unwrap();
178
179
        // Ensure deterministic ordering.
180
1
        if let Some(vec) = old_format.cas.as_mut() {
181
1
            vec.sort_by(|a, b| a.instance_name.cmp(&b.instance_name));
182
0
        }
183
1
        if let Some(vec) = new_format.cas.as_mut() {
184
1
            vec.sort_by(|a, b| a.instance_name.cmp(&b.instance_name));
185
0
        }
186
187
1
        assert_eq!(old_format, new_format);
188
189
1
        logs_assert(|lines: &[&str]| {
190
1
            if lines.len() != 1 {
191
0
                return Err(format!("Expected 1 log line, got: {lines:?}"));
192
1
            }
193
1
            let line = lines[0];
194
            // TODO(palfrey): we should be checking the whole thing, but tracing-test is broken with multi-line items
195
            // See https://github.com/dbrgn/tracing-test/issues/48
196
1
            assert!(line.ends_with("WARNING: Using deprecated map format for services. Please migrate to the new array format:"));
197
1
            Ok(())
198
1
        });
199
1
    }
200
201
    #[test]
202
1
    fn test_deserialize_none() {
203
1
        let json = json!({});
204
205
1
        let value: FullConfig = serde_json::from_value(json).unwrap();
206
1
        assert_eq!(value.cas, None);
207
1
    }
208
209
    #[derive(Debug, Deserialize, Serialize, PartialEq)]
210
    struct FullBytestreamConfig {
211
        #[serde(default, deserialize_with = "opt_bytestream")]
212
        pub bytestream: Option<Vec<WithInstanceName<ByteStreamConfig>>>,
213
    }
214
215
    #[test]
216
    #[traced_test]
217
1
    fn test_bytestream_old_config() {
218
1
        let old_format = json!({
219
1
            "bytestream": {
220
1
                "cas_stores": {
221
1
                    "": "WORKER_FAST_SLOW_STORE"
222
            }}
223
        });
224
225
1
        let new_format = json!({
226
1
            "bytestream": [
227
                {
228
1
                    "cas_store": "WORKER_FAST_SLOW_STORE",
229
                },
230
            ],
231
        });
232
233
1
        let old_format: FullBytestreamConfig = serde_json::from_value(old_format).unwrap();
234
1
        let new_format: FullBytestreamConfig = serde_json::from_value(new_format).unwrap();
235
236
1
        assert_eq!(old_format, new_format);
237
238
1
        logs_assert(|lines: &[&str]| {
239
1
            if lines.len() != 1 {
240
0
                return Err(format!("Expected 1 log line, got: {lines:?}"));
241
1
            }
242
1
            let line = lines[0];
243
            // TODO(palfrey): we should be checking the whole thing, but tracing-test is broken with multi-line items
244
            // See https://github.com/dbrgn/tracing-test/issues/48
245
1
            assert!(line.ends_with("WARNING: Using deprecated map format for services. Please migrate to the new array format:"));
246
1
            Ok(())
247
1
        });
248
1
    }
249
}