/build/source/nativelink-config/src/backcompat.rs
Line | Count | Source |
1 | | // Copyright 2026 Trace Machina, Inc. All rights reserved. |
2 | | // |
3 | | // Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // See LICENSE file for details |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | use std::collections::HashMap; |
16 | | |
17 | | use serde::{Deserialize, Deserializer, Serialize}; |
18 | | use tracing::warn; |
19 | | |
20 | | use crate::cas_server::{ByteStreamConfig, OldByteStreamConfig, WithInstanceName}; |
21 | | |
22 | | #[derive(Debug, Deserialize)] |
23 | | #[serde(untagged)] |
24 | | enum WithInstanceNameBackCompat<T> { |
25 | | Map(HashMap<String, T>), |
26 | | Vec(Vec<WithInstanceName<T>>), |
27 | | } |
28 | | |
29 | 14 | fn deprecated(old_map: &String, new_map: &String) { |
30 | 14 | warn!( |
31 | | r"WARNING: Using deprecated map format for services. Please migrate to the new array format: |
32 | | // Old: |
33 | | {} |
34 | | // New: |
35 | | {} |
36 | | ", |
37 | | old_map, new_map |
38 | | ); |
39 | 14 | } |
40 | | |
41 | | /// Use `#[serde(default, deserialize_with = "backcompat::opt_vec_with_instance_name")]` for backwards |
42 | | /// compatibility with map-based access. A deprecation warning will be written to stderr if the |
43 | | /// old format is used. |
44 | 71 | pub(crate) fn opt_vec_with_instance_name<'de, D, T>( |
45 | 71 | deserializer: D, |
46 | 71 | ) -> Result<Option<Vec<WithInstanceName<T>>>, D::Error> |
47 | 71 | where |
48 | 71 | D: Deserializer<'de>, |
49 | 71 | T: Deserialize<'de> + Serialize, |
50 | | { |
51 | 71 | let Some(back_compat) = Option::deserialize(deserializer)?0 else { |
52 | 0 | return Ok(None); |
53 | | }; |
54 | | |
55 | 71 | match back_compat { |
56 | 5 | WithInstanceNameBackCompat::Map(map) => { |
57 | | // TODO(palfrey): ideally this would be serde_json5::to_string_pretty but that doesn't exist |
58 | | // JSON is close enough to be workable for now |
59 | 5 | let serde_map = serde_json::to_string_pretty(&map).expect("valid map"); |
60 | 5 | let vec: Vec<WithInstanceName<T>> = map |
61 | 5 | .into_iter() |
62 | 5 | .map(|(instance_name, config)| WithInstanceName { |
63 | 6 | instance_name, |
64 | 6 | config, |
65 | 6 | }) |
66 | 5 | .collect(); |
67 | 5 | deprecated( |
68 | 5 | &serde_map, |
69 | | // TODO(palfrey): ideally this would be serde_json5::to_string_pretty but that doesn't exist |
70 | | // JSON is close enough to be workable for now |
71 | 5 | &serde_json::to_string_pretty(&vec).expect("valid new map"), |
72 | | ); |
73 | 5 | Ok(Some(vec)) |
74 | | } |
75 | 66 | WithInstanceNameBackCompat::Vec(vec) => Ok(Some(vec)), |
76 | | } |
77 | 71 | } |
78 | | |
79 | | #[derive(Debug, Deserialize)] |
80 | | #[serde(untagged)] |
81 | | enum ByteStreamKind { |
82 | | New(Vec<WithInstanceName<ByteStreamConfig>>), |
83 | | Old(OldByteStreamConfig), |
84 | | } |
85 | | |
86 | | /// Use `#[serde(default, deserialize_with = "backcompat::opt_bytestream")]` for backwards |
87 | | /// compatibility with older bytestream config . A deprecation warning will be written to stderr if the |
88 | | /// old format is used. |
89 | 20 | pub(crate) fn opt_bytestream<'de, D>( |
90 | 20 | deserializer: D, |
91 | 20 | ) -> Result<Option<Vec<WithInstanceName<ByteStreamConfig>>>, D::Error> |
92 | 20 | where |
93 | 20 | D: Deserializer<'de>, |
94 | | { |
95 | 20 | let Some(back_compat) = Option::deserialize(deserializer)?0 else { |
96 | 0 | return Ok(None); |
97 | | }; |
98 | | |
99 | 20 | match back_compat { |
100 | 9 | ByteStreamKind::Old(old_config) => { |
101 | 9 | if old_config.max_decoding_message_size != 0 { |
102 | 0 | warn!( |
103 | | "WARNING: max_decoding_message_size is ignored on Bytestream now. Please set on the HTTP listener instead" |
104 | | ); |
105 | 9 | } |
106 | | // TODO(palfrey): ideally this would be serde_json5::to_string_pretty but that doesn't exist |
107 | | // JSON is close enough to be workable for now |
108 | 9 | let serde_map = serde_json::to_string_pretty(&old_config).expect("valid map"); |
109 | 9 | let names = old_config.cas_stores; |
110 | 9 | let vec: Vec<WithInstanceName<_>> = names |
111 | 9 | .iter() |
112 | 9 | .map(|(instance_name, cas_store)| WithInstanceName { |
113 | 9 | instance_name: instance_name.clone(), |
114 | 9 | config: ByteStreamConfig { |
115 | 9 | cas_store: cas_store.clone(), |
116 | 9 | max_bytes_per_stream: old_config.max_bytes_per_stream, |
117 | 9 | persist_stream_on_disconnect_timeout_s: old_config |
118 | 9 | .persist_stream_on_disconnect_timeout_s, |
119 | 9 | recover_unknown_uuid_resumes: true, |
120 | 9 | }, |
121 | 9 | }) |
122 | 9 | .collect(); |
123 | 9 | deprecated( |
124 | 9 | &serde_map, |
125 | | // TODO(palfrey): ideally this would be serde_json5::to_string_pretty but that doesn't exist |
126 | | // JSON is close enough to be workable for now |
127 | 9 | &serde_json::to_string_pretty(&vec).expect("valid new map"), |
128 | | ); |
129 | 9 | Ok(Some(vec)) |
130 | | } |
131 | 11 | ByteStreamKind::New(vec) => Ok(Some(vec)), |
132 | | } |
133 | 20 | } |
134 | | |
135 | | #[cfg(test)] |
136 | | mod tests { |
137 | | use serde_json::json; |
138 | | use tracing_test::traced_test; |
139 | | |
140 | | use super::*; |
141 | | |
142 | | #[derive(Debug, Deserialize, Serialize, PartialEq)] |
143 | | struct PartialConfig { |
144 | | store: String, |
145 | | } |
146 | | |
147 | | #[derive(Debug, Deserialize, Serialize, PartialEq)] |
148 | | struct FullConfig { |
149 | | #[serde(default, deserialize_with = "opt_vec_with_instance_name")] |
150 | | cas: Option<Vec<WithInstanceName<PartialConfig>>>, |
151 | | } |
152 | | |
153 | | #[test] |
154 | | #[traced_test] |
155 | 1 | fn test_configs_deserialization() { |
156 | 1 | let old_format = json!({ |
157 | 1 | "cas": { |
158 | 1 | "foo": { "store": "foo_store" }, |
159 | 1 | "bar": { "store": "bar_store" } |
160 | | } |
161 | | }); |
162 | | |
163 | 1 | let new_format = json!({ |
164 | 1 | "cas": [ |
165 | | { |
166 | 1 | "instance_name": "foo", |
167 | 1 | "store": "foo_store" |
168 | | }, |
169 | | { |
170 | 1 | "instance_name": "bar", |
171 | 1 | "store": "bar_store" |
172 | | } |
173 | | ] |
174 | | }); |
175 | | |
176 | 1 | let mut old_format: FullConfig = serde_json::from_value(old_format).unwrap(); |
177 | 1 | let mut new_format: FullConfig = serde_json::from_value(new_format).unwrap(); |
178 | | |
179 | | // Ensure deterministic ordering. |
180 | 1 | if let Some(vec) = old_format.cas.as_mut() { |
181 | 1 | vec.sort_by(|a, b| a.instance_name.cmp(&b.instance_name)); |
182 | 0 | } |
183 | 1 | if let Some(vec) = new_format.cas.as_mut() { |
184 | 1 | vec.sort_by(|a, b| a.instance_name.cmp(&b.instance_name)); |
185 | 0 | } |
186 | | |
187 | 1 | assert_eq!(old_format, new_format); |
188 | | |
189 | 1 | logs_assert(|lines: &[&str]| { |
190 | 1 | if lines.len() != 1 { |
191 | 0 | return Err(format!("Expected 1 log line, got: {lines:?}")); |
192 | 1 | } |
193 | 1 | let line = lines[0]; |
194 | | // TODO(palfrey): we should be checking the whole thing, but tracing-test is broken with multi-line items |
195 | | // See https://github.com/dbrgn/tracing-test/issues/48 |
196 | 1 | assert!(line.ends_with("WARNING: Using deprecated map format for services. Please migrate to the new array format:")); |
197 | 1 | Ok(()) |
198 | 1 | }); |
199 | 1 | } |
200 | | |
201 | | #[test] |
202 | 1 | fn test_deserialize_none() { |
203 | 1 | let json = json!({}); |
204 | | |
205 | 1 | let value: FullConfig = serde_json::from_value(json).unwrap(); |
206 | 1 | assert_eq!(value.cas, None); |
207 | 1 | } |
208 | | |
209 | | #[derive(Debug, Deserialize, Serialize, PartialEq)] |
210 | | struct FullBytestreamConfig { |
211 | | #[serde(default, deserialize_with = "opt_bytestream")] |
212 | | pub bytestream: Option<Vec<WithInstanceName<ByteStreamConfig>>>, |
213 | | } |
214 | | |
215 | | #[test] |
216 | | #[traced_test] |
217 | 1 | fn test_bytestream_old_config() { |
218 | 1 | let old_format = json!({ |
219 | 1 | "bytestream": { |
220 | 1 | "cas_stores": { |
221 | 1 | "": "WORKER_FAST_SLOW_STORE" |
222 | | }} |
223 | | }); |
224 | | |
225 | 1 | let new_format = json!({ |
226 | 1 | "bytestream": [ |
227 | | { |
228 | 1 | "cas_store": "WORKER_FAST_SLOW_STORE", |
229 | | }, |
230 | | ], |
231 | | }); |
232 | | |
233 | 1 | let old_format: FullBytestreamConfig = serde_json::from_value(old_format).unwrap(); |
234 | 1 | let new_format: FullBytestreamConfig = serde_json::from_value(new_format).unwrap(); |
235 | | |
236 | 1 | assert_eq!(old_format, new_format); |
237 | | |
238 | 1 | logs_assert(|lines: &[&str]| { |
239 | 1 | if lines.len() != 1 { |
240 | 0 | return Err(format!("Expected 1 log line, got: {lines:?}")); |
241 | 1 | } |
242 | 1 | let line = lines[0]; |
243 | | // TODO(palfrey): we should be checking the whole thing, but tracing-test is broken with multi-line items |
244 | | // See https://github.com/dbrgn/tracing-test/issues/48 |
245 | 1 | assert!(line.ends_with("WARNING: Using deprecated map format for services. Please migrate to the new array format:")); |
246 | 1 | Ok(()) |
247 | 1 | }); |
248 | 1 | } |
249 | | } |