/build/source/nativelink-util/src/fs_util.rs
Line | Count | Source |
1 | | // Copyright 2024 The NativeLink Authors. All rights reserved. |
2 | | // |
3 | | // Licensed under the Apache License, Version 2.0 (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // http://www.apache.org/licenses/LICENSE-2.0 |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | use core::future::Future; |
16 | | use core::pin::Pin; |
17 | | use std::path::Path; |
18 | | |
19 | | use nativelink_error::{Code, Error, ResultExt, error_if, make_err}; |
20 | | use tokio::fs; |
21 | | |
22 | | /// Hardlinks an entire directory tree from source to destination. |
23 | | /// This is much faster than copying for large directory structures. |
24 | | /// |
25 | | /// # Arguments |
26 | | /// * `src_dir` - Source directory path (must exist) |
27 | | /// * `dst_dir` - Destination directory path (will be created) |
28 | | /// |
29 | | /// # Returns |
30 | | /// * `Ok(())` on success |
31 | | /// * `Err` if hardlinking fails (e.g., cross-filesystem, unsupported filesystem) |
32 | | /// |
33 | | /// # Platform Support |
34 | | /// - Linux: Full support via `fs::hard_link` |
35 | | /// - macOS: Full support via `fs::hard_link` |
36 | | /// - Windows: Requires NTFS filesystem and appropriate permissions |
37 | | /// |
38 | | /// # Errors |
39 | | /// - Source directory doesn't exist |
40 | | /// - Destination already exists |
41 | | /// - Cross-filesystem hardlinking attempted |
42 | | /// - Filesystem doesn't support hardlinks |
43 | | /// - Permission denied |
44 | 5 | pub async fn hardlink_directory_tree(src_dir: &Path, dst_dir: &Path) -> Result<(), Error>3 {
|
45 | 4 | error_if!( |
46 | 5 | !src_dir.exists(), Branch (46:9): [Folded - Ignored]
Branch (46:9): [True: 1, False: 2]
Branch (46:9): [True: 0, False: 2]
Branch (46:9): [True: 0, False: 0]
|
47 | | "Source directory does not exist: {}", |
48 | 1 | src_dir.display() |
49 | | ); |
50 | | |
51 | 1 | error_if!( |
52 | 4 | dst_dir.exists(), Branch (52:9): [Folded - Ignored]
Branch (52:9): [True: 1, False: 1]
Branch (52:9): [True: 0, False: 2]
Branch (52:9): [True: 0, False: 0]
|
53 | | "Destination directory already exists: {}", |
54 | 1 | dst_dir.display() |
55 | | ); |
56 | | |
57 | | // Create the root destination directory |
58 | 3 | fs::create_dir_all(dst_dir).await.err_tip(|| {0 |
59 | 0 | format!( |
60 | 0 | "Failed to create destination directory: {}", |
61 | 0 | dst_dir.display() |
62 | | ) |
63 | 0 | })?; |
64 | | |
65 | | // Recursively hardlink the directory tree |
66 | 3 | hardlink_directory_tree_recursive(src_dir, dst_dir).await |
67 | 5 | } |
68 | | |
69 | | /// Internal recursive function to hardlink directory contents |
70 | 4 | fn hardlink_directory_tree_recursive<'a>( |
71 | 4 | src: &'a Path, |
72 | 4 | dst: &'a Path, |
73 | 4 | ) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send + 'a>> { |
74 | 4 | Box::pin(async move { |
75 | 4 | let mut entries = fs::read_dir(src) |
76 | 4 | .await |
77 | 4 | .err_tip(|| format!("Failed to read directory: {}"0 , src0 .display0 ()))?0 ; |
78 | | |
79 | 9 | while let Some(entry5 ) = entries Branch (79:19): [True: 2, False: 2]
Branch (79:19): [True: 3, False: 2]
|
80 | 9 | .next_entry() |
81 | 9 | .await |
82 | 9 | .err_tip(|| format!("Failed to get next entry in: {}"0 , src0 .display0 ()))?0 |
83 | | { |
84 | 5 | let entry_path = entry.path(); |
85 | 5 | let file_name = entry.file_name().into_string().map_err(|os_str| {0 |
86 | 0 | make_err!( |
87 | 0 | Code::InvalidArgument, |
88 | | "Invalid UTF-8 in filename: {:?}", |
89 | | os_str |
90 | | ) |
91 | 0 | })?; |
92 | | |
93 | 5 | let dst_path = dst.join(&file_name); |
94 | 5 | let metadata = entry |
95 | 5 | .metadata() |
96 | 5 | .await |
97 | 5 | .err_tip(|| format!("Failed to get metadata for: {}"0 , entry_path.display()0 ))?0 ; |
98 | | |
99 | 5 | if metadata.is_dir() { Branch (99:16): [True: 0, False: 2]
Branch (99:16): [True: 1, False: 2]
|
100 | | // Create subdirectory and recurse |
101 | 1 | fs::create_dir(&dst_path) |
102 | 1 | .await |
103 | 1 | .err_tip(|| format!("Failed to create directory: {}"0 , dst_path.display()0 ))?0 ; |
104 | | |
105 | 1 | hardlink_directory_tree_recursive(&entry_path, &dst_path).await?0 ; |
106 | 4 | } else if metadata.is_file() { Branch (106:23): [True: 2, False: 0]
Branch (106:23): [True: 2, False: 0]
|
107 | | // Hardlink the file |
108 | 4 | fs::hard_link(&entry_path, &dst_path) |
109 | 4 | .await |
110 | 4 | .err_tip(|| {0 |
111 | 0 | format!( |
112 | 0 | "Failed to hardlink {} to {}. This may occur if the source and destination are on different filesystems", |
113 | 0 | entry_path.display(), |
114 | 0 | dst_path.display() |
115 | | ) |
116 | 0 | })?; |
117 | 0 | } else if metadata.is_symlink() { Branch (117:23): [True: 0, False: 0]
Branch (117:23): [True: 0, False: 0]
|
118 | | // Read the symlink target and create a new symlink |
119 | 0 | let target = fs::read_link(&entry_path) |
120 | 0 | .await |
121 | 0 | .err_tip(|| format!("Failed to read symlink: {}", entry_path.display()))?; |
122 | | |
123 | | #[cfg(unix)] |
124 | 0 | fs::symlink(&target, &dst_path) |
125 | 0 | .await |
126 | 0 | .err_tip(|| format!("Failed to create symlink: {}", dst_path.display()))?; |
127 | | |
128 | | #[cfg(windows)] |
129 | | { |
130 | | if target.is_dir() { |
131 | | fs::symlink_dir(&target, &dst_path).await.err_tip(|| { |
132 | | format!("Failed to create directory symlink: {}", dst_path.display()) |
133 | | })?; |
134 | | } else { |
135 | | fs::symlink_file(&target, &dst_path).await.err_tip(|| { |
136 | | format!("Failed to create file symlink: {}", dst_path.display()) |
137 | | })?; |
138 | | } |
139 | | } |
140 | 0 | } |
141 | | } |
142 | | |
143 | 4 | Ok(()) |
144 | 4 | }) |
145 | 4 | } |
146 | | |
147 | | /// Sets a directory tree to read-only recursively. |
148 | | /// This prevents actions from modifying cached directories. |
149 | | /// |
150 | | /// # Arguments |
151 | | /// * `dir` - Directory to make read-only |
152 | | /// |
153 | | /// # Platform Notes |
154 | | /// - Unix: Sets permissions to 0o555 (r-xr-xr-x) |
155 | | /// - Windows: Sets `FILE_ATTRIBUTE_READONLY` |
156 | 2 | pub async fn set_readonly_recursive(dir: &Path) -> Result<(), Error>1 {
|
157 | 2 | error_if!(!dir.exists(), "Directory does not exist: {}", dir0 .display0 ()); Branch (157:15): [Folded - Ignored]
Branch (157:15): [True: 0, False: 1]
Branch (157:15): [True: 0, False: 1]
Branch (157:15): [True: 0, False: 0]
|
158 | | |
159 | 2 | set_readonly_recursive_impl(dir).await |
160 | 2 | } |
161 | | |
162 | 6 | fn set_readonly_recursive_impl<'a>( |
163 | 6 | path: &'a Path, |
164 | 6 | ) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send + 'a>> { |
165 | 6 | Box::pin(async move { |
166 | 6 | let metadata = fs::metadata(path) |
167 | 6 | .await |
168 | 6 | .err_tip(|| format!("Failed to get metadata for: {}"0 , path0 .display0 ()))?0 ; |
169 | | |
170 | 6 | if metadata.is_dir() { Branch (170:12): [True: 1, False: 1]
Branch (170:12): [True: 2, False: 2]
|
171 | 3 | let mut entries = fs::read_dir(path) |
172 | 3 | .await |
173 | 3 | .err_tip(|| format!("Failed to read directory: {}"0 , path0 .display0 ()))?0 ; |
174 | | |
175 | 7 | while let Some(entry4 ) = entries Branch (175:23): [True: 1, False: 1]
Branch (175:23): [True: 3, False: 2]
|
176 | 7 | .next_entry() |
177 | 7 | .await |
178 | 7 | .err_tip(|| format!("Failed to get next entry in: {}"0 , path0 .display0 ()))?0 |
179 | | { |
180 | 4 | set_readonly_recursive_impl(&entry.path()).await?0 ; |
181 | | } |
182 | 3 | } |
183 | | |
184 | | // Set the file/directory to read-only |
185 | | #[cfg(unix)] |
186 | | { |
187 | | use std::os::unix::fs::PermissionsExt; |
188 | 6 | let mut perms = metadata.permissions(); |
189 | | |
190 | | // If it's a directory, set to r-xr-xr-x (555) |
191 | | // If it's a file, set to r--r--r-- (444) |
192 | 6 | let mode = if metadata.is_dir() { 0o5553 } else { 0o4443 }; Branch (192:27): [True: 1, False: 1]
Branch (192:27): [True: 2, False: 2]
|
193 | 6 | perms.set_mode(mode); |
194 | | |
195 | 6 | fs::set_permissions(path, perms) |
196 | 6 | .await |
197 | 6 | .err_tip(|| format!("Failed to set permissions for: {}"0 , path0 .display0 ()))?0 ; |
198 | | } |
199 | | |
200 | | #[cfg(windows)] |
201 | | { |
202 | | let mut perms = metadata.permissions(); |
203 | | perms.set_readonly(true); |
204 | | |
205 | | fs::set_permissions(path, perms) |
206 | | .await |
207 | | .err_tip(|| format!("Failed to set permissions for: {}", path.display()))?; |
208 | | } |
209 | | |
210 | 6 | Ok(()) |
211 | 6 | }) |
212 | 6 | } |
213 | | |
214 | | /// Calculates the total size of a directory tree in bytes. |
215 | | /// Used for cache size tracking and LRU eviction. |
216 | | /// |
217 | | /// # Arguments |
218 | | /// * `dir` - Directory to calculate size for |
219 | | /// |
220 | | /// # Returns |
221 | | /// Total size in bytes, or Error if directory cannot be read |
222 | 2 | pub async fn calculate_directory_size(dir: &Path) -> Result<u64, Error>1 {
|
223 | 2 | error_if!(!dir.exists(), "Directory does not exist: {}", dir0 .display0 ()); Branch (223:15): [Folded - Ignored]
Branch (223:15): [True: 0, False: 1]
Branch (223:15): [True: 0, False: 1]
Branch (223:15): [True: 0, False: 0]
|
224 | | |
225 | 2 | calculate_directory_size_impl(dir).await |
226 | 2 | } |
227 | | |
228 | 6 | fn calculate_directory_size_impl<'a>( |
229 | 6 | path: &'a Path, |
230 | 6 | ) -> Pin<Box<dyn Future<Output = Result<u64, Error>> + Send + 'a>> { |
231 | 6 | Box::pin(async move { |
232 | 6 | let metadata = fs::metadata(path) |
233 | 6 | .await |
234 | 6 | .err_tip(|| format!("Failed to get metadata for: {}"0 , path0 .display0 ()))?0 ; |
235 | | |
236 | 6 | if metadata.is_file() { Branch (236:12): [True: 1, False: 1]
Branch (236:12): [True: 2, False: 2]
|
237 | 3 | return Ok(metadata.len()); |
238 | 3 | } |
239 | | |
240 | 3 | if !metadata.is_dir() { Branch (240:12): [True: 0, False: 1]
Branch (240:12): [True: 0, False: 2]
|
241 | 0 | return Ok(0); |
242 | 3 | } |
243 | | |
244 | 3 | let mut total_size = 0u64; |
245 | 3 | let mut entries = fs::read_dir(path) |
246 | 3 | .await |
247 | 3 | .err_tip(|| format!("Failed to read directory: {}"0 , path0 .display0 ()))?0 ; |
248 | | |
249 | 7 | while let Some(entry4 ) = entries Branch (249:19): [True: 1, False: 1]
Branch (249:19): [True: 3, False: 2]
|
250 | 7 | .next_entry() |
251 | 7 | .await |
252 | 7 | .err_tip(|| format!("Failed to get next entry in: {}"0 , path0 .display0 ()))?0 |
253 | | { |
254 | 4 | total_size += calculate_directory_size_impl(&entry.path()).await?0 ; |
255 | | } |
256 | | |
257 | 3 | Ok(total_size) |
258 | 6 | }) |
259 | 6 | } |
260 | | |
261 | | #[cfg(test)] |
262 | | mod tests { |
263 | | use std::path::PathBuf; |
264 | | |
265 | | use tempfile::TempDir; |
266 | | use tokio::io::AsyncWriteExt; |
267 | | |
268 | | use super::*; |
269 | | |
270 | 4 | async fn create_test_directory() -> Result<(TempDir, PathBuf), Error> { |
271 | 4 | let temp_dir = TempDir::new().err_tip(|| "Failed to create temp directory")?0 ; |
272 | 4 | let test_dir = temp_dir.path().join("test_src"); |
273 | | |
274 | 4 | fs::create_dir(&test_dir).await?0 ; |
275 | | |
276 | | // Create a file |
277 | 4 | let file1 = test_dir.join("file1.txt"); |
278 | 4 | let mut f = fs::File::create(&file1).await?0 ; |
279 | 4 | f.write_all(b"Hello, World!").await?0 ; |
280 | 4 | f.sync_all().await?0 ; |
281 | 4 | drop(f); |
282 | | |
283 | | // Create a subdirectory with a file |
284 | 4 | let subdir = test_dir.join("subdir"); |
285 | 4 | fs::create_dir(&subdir).await?0 ; |
286 | | |
287 | 4 | let file2 = subdir.join("file2.txt"); |
288 | 4 | let mut f = fs::File::create(&file2).await?0 ; |
289 | 4 | f.write_all(b"Nested file").await?0 ; |
290 | 4 | f.sync_all().await?0 ; |
291 | 4 | drop(f); |
292 | | |
293 | 4 | Ok((temp_dir, test_dir)) |
294 | 4 | } |
295 | | |
296 | | #[tokio::test] |
297 | 1 | async fn test_hardlink_directory_tree() -> Result<(), Error> { |
298 | 1 | let (temp_dir, src_dir) = create_test_directory().await?0 ; |
299 | 1 | let dst_dir = temp_dir.path().join("test_dst"); |
300 | | |
301 | | // Hardlink the directory |
302 | 1 | hardlink_directory_tree(&src_dir, &dst_dir).await?0 ; |
303 | | |
304 | | // Verify structure |
305 | 1 | assert!(dst_dir.join("file1.txt").exists()); |
306 | 1 | assert!(dst_dir.join("subdir").is_dir()); |
307 | 1 | assert!(dst_dir.join("subdir/file2.txt").exists()); |
308 | | |
309 | | // Verify contents |
310 | 1 | let content1 = fs::read_to_string(dst_dir.join("file1.txt")).await?0 ; |
311 | 1 | assert_eq!(content1, "Hello, World!"); |
312 | | |
313 | 1 | let content2 = fs::read_to_string(dst_dir.join("subdir/file2.txt")).await?0 ; |
314 | 1 | assert_eq!(content2, "Nested file"); |
315 | | |
316 | | // Verify files are hardlinked (same inode on Unix) |
317 | 1 | #[cfg(unix)] |
318 | 1 | { |
319 | 1 | use std::os::unix::fs::MetadataExt; |
320 | 1 | let src_meta = fs::metadata(src_dir.join("file1.txt")).await?0 ; |
321 | 1 | let dst_meta = fs::metadata(dst_dir.join("file1.txt")).await?0 ; |
322 | 1 | assert_eq!( |
323 | 1 | src_meta.ino(), |
324 | 1 | dst_meta.ino(), |
325 | 1 | "Files should have same inode (hardlinked)"0 |
326 | 1 | ); |
327 | 1 | } |
328 | 1 | |
329 | 1 | Ok(()) |
330 | 1 | } |
331 | | |
332 | | #[tokio::test] |
333 | 1 | async fn test_set_readonly_recursive() -> Result<(), Error> { |
334 | 1 | let (_temp_dir, test_dir) = create_test_directory().await?0 ; |
335 | | |
336 | 1 | set_readonly_recursive(&test_dir).await?0 ; |
337 | | |
338 | | // Verify files are read-only |
339 | 1 | let metadata = fs::metadata(test_dir.join("file1.txt")).await?0 ; |
340 | 1 | assert!(metadata.permissions().readonly()); |
341 | | |
342 | 1 | let metadata = fs::metadata(test_dir.join("subdir/file2.txt")).await?0 ; |
343 | 1 | assert!(metadata.permissions().readonly()); |
344 | | |
345 | 2 | Ok(()) |
346 | 1 | } |
347 | | |
348 | | #[tokio::test] |
349 | 1 | async fn test_calculate_directory_size() -> Result<(), Error> { |
350 | 1 | let (_temp_dir, test_dir) = create_test_directory().await?0 ; |
351 | | |
352 | 1 | let size = calculate_directory_size(&test_dir).await?0 ; |
353 | | |
354 | | // "Hello, World!" = 13 bytes |
355 | | // "Nested file" = 11 bytes |
356 | | // Total = 24 bytes |
357 | 1 | assert_eq!(size, 24); |
358 | | |
359 | 2 | Ok(()) |
360 | 1 | } |
361 | | |
362 | | #[tokio::test] |
363 | 1 | async fn test_hardlink_nonexistent_source() { |
364 | 1 | let temp_dir = TempDir::new().unwrap(); |
365 | 1 | let src = temp_dir.path().join("nonexistent"); |
366 | 1 | let dst = temp_dir.path().join("dest"); |
367 | | |
368 | 1 | let result = hardlink_directory_tree(&src, &dst).await; |
369 | 1 | assert!(result.is_err()); |
370 | 1 | } |
371 | | |
372 | | #[tokio::test] |
373 | 1 | async fn test_hardlink_existing_destination() -> Result<(), Error> { |
374 | 1 | let (_temp_dir, src_dir) = create_test_directory().await?0 ; |
375 | 1 | let dst_dir = _temp_dir.path().join("existing"); |
376 | | |
377 | 1 | fs::create_dir(&dst_dir).await?0 ; |
378 | | |
379 | 1 | let result = hardlink_directory_tree(&src_dir, &dst_dir).await; |
380 | 1 | assert!(result.is_err()); |
381 | | |
382 | 2 | Ok(()) |
383 | 1 | } |
384 | | } |