supreme_fs/game_storage/online/
content.rs

1//! Fetching and uploading individual content-addressed blobs.
2
3use crate::game_storage::GameStorage;
4use crate::game_storage::online::multipart::{MultipartForm, SendMultipart};
5use crate::{AssetType, HybridStorageError, OnlineStorageError};
6use sha2::{Digest, Sha256};
7use std::fs;
8use std::io::{self, Read};
9
10/// Verify that `data` hashes to `expected_hash`, returning
11/// [`HybridStorageError::CorruptedDownload`] on mismatch. Shared by every
12/// path that downloads content (single-fetch and batch-fetch) so corruption
13/// is always caught the same way, and so the check can be unit tested
14/// without a live server.
15fn verify_hash(data: &[u8], expected_hash: &str) -> Result<(), HybridStorageError> {
16    let mut hasher = Sha256::new();
17    hasher.update(data);
18    let actual_hash = hex::encode(hasher.finalize());
19    if actual_hash == expected_hash {
20        Ok(())
21    } else {
22        Err(HybridStorageError::CorruptedDownload(
23            expected_hash.to_string(),
24        ))
25    }
26}
27
28/// Decompress a fetched content entry (if its filename is `.xz`-suffixed)
29/// and verify it against the hash encoded in that filename. Returns the
30/// hash and the decompressed bytes on success. Pulled out of
31/// [`GameStorage::fetch_contents_batch`]'s tar-processing loop so the
32/// decompress-then-verify pipeline can be unit tested without a live server.
33fn decode_and_verify_entry(
34    filename: &str,
35    raw: &[u8],
36) -> Result<(String, Vec<u8>), HybridStorageError> {
37    let (hash, is_xz) = match filename.strip_suffix(".xz") {
38        Some(stripped) => (stripped.to_string(), true),
39        None => (filename.to_string(), false),
40    };
41
42    let final_data = if is_xz {
43        let mut decoder = xz2::read::XzDecoder::new(raw);
44        let mut decompressed = Vec::new();
45        decoder.read_to_end(&mut decompressed)?;
46        decompressed
47    } else {
48        raw.to_vec()
49    };
50
51    verify_hash(&final_data, &hash)?;
52    Ok((hash, final_data))
53}
54
55impl GameStorage {
56    pub(crate) fn fetch_online(
57        &self,
58        asset_type: AssetType,
59        filename: &str,
60    ) -> Result<(Vec<u8>, String), OnlineStorageError> {
61        let client = self
62            .client
63            .as_ref()
64            .ok_or_else(|| io::Error::other("Online storage not configured"))?;
65        let api_url = self
66            .api_url
67            .as_ref()
68            .ok_or_else(|| io::Error::other("Online storage not configured"))?;
69
70        let sub_path = match asset_type {
71            AssetType::Content => "contents",
72            AssetType::Profile => "profiles",
73        };
74        let url = format!("{api_url}/{sub_path}/{filename}");
75
76        let response = client.get(&url).call()?;
77        if response.status().is_success() {
78            let actual_filename = response
79                .headers()
80                .get("X-Content-Filename")
81                .and_then(|h| h.to_str().ok())
82                .unwrap_or(filename)
83                .to_string();
84
85            let mut bytes = Vec::new();
86            let mut reader = response.into_body().into_reader();
87            reader.read_to_end(&mut bytes)?;
88            Ok((bytes, actual_filename))
89        } else {
90            Err(OnlineStorageError::NotFound(format!(
91                "URL: {sub_path}/{filename}"
92            )))
93        }
94    }
95
96    pub fn fetch_contents_batch(
97        &self,
98        hashes: &[&str],
99    ) -> Result<Vec<(String, Vec<u8>)>, HybridStorageError> {
100        let client = self
101            .client
102            .as_ref()
103            .ok_or_else(|| io::Error::other("Online storage not configured"))?;
104        let api_url = self
105            .api_url
106            .as_ref()
107            .ok_or_else(|| io::Error::other("Online storage not configured"))?;
108
109        let mut form = MultipartForm::new().text("action", "fetch_contents_batch");
110
111        for hash in hashes {
112            form = form.text("hashes[]", *hash);
113        }
114
115        if let Some(token) = &self.token {
116            form = form.text("api_token", token.clone());
117        }
118
119        let mut request = client.post(api_url);
120        if let Some(ref token) = self.token {
121            request = request.header("X-SSBZ-Token", token);
122        }
123
124        let response = request.send_multipart(form)?;
125        if !response.status().is_success() {
126            let err_msg = self.extract_error_message(response);
127            if err_msg.contains("404") {
128                return Err(HybridStorageError::NotFound(
129                    "None of the requested hashes found".to_string(),
130                ));
131            }
132            return Err(HybridStorageError::UploadFailed(err_msg));
133        }
134
135        let mut tar_bytes = Vec::new();
136        response
137            .into_body()
138            .into_reader()
139            .read_to_end(&mut tar_bytes)?;
140
141        let mut archive = tar::Archive::new(std::io::Cursor::new(tar_bytes));
142        let mut results = Vec::new();
143
144        for entry in archive.entries()? {
145            let mut entry = entry?;
146            let path = entry.path()?.to_path_buf();
147            let filename = path
148                .file_name()
149                .and_then(|s| s.to_str())
150                .unwrap_or_default()
151                .to_string();
152
153            let mut content = Vec::new();
154            entry.read_to_end(&mut content)?;
155
156            let (hash, final_data) = decode_and_verify_entry(&filename, &content)?;
157
158            // Save to cache
159            let save_path = self.contents_path().join(&filename);
160            fs::create_dir_all(
161                save_path
162                    .parent()
163                    .ok_or_else(|| io::Error::other("Save path has no parent directory"))?,
164            )?;
165            fs::write(&save_path, &content)?;
166
167            results.push((hash, final_data));
168        }
169
170        Ok(results)
171    }
172
173    /// Fetch content by hash: the launcher-side local cache
174    /// (`.supreme/contents/<hash>[.xz]`) first, then up to three remote
175    /// attempts (with inter-attempt sleeps) that are saved into the cache,
176    /// xz-decoded and hash-verified.
177    ///
178    /// # Errors
179    ///
180    /// Transport/decoding errors surface after the retries; offline storage
181    /// without the hash cached returns [`HybridStorageError::NotFound`].
182    pub fn get_content(&self, hash: &str) -> Result<Vec<u8>, HybridStorageError> {
183        self.get_content_with_retries(hash, 3)
184    }
185
186    /// [`GameStorage::get_content`] with a **single** remote attempt.
187    ///
188    /// Identical semantics (local `.xz`/raw cache first, remote fetch saved
189    /// into the cache, xz decoded, hash verified) minus the retry loop and
190    /// its inter-attempt sleeps. For an embedded consumer whose fetches run
191    /// on a latency-sensitive path (the hook DLL's managed files do their
192    /// cold-miss fetch on a game thread: bounded timeouts, degrade to
193    /// vanilla on failure), the retry
194    /// loop's worst case (`3 × global timeout + 1.5 s of sleeps`) is not an
195    /// acceptable stall, so it gets a one-shot variant and applies its own
196    /// retry policy (or none) instead.
197    ///
198    /// # Errors
199    ///
200    /// Same error set as [`GameStorage::get_content`], reached in one
201    /// attempt instead of up to three.
202    pub fn get_content_once(&self, hash: &str) -> Result<Vec<u8>, HybridStorageError> {
203        self.get_content_with_retries(hash, 1)
204    }
205
206    fn get_content_with_retries(
207        &self,
208        hash: &str,
209        attempts: u32,
210    ) -> Result<Vec<u8>, HybridStorageError> {
211        let local_path_xz = self.contents_path().join(format!("{hash}.xz"));
212        let local_path_raw = self.contents_path().join(hash);
213
214        if local_path_xz.exists() {
215            let compressed_data = fs::read(&local_path_xz)?;
216            let mut decoder = xz2::read::XzDecoder::new(compressed_data.as_slice());
217            let mut decompressed = Vec::new();
218            decoder.read_to_end(&mut decompressed)?;
219            return Ok(decompressed);
220        }
221
222        if local_path_raw.exists() {
223            return Ok(fs::read(&local_path_raw)?);
224        }
225
226        if self.api_url.is_some() {
227            let mut last_error = None;
228            let mut data = None;
229            let mut remote_filename = String::new();
230
231            for attempt in 1..=attempts {
232                match self.fetch_online(AssetType::Content, hash) {
233                    Ok((d, f)) => {
234                        data = Some(d);
235                        remote_filename = f;
236                        break;
237                    }
238                    Err(e) => {
239                        last_error = Some(e);
240                        if attempt < attempts {
241                            std::thread::sleep(std::time::Duration::from_millis(
242                                500 * attempt as u64,
243                            ));
244                        }
245                    }
246                }
247            }
248
249            let data = match data {
250                Some(d) => d,
251                None => {
252                    return Err(HybridStorageError::OnlineError(
253                        last_error.unwrap_or(OnlineStorageError::NotFound(hash.to_string())),
254                    ));
255                }
256            };
257
258            let is_xz = remote_filename.ends_with(".xz");
259            let save_path = self.contents_path().join(&remote_filename);
260
261            fs::create_dir_all(
262                save_path
263                    .parent()
264                    .ok_or_else(|| io::Error::other("Save path has no parent directory"))?,
265            )?;
266            fs::write(&save_path, &data)?;
267
268            let final_data = if is_xz {
269                let mut decoder = xz2::read::XzDecoder::new(data.as_slice());
270                let mut decompressed = Vec::new();
271                decoder.read_to_end(&mut decompressed)?;
272                decompressed
273            } else {
274                data
275            };
276
277            // Verify hash of uncompressed data
278            verify_hash(&final_data, hash)?;
279
280            Ok(final_data)
281        } else {
282            Err(HybridStorageError::NotFound(hash.to_string()))
283        }
284    }
285
286    pub fn upload_content(&self, hash: &str, data: &[u8]) -> Result<(), HybridStorageError> {
287        if self.api_url.is_some() {
288            let content_path_xz = self.contents_path().join(format!("{hash}.xz"));
289            if let Ok(compressed_data) = fs::read(&content_path_xz) {
290                self.upload_content_file(&format!("{hash}.xz"), &compressed_data)?;
291            } else {
292                self.upload_content_file(hash, data)?;
293            }
294        }
295
296        Ok(())
297    }
298
299    /// Internal helper to upload content file
300    pub(crate) fn upload_content_file(
301        &self,
302        filename: &str,
303        data: &[u8],
304    ) -> Result<(), HybridStorageError> {
305        self.upload_file_unified("contents", data, "content", Some(filename), None)
306    }
307}
308
309#[cfg(test)]
310mod tests {
311    use super::*;
312    use tempfile::tempdir;
313
314    #[test]
315    fn get_content_returns_local_raw_content_without_network() {
316        let dir = tempdir().unwrap();
317        let storage = GameStorage::new(dir.path());
318        fs::create_dir_all(storage.contents_path()).unwrap();
319        let hash = "deadbeef";
320        fs::write(storage.contents_path().join(hash), b"payload").unwrap();
321
322        let data = storage.get_content(hash).expect("get_content");
323        assert_eq!(data, b"payload");
324    }
325
326    #[test]
327    fn get_content_prefers_and_decompresses_xz_cache_over_raw() {
328        let dir = tempdir().unwrap();
329        let storage = GameStorage::new(dir.path());
330        fs::create_dir_all(storage.contents_path()).unwrap();
331        let hash = "cafef00d";
332
333        let raw = b"hello xz world";
334        let compressed = xz2::write::XzEncoder::new(Vec::new(), 6)
335            .and_then_finish(raw)
336            .expect("compress");
337
338        fs::write(
339            storage.contents_path().join(format!("{hash}.xz")),
340            &compressed,
341        )
342        .unwrap();
343
344        let data = storage.get_content(hash).expect("get_content");
345        assert_eq!(data, raw);
346    }
347
348    #[test]
349    fn get_content_returns_not_found_when_offline_and_uncached() {
350        let dir = tempdir().unwrap();
351        let storage = GameStorage::new(dir.path());
352        let result = storage.get_content("missing_hash");
353        assert!(matches!(result, Err(HybridStorageError::NotFound(_))));
354    }
355
356    #[test]
357    fn get_content_once_returns_local_raw_content_without_network() {
358        let dir = tempdir().unwrap();
359        let storage = GameStorage::new(dir.path());
360        fs::create_dir_all(storage.contents_path()).unwrap();
361        let hash = "feedface";
362        fs::write(storage.contents_path().join(hash), b"payload").unwrap();
363
364        let data = storage.get_content_once(hash).expect("get_content_once");
365        assert_eq!(data, b"payload");
366    }
367
368    #[test]
369    fn upload_content_is_a_no_op_when_offline() {
370        let dir = tempdir().unwrap();
371        let storage = GameStorage::new(dir.path());
372        // Offline storage has no api_url, so this should short-circuit to Ok(())
373        // without attempting any network I/O.
374        let result = storage.upload_content("somehash", b"data");
375        assert!(result.is_ok());
376    }
377
378    // A small helper trait extension so the xz2 encoder test above reads nicely.
379    trait AndThenFinish {
380        fn and_then_finish(self, data: &[u8]) -> io::Result<Vec<u8>>;
381    }
382    impl AndThenFinish for xz2::write::XzEncoder<Vec<u8>> {
383        fn and_then_finish(mut self, data: &[u8]) -> io::Result<Vec<u8>> {
384            use std::io::Write;
385            self.write_all(data)?;
386            self.finish()
387        }
388    }
389
390    /// Mirrors `tests/online_storage/helpers.rs::corrupt_chunk_hash`: flips a
391    /// character in the middle of a hash string, so a good hash no longer
392    /// matches its data.
393    fn corrupt_chunk_hash(original_hash: &str) -> String {
394        let mut corrupted = original_hash.to_string();
395        if let Some(ch) = corrupted.chars().nth(32) {
396            let new_ch = if ch == 'a' { 'b' } else { 'a' };
397            corrupted.replace_range(32..33, &new_ch.to_string());
398        }
399        corrupted
400    }
401
402    /// Mirrors `tests/online_storage/helpers.rs::corrupt_chunk_data`: flips a
403    /// couple of bytes in the middle of a data buffer, simulating
404    /// bit-rot/transport corruption without changing its length.
405    fn corrupt_chunk_data(data: &[u8]) -> Vec<u8> {
406        let mut corrupted = data.to_vec();
407        if corrupted.len() > 100 {
408            corrupted[50] = corrupted[50].wrapping_add(1);
409            corrupted[51] = corrupted[51].wrapping_add(1);
410        }
411        corrupted
412    }
413
414    fn sha256_hex(data: &[u8]) -> String {
415        let mut hasher = Sha256::new();
416        hasher.update(data);
417        hex::encode(hasher.finalize())
418    }
419
420    #[test]
421    fn verify_hash_accepts_matching_data_and_hash() {
422        let data = generate_test_content(256);
423        let hash = sha256_hex(&data);
424        assert!(verify_hash(&data, &hash).is_ok());
425    }
426
427    #[test]
428    fn verify_hash_rejects_data_corrupted_with_corrupt_chunk_data() {
429        let data = generate_test_content(256);
430        let hash = sha256_hex(&data);
431        let corrupted = corrupt_chunk_data(&data);
432
433        let result = verify_hash(&corrupted, &hash);
434        match result {
435            Err(HybridStorageError::CorruptedDownload(reported_hash)) => {
436                assert_eq!(reported_hash, hash);
437            }
438            other => panic!("expected CorruptedDownload, got {other:?}"),
439        }
440    }
441
442    #[test]
443    fn verify_hash_rejects_hash_corrupted_with_corrupt_chunk_hash() {
444        let data = generate_test_content(256);
445        let hash = sha256_hex(&data);
446        let bad_hash = corrupt_chunk_hash(&hash);
447        assert_ne!(
448            bad_hash, hash,
449            "helper should have actually changed the hash"
450        );
451
452        let result = verify_hash(&data, &bad_hash);
453        match result {
454            Err(HybridStorageError::CorruptedDownload(reported_hash)) => {
455                assert_eq!(reported_hash, bad_hash);
456            }
457            other => panic!("expected CorruptedDownload, got {other:?}"),
458        }
459    }
460
461    #[test]
462    fn decode_and_verify_entry_accepts_uncompressed_entry_with_matching_hash() {
463        let data = generate_test_content(128);
464        let hash = sha256_hex(&data);
465
466        let (returned_hash, decoded) = decode_and_verify_entry(&hash, &data).expect("decode");
467        assert_eq!(returned_hash, hash);
468        assert_eq!(decoded, data);
469    }
470
471    #[test]
472    fn decode_and_verify_entry_rejects_corrupted_uncompressed_content() {
473        let data = generate_test_content(256);
474        let hash = sha256_hex(&data);
475        let corrupted = corrupt_chunk_data(&data);
476
477        let result = decode_and_verify_entry(&hash, &corrupted);
478        assert!(matches!(
479            result,
480            Err(HybridStorageError::CorruptedDownload(reported_hash)) if reported_hash == hash
481        ));
482    }
483
484    #[test]
485    fn decode_and_verify_entry_decompresses_and_accepts_matching_xz_entry() {
486        let raw = generate_test_content(512);
487        let hash = sha256_hex(&raw);
488        let compressed = xz2::write::XzEncoder::new(Vec::new(), 6)
489            .and_then_finish(&raw)
490            .expect("compress");
491        let filename = format!("{hash}.xz");
492
493        let (returned_hash, decoded) =
494            decode_and_verify_entry(&filename, &compressed).expect("decode xz");
495        assert_eq!(returned_hash, hash);
496        assert_eq!(decoded, raw);
497    }
498
499    #[test]
500    fn decode_and_verify_entry_rejects_xz_entry_whose_decompressed_content_was_corrupted() {
501        let raw = generate_test_content(512);
502        let hash = sha256_hex(&raw);
503        let corrupted_raw = corrupt_chunk_data(&raw);
504        let compressed_corrupted = xz2::write::XzEncoder::new(Vec::new(), 6)
505            .and_then_finish(&corrupted_raw)
506            .expect("compress");
507        let filename = format!("{hash}.xz");
508
509        let result = decode_and_verify_entry(&filename, &compressed_corrupted);
510        assert!(matches!(
511            result,
512            Err(HybridStorageError::CorruptedDownload(reported_hash)) if reported_hash == hash
513        ));
514    }
515
516    /// Deterministic pseudo-random content generator, mirroring
517    /// `tests/online_storage/helpers.rs::generate_random_content`, so
518    /// corruption tests operate on realistic incompressible-ish data rather
519    /// than repeated bytes.
520    fn generate_test_content(size: usize) -> Vec<u8> {
521        let mut content = Vec::with_capacity(size);
522        let mut seed = size as u64;
523        for i in 0..size {
524            seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12345);
525            content.push(((seed / 65536) % 256) as u8);
526            if i % 256 == 0 {
527                seed = seed.wrapping_add(i as u64);
528            }
529        }
530        content
531    }
532}
supreme_fs/game_storage/online/content.rs

supreme_fs/game_storage/online/
content.rs
