From 6c1008189903c8f5fc5a8f5087ff4001eabc360d Mon Sep 17 00:00:00 2001
From: Yang Jun <jun.yang@target-energysolutions.com>
Date: Thu, 9 Jan 2025 18:09:33 +0800
Subject: [PATCH] fix: failed to download tokernizer from HF (#1060)

---
 crates/avante-tokenizers/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/crates/avante-tokenizers/src/lib.rs b/crates/avante-tokenizers/src/lib.rs
index f725c13..d89cf1c 100644
--- a/crates/avante-tokenizers/src/lib.rs
+++ b/crates/avante-tokenizers/src/lib.rs
@@ -71,7 +71,9 @@ impl HuggingFaceTokenizer {
 
         if !cached_path.exists() {
             let response = ureq::get(url).call().unwrap();
-            let _ = std::fs::write(&cached_path, response.into_string().unwrap());
+            let mut file = std::fs::File::create(&cached_path).unwrap();
+            let mut reader = response.into_reader();
+            std::io::copy(&mut reader, &mut file).unwrap();
         }
         cached_path
     }