Format token count with thousands separator

Cargo.toml

@@ -45,6 +45,7 @@ env_logger = "0.11.3"
 arboard = "3.4.0"
 pyo3 = { version = "0.23", features = ["extension-module", "abi3-py312", "generate-import-lib"] }
 globset = "0.4.15"
+num-format = "0.4"
 
 [profile.release]
 lto = "thin"

src/main.rs

@@ -15,6 +15,7 @@ use indicatif::{ProgressBar, ProgressStyle};
 use log::{debug, error};
 use serde_json::json;
 use std::path::PathBuf;
+use num_format::{Locale, ToFormattedString};
 
 // Constants
 const DEFAULT_TEMPLATE_NAME: &str = "default";
@@ -23,8 +24,8 @@ const CUSTOM_TEMPLATE_NAME: &str = "custom";
 // CLI Arguments
 #[derive(Parser)]
 #[clap(
-    name = env!("CARGO_PKG_NAME"), 
-    version = env!("CARGO_PKG_VERSION"), 
+    name = env!("CARGO_PKG_NAME"),
+    version = env!("CARGO_PKG_VERSION"),
     author = env!("CARGO_PKG_AUTHORS")
 )]
 #[command(arg_required_else_help = true)]
@@ -261,12 +262,13 @@ fn main() -> Result<()> {
         println!("{}", serde_json::to_string_pretty(&json_output)?);
         return Ok(());
     } else if args.tokens {
+        let formatted_token_count = token_count.to_formatted_string(&Locale::en);
         println!(
             "{}{}{} Token count: {}, Model info: {}",
             "[".bold().white(),
             "i".bold().blue(),
             "]".bold().white(),
-            token_count.to_string().bold().yellow(),
+            formatted_token_count.bold().yellow(),
             model_info
         );
     }

src/token.rs

@@ -2,6 +2,7 @@
 
 use colored::*;
 use tiktoken_rs::{cl100k_base, o200k_base, p50k_base, p50k_edit, r50k_base, CoreBPE};
+use num_format::{Locale, ToFormattedString};
 
 /// Returns the appropriate tokenizer based on the provided encoding.
 ///
@@ -69,13 +70,14 @@ pub fn count_tokens(rendered: &str, encoding: &Option<String>) {
     };
 
     let token_count = bpe.unwrap().encode_with_special_tokens(rendered).len();
+    let formatted_token_count = token_count.to_formatted_string(&Locale::en);
 
     println!(
         "{}{}{} Token count: {}, Model info: {}",
         "[".bold().white(),
         "i".bold().blue(),
         "]".bold().white(),
-        token_count.to_string().bold().yellow(),
+        formatted_token_count.bold().yellow(),
         model_info
     );
 }