Arg name correction: auth_token -> token (#1621)

* Arg name correction: auth_token -> token

* Arg name correction in .rs: auth_token -> token

* update from_pretrained.rs file as well

---------

Co-authored-by: Rene Ravenel <[email protected]>
Co-authored-by: Arthur Zucker <[email protected]>

Author: 3 people

bindings/python/py_src/tokenizers/__init__.pyi

@@ -971,7 +971,7 @@ class Tokenizer:
         pass
 
     @staticmethod
-    def from_pretrained(identifier, revision="main", auth_token=None):
+    def from_pretrained(identifier, revision="main", token=None):
         """
         Instantiate a new :class:`~tokenizers.Tokenizer` from an existing file on the
         Hugging Face Hub.
@@ -982,7 +982,7 @@ class Tokenizer:
                 a tokenizer.json file
             revision (:obj:`str`, defaults to `main`):
                 A branch or commit id
-            auth_token (:obj:`str`, `optional`, defaults to `None`):
+            token (:obj:`str`, `optional`, defaults to `None`):
                 An optional auth token used to access private repositories on the
                 Hugging Face Hub
 

bindings/python/src/tokenizer.rs

@@ -578,19 +578,19 @@ impl PyTokenizer {
     ///         a tokenizer.json file
     ///     revision (:obj:`str`, defaults to `main`):
     ///         A branch or commit id
-    ///     auth_token (:obj:`str`, `optional`, defaults to `None`):
+    ///     token (:obj:`str`, `optional`, defaults to `None`):
     ///         An optional auth token used to access private repositories on the
     ///         Hugging Face Hub
     ///
     /// Returns:
     ///     :class:`~tokenizers.Tokenizer`: The new tokenizer
     #[staticmethod]
-    #[pyo3(signature = (identifier, revision = String::from("main"), auth_token = None))]
-    #[pyo3(text_signature = "(identifier, revision=\"main\", auth_token=None)")]
+    #[pyo3(signature = (identifier, revision = String::from("main"), token = None))]
+    #[pyo3(text_signature = "(identifier, revision=\"main\", token=None)")]
     fn from_pretrained(
         identifier: &str,
         revision: String,
-        auth_token: Option<String>,
+        token: Option<String>,
     ) -> PyResult<Self> {
         let path = Python::with_gil(|py| -> PyResult<String> {
             let huggingface_hub = PyModule::import_bound(py, intern!(py, "huggingface_hub"))?;
@@ -601,8 +601,8 @@ impl PyTokenizer {
                 (intern!(py, "revision"), &revision),
             ]
             .into_py_dict_bound(py);
-            if let Some(auth_token) = auth_token {
-                kwargs.set_item(intern!(py, "token"), auth_token)?;
+            if let Some(token) = token {
+                kwargs.set_item(intern!(py, "token"), token)?;
             }
             let path: String = hf_hub_download.call((), Some(&kwargs))?.extract()?;
             Ok(path)

tokenizers/src/utils/from_pretrained.rs

@@ -8,15 +8,15 @@ use std::path::PathBuf;
 pub struct FromPretrainedParameters {
     pub revision: String,
     pub user_agent: HashMap<String, String>,
-    pub auth_token: Option<String>,
+    pub token: Option<String>,
 }
 
 impl Default for FromPretrainedParameters {
     fn default() -> Self {
         Self {
             revision: "main".into(),
             user_agent: HashMap::new(),
-            auth_token: None,
+            token: None,
         }
     }
 }
@@ -60,7 +60,7 @@ pub fn from_pretrained<S: AsRef<str>>(
     }
 
     let mut builder = ApiBuilder::new();
-    if let Some(token) = params.auth_token {
+    if let Some(token) = params.token {
         builder = builder.with_token(Some(token));
     }
     let api = builder.build()?;