Struct burn::data::dataset::HuggingfaceDatasetLoader

pub struct HuggingfaceDatasetLoader { /* private fields */ }

Load a dataset from huggingface datasets.

The dataset with all splits is stored in a single sqlite database (see SqliteDataset).

Example

 use burn_dataset::HuggingfaceDatasetLoader;
 use burn_dataset::SqliteDataset;
 use serde::{Deserialize, Serialize};

#[derive(Deserialize, Debug, Clone)]
struct MnistItemRaw {
    pub image_bytes: Vec<u8>,
    pub label: usize,
}

 let train_ds:SqliteDataset<MnistItemRaw> = HuggingfaceDatasetLoader::new("mnist")
      .dataset("train")
      .unwrap();

Implementations

impl HuggingfaceDatasetLoader

pub fn new(name: &str) -> HuggingfaceDatasetLoader

Create a huggingface dataset loader.

pub fn with_subset(self, subset: &str) -> HuggingfaceDatasetLoader

Create a huggingface dataset loader for a subset of the dataset.

The subset name must be one of the subsets listed in the dataset page.

If no subset names are listed, then do not use this method.

pub fn with_base_dir(self, base_dir: &str) -> HuggingfaceDatasetLoader

Specify a base directory to store the dataset.

If not specified, the dataset will be stored in ~/.cache/burn-dataset.

pub fn with_huggingface_token( self, huggingface_token: &str, ) -> HuggingfaceDatasetLoader

Specify a huggingface token to download datasets behind authentication.

You can get a token from tokens settings

pub fn with_huggingface_cache_dir( self, huggingface_cache_dir: &str, ) -> HuggingfaceDatasetLoader

Specify a huggingface cache directory to store the downloaded datasets.

If not specified, the dataset will be stored in ~/.cache/huggingface/datasets.

pub fn with_trust_remote_code( self, trust_remote_code: bool, ) -> HuggingfaceDatasetLoader

Specify whether or not to trust remote code.

If not specified, trust remote code is set to true.

pub fn dataset<I>(self, split: &str) -> Result<SqliteDataset<I>, ImporterError>

where I: DeserializeOwned + Clone,

Load the dataset.

pub fn db_file(self) -> Result<PathBuf, ImporterError>

Get the path to the sqlite database file.

If the database file does not exist, it will be downloaded and imported.