@@ -0,0 +1,2064 @@
+%22%22%22 Dataset parser interface that wraps Hugging Face datasets%0AHacked together by / Copyright 2022 Ross Wightman%0A%22%22%22%0Aimport io%0Aimport math%0Aimport torch%0Aimport torch.distributed as dist%0Afrom PIL import Image%0A%0Atry:%0A    import datasets%0Aexcept ImportError as e:%0A    print(%22Please install Hugging Face datasets package %60pip install datasets%60.%22)%0A    exit(1)%0Afrom .parser import Parser%0A%0A%0Adef get_class_labels(info):%0A    if 'label' not in info.features:%0A        return %7B%7D%0A    class_label = info.features%5B'label'%5D%0A    class_to_idx = %7Bn: class_label.str2int(n) for n in class_label.names%7D%0A    return class_to_idx%0A%0A%0Aclass ParserHfds(Parser):%0A%0A    def __init__(%0A            self,%0A            root,%0A            name,%0A            split='train',%0A            class_map=None,%0A            download=False,%0A    ):%0A        %22%22%22%0A        %22%22%22%0A        super().__init__()%0A        self.root = root%0A        self.split = split%0A        self.dataset = datasets.load_dataset(%0A            name,  # 'name' maps to path arg in hf datasets%0A            split=split,%0A            cache_dir=self.root,  # timm doesn't expect hidden cache dir for datasets, specify a path%0A            #use_auth_token=True,%0A        )%0A        # leave decode for caller, plus we want easy access to original path names...%0A        self.dataset = self.dataset.cast_column('image', datasets.Image(decode=False))%0A%0A        self.class_to_idx = get_class_labels(self.dataset.info)%0A        self.split_info = self.dataset.info.splits%5Bsplit%5D%0A        self.num_examples = self.split_info.num_examples%0A%0A    def __getitem__(self, index):%0A        item = self.dataset%5Bindex%5D%0A        image = item%5B'image'%5D%0A        if 'bytes' in image and image%5B'bytes'%5D:%0A            image = io.BytesIO(image%5B'bytes'%5D)%0A        else:%0A            assert 'path' in image and image%5B'path'%5D%0A            image = open(image%5B'path'%5D, 'rb')%0A        return image, item%5B'label'%5D%0A%0A    def __len__(self):%0A        return len(self.dataset)%0A%0A    def _filename(self, index, basename=False, absolute=False):%0A        item = self.dataset%5Bindex%5D%0A        return item%5B'image'%5D%5B'path'%5D%0A

Add initial Hugging Face Datasets parser impl.