1# Copyright 2019-2023 Huawei Technologies Co., Ltd 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15""" 16This file contains specific vision dataset loading classes. You can easily use 17these classes to load the prepared dataset. For example: 18 ImageFolderDataset: which is about ImageNet dataset. 19 Cifar10Dataset: which is cifar10 binary version dataset. 20 Cifar100Dataset: which is cifar100 binary version dataset. 21 MnistDataset: which is mnist dataset. 22 ... 23After declaring the dataset object, you can further apply dataset operations 24(e.g. filter, skip, concat, map, batch) on it. 25""" 26import os 27import numpy as np 28from scipy.io import loadmat 29from PIL import Image 30 31import mindspore._c_dataengine as cde 32 33from .datasets import VisionBaseDataset, SourceDataset, MappableDataset, Shuffle, Schema 34from .datasets_user_defined import GeneratorDataset 35from .validators import check_caltech101_dataset, check_caltech256_dataset, check_celebadataset, \ 36 check_cityscapes_dataset, check_cocodataset, check_div2k_dataset, check_emnist_dataset, check_fake_image_dataset, \ 37 check_flickr_dataset, check_flowers102dataset, check_food101_dataset, check_imagefolderdataset, \ 38 check_kittidataset, check_lfw_dataset, check_lsun_dataset, check_manifestdataset, check_mnist_cifar_dataset, \ 39 check_omniglotdataset, check_photo_tour_dataset, check_places365_dataset, check_qmnist_dataset, \ 40 check_random_dataset, check_rendered_sst2_dataset, check_sb_dataset, check_sbu_dataset, check_semeion_dataset, \ 41 check_stl10_dataset, check_sun397_dataset, check_svhn_dataset, check_usps_dataset, check_vocdataset, \ 42 check_wider_face_dataset 43 44from ..core.validator_helpers import replace_none 45 46 47class _Caltech101Dataset: 48 """ 49 Mainly for loading Caltech101 Dataset, and return two rows each time. 50 """ 51 52 def __init__(self, dataset_dir, target_type="category", decode=False): 53 self.dataset_dir = os.path.realpath(dataset_dir) 54 self.image_dir = os.path.join(self.dataset_dir, "101_ObjectCategories") 55 self.annotation_dir = os.path.join(self.dataset_dir, "Annotations") 56 self.target_type = target_type 57 if self.target_type == "category": 58 self.column_names = ["image", "category"] 59 elif self.target_type == "annotation": 60 self.column_names = ["image", "annotation"] 61 else: 62 self.column_names = ["image", "category", "annotation"] 63 self.decode = decode 64 self.classes = sorted(os.listdir(self.image_dir)) 65 if "BACKGROUND_Google" in self.classes: 66 self.classes.remove("BACKGROUND_Google") 67 name_map = {"Faces": "Faces_2", 68 "Faces_easy": "Faces_3", 69 "Motorbikes": "Motorbikes_16", 70 "airplanes": "Airplanes_Side_2"} 71 self.annotation_classes = [name_map[class_name] if class_name in name_map else class_name 72 for class_name in self.classes] 73 self.image_index = [] 74 self.image_label = [] 75 for i, image_class in enumerate(self.classes): 76 sub_dir = os.path.join(self.image_dir, image_class) 77 if not os.path.isdir(sub_dir) or not os.access(sub_dir, os.R_OK): 78 continue 79 num_images = len(os.listdir(sub_dir)) 80 self.image_index.extend(range(1, num_images + 1)) 81 self.image_label.extend(num_images * [i]) 82 83 def __getitem__(self, index): 84 image_file = os.path.join(self.image_dir, self.classes[self.image_label[index]], 85 "image_{:04d}.jpg".format(self.image_index[index])) 86 if not os.path.exists(image_file): 87 raise ValueError("The image file {} does not exist or permission denied!".format(image_file)) 88 if self.decode: 89 image = np.asarray(Image.open(image_file).convert("RGB")) 90 else: 91 image = np.fromfile(image_file, dtype=np.uint8) 92 93 if self.target_type == "category": 94 return image, self.image_label[index] 95 annotation_file = os.path.join(self.annotation_dir, self.annotation_classes[self.image_label[index]], 96 "annotation_{:04d}.mat".format(self.image_index[index])) 97 if not os.path.exists(annotation_file): 98 raise ValueError("The annotation file {} does not exist or permission denied!".format(annotation_file)) 99 annotation = loadmat(annotation_file)["obj_contour"] 100 101 if self.target_type == "annotation": 102 return image, annotation 103 return image, self.image_label[index], annotation 104 105 def __len__(self): 106 return len(self.image_index) 107 108 109class Caltech101Dataset(GeneratorDataset): 110 """ 111 Caltech 101 dataset. 112 113 The columns of the generated dataset depend on the value of `target_type` . 114 115 - When `target_type` is ``'category'``, the columns are :py:obj:`[image, category]` . 116 - When `target_type` is ``'annotation'``, the columns are :py:obj:`[image, annotation]` . 117 - When `target_type` is ``'all'``, the columns are :py:obj:`[image, category, annotation]` . 118 119 The tensor of column :py:obj:`image` is of the uint8 type. 120 The tensor of column :py:obj:`category` is of the uint32 type. 121 The tensor of column :py:obj:`annotation` is a 2-dimensional ndarray that stores the contour of the image 122 and consists of a series of points. 123 124 Args: 125 dataset_dir (str): Path to the root directory that contains the dataset. This root directory contains two 126 subdirectories, one is called 101_ObjectCategories, which stores images, 127 and the other is called Annotations, which stores annotations. 128 target_type (str, optional): Target of the image. If `target_type` is ``'category'``, 129 return category represents the target class. If `target_type` is ``'annotation'``, 130 return annotation. If `target_type` is ``'all'``, return category and annotation. 131 Default: ``None`` , means ``'category'``. 132 num_samples (int, optional): The number of images to be included in the dataset. 133 Default: ``None`` , all images. 134 num_parallel_workers (int, optional): Number of worker subprocesses to read the data. 135 Default: ``1``. 136 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 137 Default: ``None`` , expected order behavior shown in the table below. 138 decode (bool, optional): Whether or not to decode the images after reading. Default: ``False``. 139 sampler (Sampler, optional): Object used to choose samples from the 140 dataset. Default: ``None`` , expected order behavior shown in the table below. 141 num_shards (int, optional): Number of shards that the dataset will be divided 142 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 143 the maximum sample number of per shard. 144 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 145 argument can only be specified when `num_shards` is also specified. 146 147 Raises: 148 RuntimeError: If `dataset_dir` does not contain data files. 149 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 150 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 151 RuntimeError: If `num_shards` is specified but `shard_id` is None. 152 RuntimeError: If `shard_id` is specified but `num_shards` is None. 153 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 154 ValueError: If `target_type` is not ``'category'``, ``'annotation'`` or ``'all'`` . 155 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 156 157 Tutorial Examples: 158 - `Load & Process Data With Dataset Pipeline 159 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 160 161 Note: 162 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 163 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 164 165 .. include:: mindspore.dataset.sampler.txt 166 167 Examples: 168 >>> import mindspore.dataset as ds 169 >>> caltech101_dataset_directory = "/path/to/caltech101_dataset_directory" 170 >>> 171 >>> # 1) Read all samples (image files) in caltech101_dataset_directory with 8 threads 172 >>> dataset = ds.Caltech101Dataset(dataset_dir=caltech101_dataset_directory, num_parallel_workers=8) 173 >>> 174 >>> # 2) Read all samples (image files) with the target_type "annotation" 175 >>> dataset = ds.Caltech101Dataset(dataset_dir=caltech101_dataset_directory, target_type="annotation") 176 177 About Caltech101Dataset: 178 179 Pictures of objects belonging to 101 categories, about 40 to 800 images per category. 180 Most categories have about 50 images. The size of each image is roughly 300 x 200 pixels. 181 The official provides the contour data of each object in each picture, which is the annotation. 182 183 Here is the original Caltech101 dataset structure, 184 and you can unzip the dataset files into the following directory structure, which are read by MindSpore API. 185 186 .. code-block:: 187 188 . 189 └── caltech101_dataset_directory 190 ├── 101_ObjectCategories 191 │ ├── Faces 192 │ │ ├── image_0001.jpg 193 │ │ ├── image_0002.jpg 194 │ │ ... 195 │ ├── Faces_easy 196 │ │ ├── image_0001.jpg 197 │ │ ├── image_0002.jpg 198 │ │ ... 199 │ ├── ... 200 └── Annotations 201 ├── Airplanes_Side_2 202 │ ├── annotation_0001.mat 203 │ ├── annotation_0002.mat 204 │ ... 205 ├── Faces_2 206 │ ├── annotation_0001.mat 207 │ ├── annotation_0002.mat 208 │ ... 209 ├── ... 210 211 Citation: 212 213 .. code-block:: 214 215 @article{FeiFei2004LearningGV, 216 author = {Li Fei-Fei and Rob Fergus and Pietro Perona}, 217 title = {Learning Generative Visual Models from Few Training Examples: 218 An Incremental Bayesian Approach Tested on 101 Object Categories}, 219 journal = {Computer Vision and Pattern Recognition Workshop}, 220 year = {2004}, 221 url = {https://data.caltech.edu/records/mzrjq-6wc02}, 222 } 223 """ 224 225 @check_caltech101_dataset 226 def __init__(self, dataset_dir, target_type=None, num_samples=None, num_parallel_workers=1, 227 shuffle=None, decode=False, sampler=None, num_shards=None, shard_id=None): 228 self.dataset_dir = dataset_dir 229 self.target_type = replace_none(target_type, "category") 230 self.decode = replace_none(decode, False) 231 dataset = _Caltech101Dataset(self.dataset_dir, self.target_type, self.decode) 232 super().__init__(dataset, column_names=dataset.column_names, num_samples=num_samples, 233 num_parallel_workers=num_parallel_workers, shuffle=shuffle, sampler=sampler, 234 num_shards=num_shards, shard_id=shard_id) 235 236 def get_class_indexing(self): 237 """ 238 Get the class index. 239 240 Returns: 241 dict, a str-to-int mapping from label name to index. 242 """ 243 class_dict = {'Faces': 0, 'Faces_easy': 1, 'Leopards': 2, 'Motorbikes': 3, 'accordion': 4, 'airplanes': 5, 244 'anchor': 6, 'ant': 7, 'barrel': 8, 'bass': 9, 'beaver': 10, 'binocular': 11, 'bonsai': 12, 245 'brain': 13, 'brontosaurus': 14, 'buddha': 15, 'butterfly': 16, 'camera': 17, 'cannon': 18, 246 'car_side': 19, 'ceiling_fan': 20, 'cellphone': 21, 'chair': 22, 'chandelier': 23, 247 'cougar_body': 24, 'cougar_face': 25, 'crab': 26, 'crayfish': 27, 'crocodile': 28, 248 'crocodile_head': 29, 'cup': 30, 'dalmatian': 31, 'dollar_bill': 32, 'dolphin': 33, 249 'dragonfly': 34, 'electric_guitar': 35, 'elephant': 36, 'emu': 37, 'euphonium': 38, 'ewer': 39, 250 'ferry': 40, 'flamingo': 41, 'flamingo_head': 42, 'garfield': 43, 'gerenuk': 44, 'gramophone': 45, 251 'grand_piano': 46, 'hawksbill': 47, 'headphone': 48, 'hedgehog': 49, 'helicopter': 50, 'ibis': 51, 252 'inline_skate': 52, 'joshua_tree': 53, 'kangaroo': 54, 'ketch': 55, 'lamp': 56, 'laptop': 57, 253 'llama': 58, 'lobster': 59, 'lotus': 60, 'mandolin': 61, 'mayfly': 62, 'menorah': 63, 254 'metronome': 64, 'minaret': 65, 'nautilus': 66, 'octopus': 67, 'okapi': 68, 'pagoda': 69, 255 'panda': 70, 'pigeon': 71, 'pizza': 72, 'platypus': 73, 'pyramid': 74, 'revolver': 75, 256 'rhino': 76, 'rooster': 77, 'saxophone': 78, 'schooner': 79, 'scissors': 80, 'scorpion': 81, 257 'sea_horse': 82, 'snoopy': 83, 'soccer_ball': 84, 'stapler': 85, 'starfish': 86, 258 'stegosaurus': 87, 'stop_sign': 88, 'strawberry': 89, 'sunflower': 90, 'tick': 91, 259 'trilobite': 92, 'umbrella': 93, 'watch': 94, 'water_lilly': 95, 'wheelchair': 96, 'wild_cat': 97, 260 'windsor_chair': 98, 'wrench': 99, 'yin_yang': 100} 261 return class_dict 262 263 264class Caltech256Dataset(MappableDataset, VisionBaseDataset): 265 """ 266 Caltech 256 dataset. 267 268 The generated dataset has two columns: :py:obj:`[image, label]` . 269 The tensor of column :py:obj:`image` is of the uint8 type. 270 The tensor of column :py:obj:`label` is of the uint32 type. 271 272 Args: 273 dataset_dir (str): Path to the root directory that contains the dataset. 274 num_samples (int, optional): The number of images to be included in the dataset. 275 Default: ``None`` , all images. 276 num_parallel_workers (int, optional): Number of worker threads to read the data. 277 Default: ``None`` , will use global default workers(8), it can be set 278 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 279 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 280 Default: ``None`` , expected order behavior shown in the table below. 281 decode (bool, optional): Whether or not to decode the images after reading. Default: ``False``. 282 sampler (Sampler, optional): Object used to choose samples from the 283 dataset. Default: ``None`` , expected order behavior shown in the table below. 284 num_shards (int, optional): Number of shards that the dataset will be divided 285 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 286 the maximum sample number of per shard. 287 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 288 argument can only be specified when `num_shards` is also specified. 289 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 290 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 291 Default: ``None`` , which means no cache is used. 292 293 Raises: 294 RuntimeError: If `dataset_dir` does not contain data files. 295 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 296 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 297 RuntimeError: If `num_shards` is specified but `shard_id` is None. 298 RuntimeError: If `shard_id` is specified but `num_shards` is None. 299 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 300 ValueError: If `target_type` is not ``'category'``, ``'annotation'`` or ``'all'``. 301 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 302 303 Tutorial Examples: 304 - `Load & Process Data With Dataset Pipeline 305 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 306 307 Note: 308 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 309 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 310 311 .. include:: mindspore.dataset.sampler.txt 312 313 Examples: 314 >>> import mindspore.dataset as ds 315 >>> caltech256_dataset_dir = "/path/to/caltech256_dataset_directory" 316 >>> 317 >>> # 1) Read all samples (image files) in caltech256_dataset_dir with 8 threads 318 >>> dataset = ds.Caltech256Dataset(dataset_dir=caltech256_dataset_dir, num_parallel_workers=8) 319 320 About Caltech256Dataset: 321 322 Caltech-256 is an object recognition dataset containing 30,607 real-world images, of different sizes, 323 spanning 257 classes (256 object classes and an additional clutter class). 324 Each class is represented by at least 80 images. The dataset is a superset of the Caltech-101 dataset. 325 326 .. code-block:: 327 328 . 329 └── caltech256_dataset_directory 330 ├── 001.ak47 331 │ ├── 001_0001.jpg 332 │ ├── 001_0002.jpg 333 │ ... 334 ├── 002.american-flag 335 │ ├── 002_0001.jpg 336 │ ├── 002_0002.jpg 337 │ ... 338 ├── 003.backpack 339 │ ├── 003_0001.jpg 340 │ ├── 003_0002.jpg 341 │ ... 342 ├── ... 343 344 Citation: 345 346 .. code-block:: 347 348 @article{griffin2007caltech, 349 title = {Caltech-256 object category dataset}, 350 added-at = {2021-01-21T02:54:42.000+0100}, 351 author = {Griffin, Gregory and Holub, Alex and Perona, Pietro}, 352 biburl = {https://www.bibsonomy.org/bibtex/21f746f23ff0307826cca3e3be45f8de7/s364315}, 353 interhash = {bfe1e648c1778c04baa60f23d1223375}, 354 intrahash = {1f746f23ff0307826cca3e3be45f8de7}, 355 publisher = {California Institute of Technology}, 356 timestamp = {2021-01-21T02:54:42.000+0100}, 357 year = {2007} 358 } 359 """ 360 361 @check_caltech256_dataset 362 def __init__(self, dataset_dir, num_samples=None, num_parallel_workers=None, shuffle=None, decode=False, 363 sampler=None, num_shards=None, shard_id=None, cache=None): 364 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 365 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 366 367 self.dataset_dir = dataset_dir 368 self.decode = replace_none(decode, False) 369 370 def parse(self, children=None): 371 return cde.Caltech256Node(self.dataset_dir, self.decode, self.sampler) 372 373 374class CelebADataset(MappableDataset, VisionBaseDataset): 375 """ 376 CelebA(CelebFaces Attributes) dataset. 377 378 Only support to read `list_attr_celeba.txt` currently, which is the attribute annotations of the dataset. 379 The generated dataset has two columns: :py:obj:`[image, attr]` . 380 The tensor of column :py:obj:`image` is of the uint8 type. 381 The tensor of column :py:obj:`attr` is of the uint32 type and one hot encoded. 382 383 Args: 384 dataset_dir (str): Path to the root directory that contains the dataset. 385 num_parallel_workers (int, optional): Number of worker threads to read the data. 386 Default: ``None`` , will use global default workers(8), it can be set 387 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 388 shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` . 389 usage (str, optional): Specify the ``'train'``, ``'valid'``, ``'test'`` part or ``'all'`` 390 parts of dataset. Default: ``'all'``, will read all samples. 391 sampler (Sampler, optional): Object used to choose samples from the dataset. Default: ``None`` . 392 decode (bool, optional): Whether to decode the images after reading. Default: ``False``. 393 extensions (list[str], optional): List of file extensions to be included in the dataset. Default: ``None`` . 394 num_samples (int, optional): The number of images to be included in the dataset. 395 Default: ``None`` , will include all images. 396 num_shards (int, optional): Number of shards that the dataset will be divided 397 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 398 the maximum sample number of per shard. 399 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 400 argument can only be specified when `num_shards` is also specified. 401 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 402 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 403 Default: ``None`` , which means no cache is used. 404 decrypt (callable, optional): Image decryption function, which accepts the path of the encrypted image file 405 and returns the decrypted bytes data. Default: ``None`` , no decryption. 406 407 Raises: 408 RuntimeError: If `dataset_dir` does not contain data files. 409 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 410 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 411 RuntimeError: If `num_shards` is specified but `shard_id` is None. 412 RuntimeError: If `shard_id` is specified but `num_shards` is None. 413 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 414 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 415 ValueError: If `usage` is not ``'train'``, ``'valid'``, ``'test'`` or ``'all'``. 416 417 Tutorial Examples: 418 - `Load & Process Data With Dataset Pipeline 419 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 420 421 Note: 422 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 423 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 424 425 .. include:: mindspore.dataset.sampler.txt 426 427 Examples: 428 >>> import mindspore.dataset as ds 429 >>> celeba_dataset_dir = "/path/to/celeba_dataset_directory" 430 >>> 431 >>> # Read 5 samples from CelebA dataset 432 >>> dataset = ds.CelebADataset(dataset_dir=celeba_dataset_dir, usage='train', num_samples=5) 433 >>> 434 >>> # Note: In celeba dataset, each data dictionary owns keys "image" and "attr" 435 436 About CelebA dataset: 437 438 CelebFaces Attributes Dataset (CelebA) is a large-scale dataset 439 with more than 200K celebrity images, each with 40 attribute annotations. 440 441 The images in this dataset cover large pose variations and background clutter. 442 CelebA has large diversities, large quantities, and rich annotations, including 443 444 * 10,177 number of identities, 445 * 202,599 number of images, 446 * 5 landmark locations, 40 binary attributes annotations per image. 447 448 The dataset can be employed as the training and test sets for the following computer 449 vision tasks: attribute recognition, detection, landmark (or facial part) and 450 localization. 451 452 Original CelebA dataset structure: 453 454 .. code-block:: 455 456 . 457 └── CelebA 458 ├── README.md 459 ├── Img 460 │ ├── img_celeba.7z 461 │ ├── img_align_celeba_png.7z 462 │ └── img_align_celeba.zip 463 ├── Eval 464 │ └── list_eval_partition.txt 465 └── Anno 466 ├── list_landmarks_celeba.txt 467 ├── list_landmarks_align_celeba.txt 468 ├── list_bbox_celeba.txt 469 ├── list_attr_celeba.txt 470 └── identity_CelebA.txt 471 472 You can unzip the dataset files into the following structure and read by MindSpore's API. 473 474 .. code-block:: 475 476 . 477 └── celeba_dataset_directory 478 ├── list_attr_celeba.txt 479 ├── 000001.jpg 480 ├── 000002.jpg 481 ├── 000003.jpg 482 ├── ... 483 484 Citation: 485 486 .. code-block:: 487 488 @article{DBLP:journals/corr/LiuLWT14, 489 author = {Ziwei Liu and Ping Luo and Xiaogang Wang and Xiaoou Tang}, 490 title = {Deep Learning Attributes in the Wild}, 491 journal = {CoRR}, 492 volume = {abs/1411.7766}, 493 year = {2014}, 494 url = {http://arxiv.org/abs/1411.7766}, 495 archivePrefix = {arXiv}, 496 eprint = {1411.7766}, 497 timestamp = {Tue, 10 Dec 2019 15:37:26 +0100}, 498 biburl = {https://dblp.org/rec/journals/corr/LiuLWT14.bib}, 499 bibsource = {dblp computer science bibliography, https://dblp.org}, 500 howpublished = {http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html} 501 } 502 """ 503 504 @check_celebadataset 505 def __init__(self, dataset_dir, num_parallel_workers=None, shuffle=None, usage='all', sampler=None, decode=False, 506 extensions=None, num_samples=None, num_shards=None, shard_id=None, cache=None, decrypt=None): 507 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 508 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 509 self.dataset_dir = dataset_dir 510 self.decode = replace_none(decode, False) 511 self.extensions = replace_none(extensions, []) 512 self.usage = replace_none(usage, "all") 513 self.decrypt = decrypt 514 515 def parse(self, children=None): 516 if self.usage != "all": 517 dataset_dir = os.path.realpath(self.dataset_dir) 518 partition_file = os.path.join(dataset_dir, "list_eval_partition.txt") 519 if os.path.exists(partition_file) is False: 520 raise RuntimeError("Partition file can not be found when usage is not 'all'.") 521 return cde.CelebANode(self.dataset_dir, self.usage, self.sampler, self.decode, 522 self.extensions, self.decrypt) 523 524 525 526class Cifar10Dataset(MappableDataset, VisionBaseDataset): 527 """ 528 CIFAR-10 dataset. 529 530 This api only supports parsing CIFAR-10 file in binary version now. 531 The generated dataset has two columns :py:obj:`[image, label]` . 532 The tensor of column :py:obj:`image` is of the uint8 type. 533 The tensor of column :py:obj:`label` is a scalar of the uint32 type. 534 535 Args: 536 dataset_dir (str): Path to the root directory that contains the dataset. 537 usage (str, optional): Usage of this dataset, can be ``'train'`` , ``'test'`` or ``'all'`` . 538 ``'train'`` will read from 50,000 train samples, ``'test'`` will read from 10,000 test 539 samples, ``'all'`` will read from all 60,000 samples. Default: ``None`` , all samples. 540 num_samples (int, optional): The number of images to be included in the dataset. 541 Default: ``None`` , all images. 542 num_parallel_workers (int, optional): Number of worker threads to read the data. 543 Default: ``None`` , will use global default workers(8), it can be set 544 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 545 shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected 546 order behavior shown in the table below. 547 sampler (Sampler, optional): Object used to choose samples from the 548 dataset. Default: ``None`` , expected order behavior shown in the table below. 549 num_shards (int, optional): Number of shards that the dataset will be divided 550 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 551 the maximum sample number of per shard. 552 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 553 argument can only be specified when `num_shards` is also specified. 554 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 555 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 556 Default: ``None`` , which means no cache is used. 557 558 Raises: 559 RuntimeError: If `dataset_dir` does not contain data files. 560 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 561 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 562 RuntimeError: If `num_shards` is specified but `shard_id` is None. 563 RuntimeError: If `shard_id` is specified but `num_shards` is None. 564 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 565 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 566 ValueError: If `usage` is not ``'train'`` , ``'test'`` or ``'all'`` . 567 568 Tutorial Examples: 569 - `Load & Process Data With Dataset Pipeline 570 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 571 572 Note: 573 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 574 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 575 576 .. include:: mindspore.dataset.sampler.txt 577 578 Examples: 579 >>> import mindspore.dataset as ds 580 >>> cifar10_dataset_dir = "/path/to/cifar10_dataset_directory" 581 >>> 582 >>> # 1) Get all samples from CIFAR10 dataset in sequence 583 >>> dataset = ds.Cifar10Dataset(dataset_dir=cifar10_dataset_dir, shuffle=False) 584 >>> 585 >>> # 2) Randomly select 350 samples from CIFAR10 dataset 586 >>> dataset = ds.Cifar10Dataset(dataset_dir=cifar10_dataset_dir, num_samples=350, shuffle=True) 587 >>> 588 >>> # 3) Get samples from CIFAR10 dataset for shard 0 in a 2-way distributed training 589 >>> dataset = ds.Cifar10Dataset(dataset_dir=cifar10_dataset_dir, num_shards=2, shard_id=0) 590 >>> 591 >>> # In CIFAR10 dataset, each dictionary has keys "image" and "label" 592 593 About CIFAR-10 dataset: 594 595 The CIFAR-10 dataset consists of 60000 32x32 color images in 10 classes, 596 with 6000 images per class. There are 50000 training images and 10000 test images. 597 The 10 different classes represent airplanes, cars, birds, cats, deer, dogs, frogs, horses, ships, and trucks. 598 599 Here is the original CIFAR-10 dataset structure. 600 You can unzip the dataset files into the following directory structure and read by MindSpore's API. 601 602 .. code-block:: 603 604 . 605 └── cifar-10-batches-bin 606 ├── data_batch_1.bin 607 ├── data_batch_2.bin 608 ├── data_batch_3.bin 609 ├── data_batch_4.bin 610 ├── data_batch_5.bin 611 ├── test_batch.bin 612 ├── readme.html 613 └── batches.meta.txt 614 615 Citation: 616 617 .. code-block:: 618 619 @techreport{Krizhevsky09, 620 author = {Alex Krizhevsky}, 621 title = {Learning multiple layers of features from tiny images}, 622 institution = {}, 623 year = {2009}, 624 howpublished = {http://www.cs.toronto.edu/~kriz/cifar.html} 625 } 626 """ 627 628 @check_mnist_cifar_dataset 629 def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=None, shuffle=None, 630 sampler=None, num_shards=None, shard_id=None, cache=None): 631 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 632 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 633 634 self.dataset_dir = dataset_dir 635 self.usage = replace_none(usage, "all") 636 637 def parse(self, children=None): 638 return cde.Cifar10Node(self.dataset_dir, self.usage, self.sampler) 639 640 641class Cifar100Dataset(MappableDataset, VisionBaseDataset): 642 """ 643 CIFAR-100 dataset. 644 645 The generated dataset has three columns :py:obj:`[image, coarse_label, fine_label]` . 646 The tensor of column :py:obj:`image` is of the uint8 type. 647 The tensor of column :py:obj:`coarse_label` and :py:obj:`fine_labels` are each a scalar of uint32 type. 648 649 Args: 650 dataset_dir (str): Path to the root directory that contains the dataset. 651 usage (str, optional): Usage of this dataset, can be ``'train'`` , ``'test'`` or ``'all'`` . 652 ``'train'`` will read from 50,000 train samples, ``'test'`` will read from 10,000 test samples, 653 ``'all'`` will read from all 60,000 samples. Default: ``None`` , all samples. 654 num_samples (int, optional): The number of images to be included in the dataset. 655 Default: ``None`` , all images. 656 num_parallel_workers (int, optional): Number of worker threads to read the data. 657 Default: ``None`` , will use global default workers(8), it can be set 658 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 659 shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected 660 order behavior shown in the table below. 661 sampler (Sampler, optional): Object used to choose samples from the 662 dataset. Default: ``None`` , expected order behavior shown in the table below. 663 num_shards (int, optional): Number of shards that the dataset will be divided 664 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 665 the maximum sample number of per shard. 666 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 667 argument can only be specified when `num_shards` is also specified. 668 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 669 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 670 Default: ``None`` , which means no cache is used. 671 672 Raises: 673 RuntimeError: If `dataset_dir` does not contain data files. 674 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 675 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 676 RuntimeError: If `num_shards` is specified but `shard_id` is None. 677 RuntimeError: If `shard_id` is specified but `num_shards` is None. 678 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 679 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 680 ValueError: If `usage` is not ``'train'`` , ``'test'`` or ``'all'`` . 681 682 Tutorial Examples: 683 - `Load & Process Data With Dataset Pipeline 684 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 685 686 Note: 687 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 688 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 689 690 .. include:: mindspore.dataset.sampler.txt 691 692 Examples: 693 >>> import mindspore.dataset as ds 694 >>> cifar100_dataset_dir = "/path/to/cifar100_dataset_directory" 695 >>> 696 >>> # 1) Get all samples from CIFAR100 dataset in sequence 697 >>> dataset = ds.Cifar100Dataset(dataset_dir=cifar100_dataset_dir, shuffle=False) 698 >>> 699 >>> # 2) Randomly select 350 samples from CIFAR100 dataset 700 >>> dataset = ds.Cifar100Dataset(dataset_dir=cifar100_dataset_dir, num_samples=350, shuffle=True) 701 >>> 702 >>> # In CIFAR100 dataset, each dictionary has 3 keys: "image", "fine_label" and "coarse_label" 703 704 About CIFAR-100 dataset: 705 706 This dataset is just like the CIFAR-10, except it has 100 classes containing 600 images 707 each. There are 500 training images and 100 testing images per class. The 100 classes in 708 the CIFAR-100 are grouped into 20 superclasses. Each image comes with a "fine" label (the 709 class to which it belongs) and a "coarse" label (the superclass to which it belongs). 710 711 Here is the original CIFAR-100 dataset structure. 712 You can unzip the dataset files into the following directory structure and read by MindSpore's API. 713 714 .. code-block:: 715 716 . 717 └── cifar-100-binary 718 ├── train.bin 719 ├── test.bin 720 ├── fine_label_names.txt 721 └── coarse_label_names.txt 722 723 Citation: 724 725 .. code-block:: 726 727 @techreport{Krizhevsky09, 728 author = {Alex Krizhevsky}, 729 title = {Learning multiple layers of features from tiny images}, 730 institution = {}, 731 year = {2009}, 732 howpublished = {http://www.cs.toronto.edu/~kriz/cifar.html} 733 } 734 """ 735 736 @check_mnist_cifar_dataset 737 def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=None, shuffle=None, 738 sampler=None, num_shards=None, shard_id=None, cache=None): 739 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 740 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 741 742 self.dataset_dir = dataset_dir 743 self.usage = replace_none(usage, "all") 744 745 def parse(self, children=None): 746 return cde.Cifar100Node(self.dataset_dir, self.usage, self.sampler) 747 748 749class CityscapesDataset(MappableDataset, VisionBaseDataset): 750 """ 751 Cityscapes dataset. 752 753 The generated dataset has two columns :py:obj:`[image, task]` . 754 The tensor of column :py:obj:`image` is of the uint8 type. 755 The tensor of column :py:obj:`task` is of the uint8 type if `task` is not ``'polygon'`` otherwise task is 756 a string tensor with serialize json. 757 758 Args: 759 dataset_dir (str): Path to the root directory that contains the dataset. 760 usage (str, optional): Acceptable usages include ``'train'``, ``'test'``, ``'val'`` or ``'all'`` 761 if `quality_mode` is ``'fine'`` otherwise ``'train'``, ``'train_extra'``, ``'val'`` or 762 ``'all'``. Default: ``'train'``. 763 quality_mode (str, optional): Acceptable quality_modes include ``'fine'`` or ``'coarse'``. 764 Default: ``'fine'``. 765 task (str, optional): Acceptable tasks include ``'instance'``, 766 ``'semantic'``, ``'polygon'`` or ``'color'``. Default: ``'instance'``. 767 num_samples (int, optional): The number of images to be included in the dataset. 768 Default: ``None`` , all images. 769 num_parallel_workers (int, optional): Number of worker threads to read the data. 770 Default: ``None`` , will use global default workers(8), it can be set 771 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 772 shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected 773 order behavior shown in the table below. 774 decode (bool, optional): Decode the images after reading. Default: ``None``, default to be ``False``. 775 sampler (Sampler, optional): Object used to choose samples from the 776 dataset. Default: ``None`` , expected order behavior shown in the table below. 777 num_shards (int, optional): Number of shards that the dataset will be divided 778 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 779 the max sample number of per shard. 780 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 781 argument can only be specified when `num_shards` is also specified. 782 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 783 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 784 Default: ``None`` , which means no cache is used. 785 786 Raises: 787 RuntimeError: If `dataset_dir` is invalid or does not contain data files. 788 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 789 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 790 RuntimeError: If `num_shards` is specified but `shard_id` is None. 791 RuntimeError: If `shard_id` is specified but `num_shards` is None. 792 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 793 ValueError: If `dataset_dir` is not exist. 794 ValueError: If `task` is not ``'instance'``, ``'semantic'``, ``'polygon'`` or ``'color'``. 795 ValueError: If `quality_mode` is not ``'fine'`` or ``'coarse'``. 796 ValueError: If `usage` is invalid. 797 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 798 799 Tutorial Examples: 800 - `Load & Process Data With Dataset Pipeline 801 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 802 803 Note: 804 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 805 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 806 807 .. include:: mindspore.dataset.sampler.txt 808 809 Examples: 810 >>> import mindspore.dataset as ds 811 >>> cityscapes_dataset_dir = "/path/to/cityscapes_dataset_directory" 812 >>> 813 >>> # 1) Get all samples from Cityscapes dataset in sequence 814 >>> dataset = ds.CityscapesDataset(dataset_dir=cityscapes_dataset_dir, task="instance", quality_mode="fine", 815 ... usage="train", shuffle=False, num_parallel_workers=1) 816 >>> 817 >>> # 2) Randomly select 350 samples from Cityscapes dataset 818 >>> dataset = ds.CityscapesDataset(dataset_dir=cityscapes_dataset_dir, num_samples=350, shuffle=True, 819 ... num_parallel_workers=1) 820 >>> 821 >>> # 3) Get samples from Cityscapes dataset for shard 0 in a 2-way distributed training 822 >>> dataset = ds.CityscapesDataset(dataset_dir=cityscapes_dataset_dir, num_shards=2, shard_id=0, 823 ... num_parallel_workers=1) 824 >>> 825 >>> # In Cityscapes dataset, each dictionary has keys "image" and "task" 826 827 About Cityscapes dataset: 828 829 The Cityscapes dataset consists of 5000 color images with high quality dense pixel annotations and 830 19998 color images with coarser polygonal annotations in 50 cities. There are 30 classes in this 831 dataset and the polygonal annotations include dense semantic segmentation and instance segmentation 832 for vehicle and people. 833 834 You can unzip the dataset files into the following directory structure and read by MindSpore's API. 835 836 Taking the quality_mode of `fine` as an example. 837 838 .. code-block:: 839 840 . 841 └── Cityscapes 842 ├── leftImg8bit 843 | ├── train 844 | | ├── aachen 845 | | | ├── aachen_000000_000019_leftImg8bit.png 846 | | | ├── aachen_000001_000019_leftImg8bit.png 847 | | | ├── ... 848 | | ├── bochum 849 | | | ├── ... 850 | | ├── ... 851 | ├── test 852 | | ├── ... 853 | ├── val 854 | | ├── ... 855 └── gtFine 856 ├── train 857 | ├── aachen 858 | | ├── aachen_000000_000019_gtFine_color.png 859 | | ├── aachen_000000_000019_gtFine_instanceIds.png 860 | | ├── aachen_000000_000019_gtFine_labelIds.png 861 | | ├── aachen_000000_000019_gtFine_polygons.json 862 | | ├── aachen_000001_000019_gtFine_color.png 863 | | ├── aachen_000001_000019_gtFine_instanceIds.png 864 | | ├── aachen_000001_000019_gtFine_labelIds.png 865 | | ├── aachen_000001_000019_gtFine_polygons.json 866 | | ├── ... 867 | ├── bochum 868 | | ├── ... 869 | ├── ... 870 ├── test 871 | ├── ... 872 └── val 873 ├── ... 874 875 Citation: 876 877 .. code-block:: 878 879 @inproceedings{Cordts2016Cityscapes, 880 title = {The Cityscapes Dataset for Semantic Urban Scene Understanding}, 881 author = {Cordts, Marius and Omran, Mohamed and Ramos, Sebastian and Rehfeld, Timo and Enzweiler, 882 Markus and Benenson, Rodrigo and Franke, Uwe and Roth, Stefan and Schiele, Bernt}, 883 booktitle = {Proc. of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, 884 year = {2016} 885 } 886 """ 887 888 @check_cityscapes_dataset 889 def __init__(self, dataset_dir, usage="train", quality_mode="fine", task="instance", num_samples=None, 890 num_parallel_workers=None, shuffle=None, decode=None, sampler=None, num_shards=None, 891 shard_id=None, cache=None): 892 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 893 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 894 895 self.dataset_dir = dataset_dir 896 self.task = task 897 self.quality_mode = quality_mode 898 self.usage = usage 899 self.decode = replace_none(decode, False) 900 901 def parse(self, children=None): 902 return cde.CityscapesNode(self.dataset_dir, self.usage, self.quality_mode, self.task, self.decode, self.sampler) 903 904 905class CocoDataset(MappableDataset, VisionBaseDataset): 906 """ 907 COCO(Common Objects in Context) dataset. 908 909 CocoDataset supports five kinds of tasks, which are Object Detection, Keypoint Detection, Stuff Segmentation, 910 Panoptic Segmentation and Captioning of 2017 Train/Val/Test dataset. 911 912 Args: 913 dataset_dir (str): Path to the root directory that contains the dataset. 914 annotation_file (str): Path to the annotation JSON file. 915 task (str, optional): Set the task type for reading COCO data. Supported task types: 916 ``'Detection'``, ``'Stuff'``, ``'Panoptic'``, ``'Keypoint'`` and ``'Captioning'``. 917 Default: ``'Detection'``. 918 num_samples (int, optional): The number of images to be included in the dataset. 919 Default: ``None`` , all images. 920 num_parallel_workers (int, optional): Number of worker threads to read the data. 921 Default: ``None`` , will use global default workers(8), it can be set 922 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 923 shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected 924 order behavior shown in the table below. 925 decode (bool, optional): Decode the images after reading. Default: ``False``. 926 sampler (Sampler, optional): Object used to choose samples from the dataset. 927 Default: ``None`` , expected order behavior shown in the table below. 928 num_shards (int, optional): Number of shards that the dataset will be divided 929 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 930 the maximum sample number of per shard. 931 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 932 argument can only be specified when `num_shards` is also specified. 933 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 934 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 935 Default: ``None`` , which means no cache is used. 936 extra_metadata(bool, optional): Flag to add extra meta-data to row. If True, an additional column will be 937 output at the end :py:obj:`[_meta-filename, dtype=string]` . Default: ``False``. 938 decrypt (callable, optional): Image decryption function, which accepts the path of the encrypted image file 939 and returns the decrypted bytes data. Default: ``None`` , no decryption. 940 941 The generated dataset with different task setting has different output columns: 942 943 +-------------------------+----------------------------------------------+ 944 | `task` | Output column | 945 +=========================+==============================================+ 946 | Detection | [image, dtype=uint8] | 947 | | | 948 | | [bbox, dtype=float32] | 949 | | | 950 | | [category_id, dtype=uint32] | 951 | | | 952 | | [iscrowd, dtype=uint32] | 953 +-------------------------+----------------------------------------------+ 954 | Stuff | [image, dtype=uint8] | 955 | | | 956 | | [segmentation, dtype=float32] | 957 | | | 958 | | [iscrowd, dtype=uint32] | 959 +-------------------------+----------------------------------------------+ 960 | Keypoint | [image, dtype=uint8] | 961 | | | 962 | | [keypoints, dtype=float32] | 963 | | | 964 | | [num_keypoints, dtype=uint32] | 965 +-------------------------+----------------------------------------------+ 966 | Panoptic | [image, dtype=uint8] | 967 | | | 968 | | [bbox, dtype=float32] | 969 | | | 970 | | [category_id, dtype=uint32] | 971 | | | 972 | | [iscrowd, dtype=uint32] | 973 | | | 974 | | [area, dtype=uint32] | 975 +-------------------------+----------------------------------------------+ 976 | Captioning | [image, dtype=uint8] | 977 | | | 978 | | [captions, dtype=string] | 979 +-------------------------+----------------------------------------------+ 980 981 Raises: 982 RuntimeError: If `dataset_dir` does not contain data files. 983 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 984 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 985 RuntimeError: If `num_shards` is specified but `shard_id` is None. 986 RuntimeError: If `shard_id` is specified but `num_shards` is None. 987 RuntimeError: If parse JSON file failed. 988 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 989 ValueError: If `task` is not ``'Detection'``, ``'Stuff'``, ``'Panoptic'``, ``'Keypoint'`` 990 or ``'Captioning'``. 991 ValueError: If `annotation_file` is not exist. 992 ValueError: If `dataset_dir` is not exist. 993 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 994 995 Tutorial Examples: 996 - `Load & Process Data With Dataset Pipeline 997 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 998 999 Note: 1000 - Column '[_meta-filename, dtype=string]' won't be output unless an explicit rename dataset op is added 1001 to remove the prefix('_meta-'). 1002 - Not support :class:`mindspore.dataset.PKSampler` for `sampler` parameter yet. 1003 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 1004 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 1005 1006 .. include:: mindspore.dataset.sampler.txt 1007 1008 Examples: 1009 >>> import mindspore.dataset as ds 1010 >>> coco_dataset_dir = "/path/to/coco_dataset_directory/images" 1011 >>> coco_annotation_file = "/path/to/coco_dataset_directory/annotation_file" 1012 >>> 1013 >>> # 1) Read COCO data for Detection task 1014 >>> dataset = ds.CocoDataset(dataset_dir=coco_dataset_dir, 1015 ... annotation_file=coco_annotation_file, 1016 ... task='Detection') 1017 >>> 1018 >>> # 2) Read COCO data for Stuff task 1019 >>> dataset = ds.CocoDataset(dataset_dir=coco_dataset_dir, 1020 ... annotation_file=coco_annotation_file, 1021 ... task='Stuff') 1022 >>> 1023 >>> # 3) Read COCO data for Panoptic task 1024 >>> dataset = ds.CocoDataset(dataset_dir=coco_dataset_dir, 1025 ... annotation_file=coco_annotation_file, 1026 ... task='Panoptic') 1027 >>> 1028 >>> # 4) Read COCO data for Keypoint task 1029 >>> dataset = ds.CocoDataset(dataset_dir=coco_dataset_dir, 1030 ... annotation_file=coco_annotation_file, 1031 ... task='Keypoint') 1032 >>> 1033 >>> # 5) Read COCO data for Captioning task 1034 >>> dataset = ds.CocoDataset(dataset_dir=coco_dataset_dir, 1035 ... annotation_file=coco_annotation_file, 1036 ... task='Captioning') 1037 >>> 1038 >>> # In COCO dataset, each dictionary has keys "image" and "annotation" 1039 1040 About COCO dataset: 1041 1042 COCO(Microsoft Common Objects in Context) is a large-scale object detection, segmentation, and captioning dataset 1043 with several features: Object segmentation, Recognition in context, Superpixel stuff segmentation, 1044 330K images (>200K labeled), 1.5 million object instances, 80 object categories, 91 stuff categories, 1045 5 captions per image, 250,000 people with keypoints. In contrast to the popular ImageNet dataset, COCO has fewer 1046 categories but more instances in per category. 1047 1048 You can unzip the original COCO-2017 dataset files into this directory structure and read by MindSpore's API. 1049 1050 .. code-block:: 1051 1052 . 1053 └── coco_dataset_directory 1054 ├── train2017 1055 │ ├── 000000000009.jpg 1056 │ ├── 000000000025.jpg 1057 │ ├── ... 1058 ├── test2017 1059 │ ├── 000000000001.jpg 1060 │ ├── 000000058136.jpg 1061 │ ├── ... 1062 ├── val2017 1063 │ ├── 000000000139.jpg 1064 │ ├── 000000057027.jpg 1065 │ ├── ... 1066 └── annotations 1067 ├── captions_train2017.json 1068 ├── captions_val2017.json 1069 ├── instances_train2017.json 1070 ├── instances_val2017.json 1071 ├── person_keypoints_train2017.json 1072 └── person_keypoints_val2017.json 1073 1074 Citation: 1075 1076 .. code-block:: 1077 1078 @article{DBLP:journals/corr/LinMBHPRDZ14, 1079 author = {Tsung{-}Yi Lin and Michael Maire and Serge J. Belongie and 1080 Lubomir D. Bourdev and Ross B. Girshick and James Hays and 1081 Pietro Perona and Deva Ramanan and Piotr Doll{\'{a}}r and C. Lawrence Zitnick}, 1082 title = {Microsoft {COCO:} Common Objects in Context}, 1083 journal = {CoRR}, 1084 volume = {abs/1405.0312}, 1085 year = {2014}, 1086 url = {http://arxiv.org/abs/1405.0312}, 1087 archivePrefix = {arXiv}, 1088 eprint = {1405.0312}, 1089 timestamp = {Mon, 13 Aug 2018 16:48:13 +0200}, 1090 biburl = {https://dblp.org/rec/journals/corr/LinMBHPRDZ14.bib}, 1091 bibsource = {dblp computer science bibliography, https://dblp.org} 1092 } 1093 """ 1094 1095 @check_cocodataset 1096 def __init__(self, dataset_dir, annotation_file, task="Detection", num_samples=None, num_parallel_workers=None, 1097 shuffle=None, decode=False, sampler=None, num_shards=None, shard_id=None, cache=None, 1098 extra_metadata=False, decrypt=None): 1099 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 1100 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 1101 self.dataset_dir = dataset_dir 1102 self.annotation_file = annotation_file 1103 self.task = replace_none(task, "Detection") 1104 self.decode = replace_none(decode, False) 1105 self.extra_metadata = extra_metadata 1106 self.decrypt = decrypt 1107 1108 def parse(self, children=None): 1109 return cde.CocoNode(self.dataset_dir, self.annotation_file, self.task, self.decode, self.sampler, 1110 self.extra_metadata, self.decrypt) 1111 1112 def get_class_indexing(self): 1113 """ 1114 Get the mapping dictionary from category names to category indexes. 1115 1116 This dictionary can be used to look up which category name corresponds to a particular category index. 1117 1118 Returns: 1119 Dict[str, List[int]], the mappings from category names to category index list. The first 1120 element of the list is always the category ID. Only in Panoptic tasks, the second element 1121 of the list indicates whether the category is a thing or a stuff. 1122 1123 Examples: 1124 >>> import mindspore.dataset as ds 1125 >>> coco_dataset_dir = "/path/to/coco_dataset_directory/images" 1126 >>> coco_annotation_file = "/path/to/coco_dataset_directory/annotation_file" 1127 >>> 1128 >>> # Read COCO data for Detection task 1129 >>> dataset = ds.CocoDataset(dataset_dir=coco_dataset_dir, 1130 ... annotation_file=coco_annotation_file, 1131 ... task='Detection') 1132 >>> 1133 >>> class_indexing = dataset.get_class_indexing() 1134 """ 1135 if self.task not in {"Detection", "Panoptic"}: 1136 raise NotImplementedError("Only 'Detection' and 'Panoptic' support get_class_indexing.") 1137 if self._class_indexing is None: 1138 runtime_getter = self._init_tree_getters() 1139 self._class_indexing = dict(runtime_getter[0].GetClassIndexing()) 1140 return self._class_indexing 1141 1142 1143class DIV2KDataset(MappableDataset, VisionBaseDataset): 1144 """ 1145 DIV2K(DIVerse 2K resolution image) dataset. 1146 1147 The generated dataset has two columns :py:obj:`[hr_image, lr_image]` . 1148 The tensor of column :py:obj:`hr_image` and the tensor of column :py:obj:`lr_image` are of the uint8 type. 1149 1150 Args: 1151 dataset_dir (str): Path to the root directory that contains the dataset. 1152 usage (str, optional): Acceptable usages include ``'train'``, ``'valid'`` or ``'all'``. 1153 Default: ``'train'``. 1154 downgrade (str, optional): Acceptable downgrades include ``'bicubic'``, ``'unknown'``, ``'mild'``, 1155 ``'difficult'`` or ``'wild'``. Default: ``'bicubic'``. 1156 scale (int, optional): Acceptable scales include ``2``, ``3``, ``4`` or ``8``. Default: ``2``. 1157 When `downgrade` is ``'bicubic'``, scale can be ``2``, ``3``, ``4``, ``8``. 1158 When `downgrade` is ``'unknown'``, scale can only be ``2``, ``3``, ``4``. 1159 When `downgrade` is ``'mild'``, ``'difficult'`` or ``'wild'``, scale can only be ``4``. 1160 num_samples (int, optional): The number of images to be included in the dataset. 1161 Default: ``None`` , all images. 1162 num_parallel_workers (int, optional): Number of worker threads to read the data. 1163 Default: ``None`` , will use global default workers(8), it can be set 1164 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 1165 shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected 1166 order behavior shown in the table below. 1167 decode (bool, optional): Decode the images after reading. Default: ``None``, set to ``False``. 1168 sampler (Sampler, optional): Object used to choose samples from the 1169 dataset. Default: ``None`` , expected order behavior shown in the table below. 1170 num_shards (int, optional): Number of shards that the dataset will be divided 1171 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 1172 the max sample number of per shard. 1173 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 1174 argument can only be specified when `num_shards` is also specified. 1175 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 1176 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 1177 Default: ``None`` , which means no cache is used. 1178 1179 Raises: 1180 RuntimeError: If `dataset_dir` is invalid or does not contain data files. 1181 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 1182 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 1183 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 1184 RuntimeError: If `num_shards` is specified but `shard_id` is None. 1185 RuntimeError: If `shard_id` is specified but `num_shards` is None. 1186 ValueError: If `dataset_dir` is not exist. 1187 ValueError: If `usage` is invalid. 1188 ValueError: If `downgrade` is invalid. 1189 ValueError: If `scale` is invalid. 1190 ValueError: If `scale` equal to ``8`` and downgrade not equal to ``'bicubic'``. 1191 ValueError: If `downgrade` is ``'mild'``, ``'difficult'`` or ``'wild'``, and `scale` 1192 not equal to ``4``. 1193 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 1194 1195 Tutorial Examples: 1196 - `Load & Process Data With Dataset Pipeline 1197 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 1198 1199 Note: 1200 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 1201 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 1202 1203 .. include:: mindspore.dataset.sampler.txt 1204 1205 Examples: 1206 >>> import mindspore.dataset as ds 1207 >>> div2k_dataset_dir = "/path/to/div2k_dataset_directory" 1208 >>> 1209 >>> # 1) Get all samples from DIV2K dataset in sequence 1210 >>> dataset = ds.DIV2KDataset(dataset_dir=div2k_dataset_dir, usage="train", scale=2, downgrade="bicubic", 1211 ... shuffle=False) 1212 >>> 1213 >>> # 2) Randomly select 350 samples from DIV2K dataset 1214 >>> dataset = ds.DIV2KDataset(dataset_dir=div2k_dataset_dir, usage="train", scale=2, downgrade="bicubic", 1215 ... num_samples=350, shuffle=True) 1216 >>> 1217 >>> # 3) Get samples from DIV2K dataset for shard 0 in a 2-way distributed training 1218 >>> dataset = ds.DIV2KDataset(dataset_dir=div2k_dataset_dir, usage="train", scale=2, downgrade="bicubic", 1219 ... num_shards=2, shard_id=0) 1220 >>> 1221 >>> # In DIV2K dataset, each dictionary has keys "hr_image" and "lr_image" 1222 1223 About DIV2K dataset: 1224 1225 The DIV2K dataset consists of 1000 2K resolution images, among which 800 images are for training, 100 images 1226 are for validation and 100 images are for testing. NTIRE 2017 and NTIRE 2018 include only training dataset 1227 and validation dataset. 1228 1229 You can unzip the dataset files into the following directory structure and read by MindSpore's API. 1230 1231 Take the training set as an example. 1232 1233 .. code-block:: 1234 1235 . 1236 └── DIV2K 1237 ├── DIV2K_train_HR 1238 | ├── 0001.png 1239 | ├── 0002.png 1240 | ├── ... 1241 ├── DIV2K_train_LR_bicubic 1242 | ├── X2 1243 | | ├── 0001x2.png 1244 | | ├── 0002x2.png 1245 | | ├── ... 1246 | ├── X3 1247 | | ├── 0001x3.png 1248 | | ├── 0002x3.png 1249 | | ├── ... 1250 | └── X4 1251 | ├── 0001x4.png 1252 | ├── 0002x4.png 1253 | ├── ... 1254 ├── DIV2K_train_LR_unknown 1255 | ├── X2 1256 | | ├── 0001x2.png 1257 | | ├── 0002x2.png 1258 | | ├── ... 1259 | ├── X3 1260 | | ├── 0001x3.png 1261 | | ├── 0002x3.png 1262 | | ├── ... 1263 | └── X4 1264 | ├── 0001x4.png 1265 | ├── 0002x4.png 1266 | ├── ... 1267 ├── DIV2K_train_LR_mild 1268 | ├── 0001x4m.png 1269 | ├── 0002x4m.png 1270 | ├── ... 1271 ├── DIV2K_train_LR_difficult 1272 | ├── 0001x4d.png 1273 | ├── 0002x4d.png 1274 | ├── ... 1275 ├── DIV2K_train_LR_wild 1276 | ├── 0001x4w.png 1277 | ├── 0002x4w.png 1278 | ├── ... 1279 └── DIV2K_train_LR_x8 1280 ├── 0001x8.png 1281 ├── 0002x8.png 1282 ├── ... 1283 1284 Citation: 1285 1286 .. code-block:: 1287 1288 @InProceedings{Agustsson_2017_CVPR_Workshops, 1289 author = {Agustsson, Eirikur and Timofte, Radu}, 1290 title = {NTIRE 2017 Challenge on Single Image Super-Resolution: Dataset and Study}, 1291 booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR) Workshops}, 1292 url = "http://www.vision.ee.ethz.ch/~timofter/publications/Agustsson-CVPRW-2017.pdf", 1293 month = {July}, 1294 year = {2017} 1295 } 1296 """ 1297 1298 @check_div2k_dataset 1299 def __init__(self, dataset_dir, usage="train", downgrade="bicubic", scale=2, num_samples=None, 1300 num_parallel_workers=None, shuffle=None, decode=None, sampler=None, num_shards=None, 1301 shard_id=None, cache=None): 1302 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 1303 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 1304 1305 self.dataset_dir = dataset_dir 1306 self.usage = usage 1307 self.scale = scale 1308 self.downgrade = downgrade 1309 self.decode = replace_none(decode, False) 1310 1311 def parse(self, children=None): 1312 return cde.DIV2KNode(self.dataset_dir, self.usage, self.downgrade, self.scale, self.decode, self.sampler) 1313 1314 1315class EMnistDataset(MappableDataset, VisionBaseDataset): 1316 """ 1317 EMNIST(Extended MNIST) dataset. 1318 1319 The generated dataset has two columns :py:obj:`[image, label]` . 1320 The tensor of column :py:obj:`image` is of the uint8 type. 1321 The tensor of column :py:obj:`label` is a scalar of the uint32 type. 1322 1323 Args: 1324 dataset_dir (str): Path to the root directory that contains the dataset. 1325 name (str): Name of splits for this dataset, can be ``'byclass'``, ``'bymerge'``, ``'balanced'``, 1326 ``'letters'``, ``'digits'`` or ``'mnist'``. 1327 usage (str, optional): Usage of this dataset, can be ``'train'``, ``'test'`` or ``'all'``. 1328 ``'train'`` will read from 60,000 train samples, ``'test'`` will read from 10,000 test samples, 1329 ``'all'`` will read from all 70,000 samples. Default: ``None`` , will read all samples. 1330 num_samples (int, optional): The number of images to be included in the dataset. 1331 Default: ``None`` , will read all images. 1332 num_parallel_workers (int, optional): Number of worker threads to read the data. 1333 Default: ``None`` , will use global default workers(8), it can be set 1334 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 1335 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 1336 Default: ``None`` , expected order behavior shown in the table below. 1337 sampler (Sampler, optional): Object used to choose samples from the 1338 dataset. Default: ``None`` , expected order behavior shown in the table below. 1339 num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` . 1340 When this argument is specified, `num_samples` reflects the max sample number of per shard. 1341 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 1342 argument can only be specified when `num_shards` is also specified. 1343 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 1344 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 1345 Default: ``None`` , which means no cache is used. 1346 1347 Raises: 1348 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 1349 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 1350 RuntimeError: If `num_shards` is specified but `shard_id` is None. 1351 RuntimeError: If `shard_id` is specified but `num_shards` is None. 1352 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 1353 1354 Tutorial Examples: 1355 - `Load & Process Data With Dataset Pipeline 1356 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 1357 1358 Note: 1359 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 1360 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 1361 1362 .. include:: mindspore.dataset.sampler.txt 1363 1364 Examples: 1365 >>> import mindspore.dataset as ds 1366 >>> emnist_dataset_dir = "/path/to/emnist_dataset_directory" 1367 >>> 1368 >>> # Read 3 samples from EMNIST dataset 1369 >>> dataset = ds.EMnistDataset(dataset_dir=emnist_dataset_dir, name="mnist", num_samples=3) 1370 >>> 1371 >>> # Note: In emnist_dataset dataset, each dictionary has keys "image" and "label" 1372 1373 About EMNIST dataset: 1374 1375 The EMNIST dataset is a set of handwritten character digits derived from the NIST Special 1376 Database 19 and converted to a 28x28 pixel image format and dataset structure that directly 1377 matches the MNIST dataset. Further information on the dataset contents and conversion process 1378 can be found in the paper available at https://arxiv.org/abs/1702.05373v1. 1379 1380 The numbers of characters and classes of each split of EMNIST are as follows: 1381 1382 By Class: 814,255 characters and 62 unbalanced classes. 1383 By Merge: 814,255 characters and 47 unbalanced classes. 1384 Balanced: 131,600 characters and 47 balanced classes. 1385 Letters: 145,600 characters and 26 balanced classes. 1386 Digits: 280,000 characters and 10 balanced classes. 1387 MNIST: 70,000 characters and 10 balanced classes. 1388 1389 Here is the original EMNIST dataset structure. 1390 You can unzip the dataset files into this directory structure and read by MindSpore's API. 1391 1392 .. code-block:: 1393 1394 . 1395 └── mnist_dataset_dir 1396 ├── emnist-mnist-train-images-idx3-ubyte 1397 ├── emnist-mnist-train-labels-idx1-ubyte 1398 ├── emnist-mnist-test-images-idx3-ubyte 1399 ├── emnist-mnist-test-labels-idx1-ubyte 1400 ├── ... 1401 1402 Citation: 1403 1404 .. code-block:: 1405 1406 @article{cohen_afshar_tapson_schaik_2017, 1407 title = {EMNIST: Extending MNIST to handwritten letters}, 1408 DOI = {10.1109/ijcnn.2017.7966217}, 1409 journal = {2017 International Joint Conference on Neural Networks (IJCNN)}, 1410 author = {Cohen, Gregory and Afshar, Saeed and Tapson, Jonathan and Schaik, Andre Van}, 1411 year = {2017}, 1412 howpublished = {https://www.westernsydney.edu.au/icns/reproducible_research/ 1413 publication_support_materials/emnist} 1414 } 1415 """ 1416 1417 @check_emnist_dataset 1418 def __init__(self, dataset_dir, name, usage=None, num_samples=None, num_parallel_workers=None, 1419 shuffle=None, sampler=None, num_shards=None, shard_id=None, cache=None): 1420 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 1421 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 1422 1423 self.dataset_dir = dataset_dir 1424 self.name = name 1425 self.usage = replace_none(usage, "all") 1426 1427 def parse(self, children=None): 1428 return cde.EMnistNode(self.dataset_dir, self.name, self.usage, self.sampler) 1429 1430 1431class FakeImageDataset(MappableDataset, VisionBaseDataset): 1432 """ 1433 A source dataset for generating fake images. 1434 1435 The generated dataset has two columns :py:obj:`[image, label]` . 1436 The tensor of column :py:obj:`image` is of the uint8 type. 1437 The column :py:obj:`label` is a scalar of the uint32 type. 1438 1439 Args: 1440 num_images (int, optional): Number of images to generate in the dataset. Default: ``1000``. 1441 image_size (tuple, optional): Size of the fake image. Default: ``(224, 224, 3)``. 1442 num_classes (int, optional): Number of classes in the dataset. Default: ``10``. 1443 base_seed (int, optional): Offsets the index-based random seed used to generate each image. 1444 Default: ``0``. 1445 num_samples (int, optional): The number of images to be included in the dataset. 1446 Default: ``None`` , will read all images. 1447 num_parallel_workers (int, optional): Number of worker threads to read the data. 1448 Default: ``None`` , will use global default workers(8), it can be set 1449 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 1450 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 1451 Default: ``None`` , expected order behavior shown in the table below. 1452 sampler (Sampler, optional): Object used to choose samples from the 1453 dataset. Default: ``None`` , expected order behavior shown in the table below. 1454 num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` . 1455 When this argument is specified, `num_samples` reflects the max sample number of per shard. 1456 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 1457 argument can only be specified when `num_shards` is also specified. 1458 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 1459 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 1460 Default: ``None`` , which means no cache is used. 1461 1462 Raises: 1463 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 1464 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 1465 RuntimeError: If `num_shards` is specified but `shard_id` is None. 1466 RuntimeError: If `shard_id` is specified but `num_shards` is None. 1467 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 1468 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 1469 1470 Tutorial Examples: 1471 - `Load & Process Data With Dataset Pipeline 1472 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 1473 1474 Note: 1475 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 1476 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 1477 1478 .. include:: mindspore.dataset.sampler.txt 1479 1480 Examples: 1481 >>> import mindspore.dataset as ds 1482 >>> # Read 3 samples from FakeImage dataset 1483 >>> dataset = ds.FakeImageDataset(num_images=1000, image_size=(224,224,3), 1484 ... num_classes=10, base_seed=0, num_samples=3) 1485 """ 1486 1487 @check_fake_image_dataset 1488 def __init__(self, num_images=1000, image_size=(224, 224, 3), num_classes=10, base_seed=0, num_samples=None, 1489 num_parallel_workers=None, shuffle=None, sampler=None, num_shards=None, shard_id=None, cache=None): 1490 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 1491 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 1492 1493 self.num_images = num_images 1494 self.image_size = image_size 1495 self.num_classes = num_classes 1496 self.base_seed = base_seed 1497 1498 def parse(self, children=None): 1499 return cde.FakeImageNode(self.num_images, self.image_size, self.num_classes, self.base_seed, self.sampler) 1500 1501 1502class FashionMnistDataset(MappableDataset, VisionBaseDataset): 1503 """ 1504 Fashion-MNIST dataset. 1505 1506 The generated dataset has two columns :py:obj:`[image, label]` . 1507 The tensor of column :py:obj:`image` is of the uint8 type. 1508 The column :py:obj:`label` is a scalar of the uint32 type. 1509 1510 Args: 1511 dataset_dir (str): Path to the root directory that contains the dataset. 1512 usage (str, optional): Usage of this dataset, can be ``'train'``, ``'test'`` or ``'all'`` . 1513 ``'train'`` will read from 60,000 train samples, ``'test'`` will read from 10,000 test 1514 samples, ``'all'`` will read from all 70,000 samples. 1515 Default: ``None`` , will read all samples. 1516 num_samples (int, optional): The number of images to be included in the dataset. 1517 Default: ``None`` , will read all images. 1518 num_parallel_workers (int, optional): Number of worker threads to read the data. 1519 Default: ``None`` , will use global default workers(8), it can be set 1520 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 1521 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 1522 Default: ``None`` , expected order behavior shown in the table below. 1523 sampler (Sampler, optional): Object used to choose samples from the dataset. 1524 Default: ``None`` , expected order behavior shown in the table below. 1525 num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` . 1526 When this argument is specified, `num_samples` reflects the maximum sample number of per shard. 1527 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 1528 argument can only be specified when `num_shards` is also specified. 1529 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 1530 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 1531 Default: ``None`` , which means no cache is used. 1532 1533 Raises: 1534 RuntimeError: If `dataset_dir` does not contain data files. 1535 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 1536 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 1537 RuntimeError: If `num_shards` is specified but `shard_id` is None. 1538 RuntimeError: If `shard_id` is specified but `num_shards` is None. 1539 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 1540 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 1541 1542 Tutorial Examples: 1543 - `Load & Process Data With Dataset Pipeline 1544 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 1545 1546 Note: 1547 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 1548 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 1549 1550 .. include:: mindspore.dataset.sampler.txt 1551 1552 Examples: 1553 >>> import mindspore.dataset as ds 1554 >>> fashion_mnist_dataset_dir = "/path/to/fashion_mnist_dataset_directory" 1555 >>> 1556 >>> # Read 3 samples from FASHIONMNIST dataset 1557 >>> dataset = ds.FashionMnistDataset(dataset_dir=fashion_mnist_dataset_dir, num_samples=3) 1558 >>> 1559 >>> # Note: In FASHIONMNIST dataset, each dictionary has keys "image" and "label" 1560 1561 About Fashion-MNIST dataset: 1562 1563 Fashion-MNIST is a dataset of Zalando's article images—consisting of a training set of 60,000 examples and 1564 a test set of 10,000 examples. Each example is a 28x28 grayscale image, associated with a label from 10 classes. 1565 We intend Fashion-MNIST to serve as a direct drop-in replacement for the original MNIST dataset for benchmarking 1566 machine learning algorithms. It shares the same image size and structure of training and testing splits. 1567 1568 You can unzip the dataset files into this directory structure and read by MindSpore's API. 1569 1570 .. code-block:: 1571 1572 . 1573 └── fashionmnist_dataset_dir 1574 ├── t10k-images-idx3-ubyte 1575 ├── t10k-labels-idx1-ubyte 1576 ├── train-images-idx3-ubyte 1577 └── train-labels-idx1-ubyte 1578 1579 Citation: 1580 1581 .. code-block:: 1582 1583 @online{xiao2017/online, 1584 author = {Han Xiao and Kashif Rasul and Roland Vollgraf}, 1585 title = {Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms}, 1586 date = {2017-08-28}, 1587 year = {2017}, 1588 eprintclass = {cs.LG}, 1589 eprinttype = {arXiv}, 1590 eprint = {cs.LG/1708.07747}, 1591 } 1592 """ 1593 1594 @check_mnist_cifar_dataset 1595 def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=None, shuffle=None, 1596 sampler=None, num_shards=None, shard_id=None, cache=None): 1597 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 1598 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 1599 1600 self.dataset_dir = dataset_dir 1601 self.usage = replace_none(usage, "all") 1602 1603 def parse(self, children=None): 1604 return cde.FashionMnistNode(self.dataset_dir, self.usage, self.sampler) 1605 1606 1607class FlickrDataset(MappableDataset, VisionBaseDataset): 1608 """ 1609 Flickr8k and Flickr30k datasets. 1610 1611 The generated dataset has two columns :py:obj:`[image, annotation]` . 1612 The tensor of column :py:obj:`image` is of the uint8 type. 1613 The tensor of column :py:obj:`annotation` is a tensor which contains 5 annotations string, 1614 such as ["a", "b", "c", "d", "e"]. 1615 1616 Args: 1617 dataset_dir (str): Path to the root directory that contains the dataset. 1618 annotation_file (str): Path to the root directory that contains the annotation. 1619 num_samples (int, optional): The number of images to be included in the dataset. 1620 Default: ``None`` , all images. 1621 num_parallel_workers (int, optional): Number of worker threads to read the data. 1622 Default: ``None`` , will use global default workers(8), it can be set 1623 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 1624 shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected 1625 order behavior shown in the table below. 1626 decode (bool, optional): Decode the images after reading. Default: ``None`` . 1627 sampler (Sampler, optional): Object used to choose samples from the 1628 dataset. Default: ``None`` , expected order behavior shown in the table below. 1629 num_shards (int, optional): Number of shards that the dataset will be divided 1630 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 1631 the max sample number of per shard. 1632 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 1633 argument can only be specified when `num_shards` is also specified. 1634 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 1635 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 1636 Default: ``None`` , which means no cache is used. 1637 1638 Raises: 1639 RuntimeError: If `dataset_dir` is not valid or does not contain data files. 1640 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 1641 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 1642 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 1643 RuntimeError: If `num_shards` is specified but `shard_id` is None. 1644 RuntimeError: If `shard_id` is specified but `num_shards` is None. 1645 ValueError: If `dataset_dir` is not exist. 1646 ValueError: If `annotation_file` is not exist. 1647 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 1648 1649 Tutorial Examples: 1650 - `Load & Process Data With Dataset Pipeline 1651 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 1652 1653 Note: 1654 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 1655 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 1656 1657 .. include:: mindspore.dataset.sampler.txt 1658 1659 Examples: 1660 >>> import mindspore.dataset as ds 1661 >>> flickr_dataset_dir = "/path/to/flickr_dataset_directory" 1662 >>> annotation_file = "/path/to/flickr_annotation_file" 1663 >>> 1664 >>> # 1) Get all samples from FLICKR dataset in sequence 1665 >>> dataset = ds.FlickrDataset(dataset_dir=flickr_dataset_dir, 1666 ... annotation_file=annotation_file, 1667 ... shuffle=False) 1668 >>> 1669 >>> # 2) Randomly select 350 samples from FLICKR dataset 1670 >>> dataset = ds.FlickrDataset(dataset_dir=flickr_dataset_dir, 1671 ... annotation_file=annotation_file, 1672 ... num_samples=350, 1673 ... shuffle=True) 1674 >>> 1675 >>> # 3) Get samples from FLICKR dataset for shard 0 in a 2-way distributed training 1676 >>> dataset = ds.FlickrDataset(dataset_dir=flickr_dataset_dir, 1677 ... annotation_file=annotation_file, 1678 ... num_shards=2, 1679 ... shard_id=0) 1680 >>> 1681 >>> # In FLICKR dataset, each dictionary has keys "image" and "annotation" 1682 1683 About Flickr8k dataset: 1684 1685 The Flickr8k dataset consists of 8092 color images. There are 40460 annotations in the Flickr8k.token.txt, 1686 each image has 5 annotations. 1687 1688 You can unzip the dataset files into the following directory structure and read by MindSpore's API. 1689 1690 .. code-block:: 1691 1692 . 1693 └── Flickr8k 1694 ├── Flickr8k_Dataset 1695 │ ├── 1000268201_693b08cb0e.jpg 1696 │ ├── 1001773457_577c3a7d70.jpg 1697 │ ├── ... 1698 └── Flickr8k.token.txt 1699 1700 Citation: 1701 1702 .. code-block:: 1703 1704 @article{DBLP:journals/jair/HodoshYH13, 1705 author = {Micah Hodosh and Peter Young and Julia Hockenmaier}, 1706 title = {Framing Image Description as a Ranking Task: Data, Models and Evaluation Metrics}, 1707 journal = {J. Artif. Intell. Res.}, 1708 volume = {47}, 1709 pages = {853--899}, 1710 year = {2013}, 1711 url = {https://doi.org/10.1613/jair.3994}, 1712 doi = {10.1613/jair.3994}, 1713 timestamp = {Mon, 21 Jan 2019 15:01:17 +0100}, 1714 biburl = {https://dblp.org/rec/journals/jair/HodoshYH13.bib}, 1715 bibsource = {dblp computer science bibliography, https://dblp.org} 1716 } 1717 1718 About Flickr30k dataset: 1719 1720 The Flickr30k dataset consists of 31783 color images. There are 158915 annotations in 1721 the results_20130124.token, each image has 5 annotations. 1722 1723 You can unzip the dataset files into the following directory structure and read by MindSpore's API. 1724 1725 .. code-block:: 1726 1727 . 1728 └── Flickr30k 1729 ├── flickr30k-images 1730 │ ├── 1000092795.jpg 1731 │ ├── 10002456.jpg 1732 │ ├── ... 1733 └── results_20130124.token 1734 1735 Citation: 1736 1737 .. code-block:: 1738 1739 @article{DBLP:journals/tacl/YoungLHH14, 1740 author = {Peter Young and Alice Lai and Micah Hodosh and Julia Hockenmaier}, 1741 title = {From image descriptions to visual denotations: New similarity metrics 1742 for semantic inference over event descriptions}, 1743 journal = {Trans. Assoc. Comput. Linguistics}, 1744 volume = {2}, 1745 pages = {67--78}, 1746 year = {2014}, 1747 url = {https://tacl2013.cs.columbia.edu/ojs/index.php/tacl/article/view/229}, 1748 timestamp = {Wed, 17 Feb 2021 21:55:25 +0100}, 1749 biburl = {https://dblp.org/rec/journals/tacl/YoungLHH14.bib}, 1750 bibsource = {dblp computer science bibliography, https://dblp.org} 1751 } 1752 """ 1753 1754 @check_flickr_dataset 1755 def __init__(self, dataset_dir, annotation_file, num_samples=None, num_parallel_workers=None, shuffle=None, 1756 decode=None, sampler=None, num_shards=None, shard_id=None, cache=None): 1757 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 1758 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 1759 1760 self.dataset_dir = dataset_dir 1761 self.annotation_file = annotation_file 1762 self.decode = replace_none(decode, False) 1763 1764 def parse(self, children=None): 1765 return cde.FlickrNode(self.dataset_dir, self.annotation_file, self.decode, self.sampler) 1766 1767 1768class _Flowers102Dataset: 1769 """ 1770 Mainly for loading Flowers102 Dataset, and return one row each time. 1771 """ 1772 1773 def __init__(self, dataset_dir, task, usage, decode): 1774 self.dataset_dir = os.path.realpath(dataset_dir) 1775 self.task = task 1776 self.usage = usage 1777 self.decode = decode 1778 1779 if self.task == "Classification": 1780 self.column_names = ["image", "label"] 1781 else: 1782 self.column_names = ["image", "segmentation", "label"] 1783 1784 labels_path = os.path.join(self.dataset_dir, "imagelabels.mat") 1785 setid_path = os.path.join(self.dataset_dir, "setid.mat") 1786 # minus one to transform 1~102 to 0 ~ 101 1787 self.labels = (loadmat(labels_path)["labels"][0] - 1).astype(np.uint32) 1788 self.setid = loadmat(setid_path) 1789 1790 if self.usage == 'train': 1791 self.indices = self.setid["trnid"][0].tolist() 1792 elif self.usage == 'test': 1793 self.indices = self.setid["tstid"][0].tolist() 1794 elif self.usage == 'valid': 1795 self.indices = self.setid["valid"][0].tolist() 1796 elif self.usage == 'all': 1797 self.indices = self.setid["trnid"][0].tolist() 1798 self.indices += self.setid["tstid"][0].tolist() 1799 self.indices += self.setid["valid"][0].tolist() 1800 else: 1801 raise ValueError("Input usage is not within the valid set of ['train', 'valid', 'test', 'all'].") 1802 1803 def __getitem__(self, index): 1804 # range: 1 ~ 8189 1805 image_path = os.path.join(self.dataset_dir, "jpg", "image_" + str(self.indices[index]).zfill(5) + ".jpg") 1806 if not os.path.exists(image_path): 1807 raise RuntimeError("Can not find image file: " + image_path) 1808 1809 if self.decode is True: 1810 image = np.asarray(Image.open(image_path).convert("RGB")) 1811 else: 1812 image = np.fromfile(image_path, dtype=np.uint8) 1813 1814 label = self.labels[self.indices[index] - 1] 1815 1816 if self.task == "Segmentation": 1817 segmentation_path = \ 1818 os.path.join(self.dataset_dir, "segmim", "segmim_" + str(self.indices[index]).zfill(5) + ".jpg") 1819 if not os.path.exists(segmentation_path): 1820 raise RuntimeError("Can not find segmentation file: " + segmentation_path) 1821 if self.decode is True: 1822 segmentation = np.asarray(Image.open(segmentation_path).convert("RGB")) 1823 else: 1824 segmentation = np.fromfile(segmentation_path, dtype=np.uint8) 1825 return image, segmentation, label 1826 1827 return image, label 1828 1829 def __len__(self): 1830 return len(self.indices) 1831 1832 1833class Flowers102Dataset(GeneratorDataset): 1834 """ 1835 Oxfird 102 Flower dataset. 1836 1837 According to the given `task` configuration, the generated dataset has different output columns: 1838 - `task` = 'Classification', output columns: `[image, dtype=uint8]` , `[label, dtype=uint32]` . 1839 - `task` = 'Segmentation', 1840 output columns: `[image, dtype=uint8]` , `[segmentation, dtype=uint8]` , `[label, dtype=uint32]` . 1841 1842 Args: 1843 dataset_dir (str): Path to the root directory that contains the dataset. 1844 task (str, optional): Specify the ``'Classification'`` or ``'Segmentation'`` task. 1845 Default: ``'Classification'``. 1846 usage (str, optional): Specify the ``'train'``, ``'valid'``, ``'test'`` part or ``'all'`` 1847 parts of dataset. Default: 'all', will read all samples. 1848 num_samples (int, optional): The number of samples to be included in the dataset. 1849 Default: ``None`` , all images. 1850 num_parallel_workers (int, optional): Number of worker subprocesses used to 1851 fetch the dataset in parallel. Default: ``1``. 1852 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 1853 Default: ``None`` , expected order behavior shown in the table below. 1854 decode (bool, optional): Whether or not to decode the images and segmentations after reading. 1855 Default: ``False``. 1856 sampler (Union[Sampler, Iterable], optional): Object used to choose samples from the dataset. 1857 Default: ``None`` , expected order behavior shown in the table below. 1858 num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` . 1859 When this argument is specified, `num_samples` reflects the max sample number of per shard. 1860 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . 1861 This argument must be specified only when `num_shards` is also specified. 1862 1863 Raises: 1864 RuntimeError: If `dataset_dir` does not contain data files. 1865 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 1866 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 1867 RuntimeError: If `num_shards` is specified but `shard_id` is None. 1868 RuntimeError: If `shard_id` is specified but `num_shards` is None. 1869 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 1870 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 1871 1872 Tutorial Examples: 1873 - `Load & Process Data With Dataset Pipeline 1874 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 1875 1876 Note: 1877 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 1878 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 1879 1880 .. include:: mindspore.dataset.sampler.txt 1881 1882 Examples: 1883 >>> import mindspore.dataset as ds 1884 >>> flowers102_dataset_dir = "/path/to/flowers102_dataset_directory" 1885 >>> dataset = ds.Flowers102Dataset(dataset_dir=flowers102_dataset_dir, 1886 ... task="Classification", 1887 ... usage="all", 1888 ... decode=True) 1889 1890 About Flowers102 dataset: 1891 1892 Flowers102 dataset consists of 102 flower categories. 1893 The flowers commonly occur in the United Kingdom. 1894 Each class consists of between 40 and 258 images. 1895 1896 Here is the original Flowers102 dataset structure. 1897 You can unzip the dataset files into this directory structure and read by MindSpore's API. 1898 1899 .. code-block:: 1900 1901 . 1902 └── flowes102_dataset_dir 1903 ├── imagelabels.mat 1904 ├── setid.mat 1905 ├── jpg 1906 ├── image_00001.jpg 1907 ├── image_00002.jpg 1908 ├── ... 1909 ├── segmim 1910 ├── segmim_00001.jpg 1911 ├── segmim_00002.jpg 1912 ├── ... 1913 1914 Citation: 1915 1916 .. code-block:: 1917 1918 @InProceedings{Nilsback08, 1919 author = "Maria-Elena Nilsback and Andrew Zisserman", 1920 title = "Automated Flower Classification over a Large Number of Classes", 1921 booktitle = "Indian Conference on Computer Vision, Graphics and Image Processing", 1922 month = "Dec", 1923 year = "2008", 1924 } 1925 """ 1926 1927 @check_flowers102dataset 1928 def __init__(self, dataset_dir, task="Classification", usage="all", num_samples=None, num_parallel_workers=1, 1929 shuffle=None, decode=False, sampler=None, num_shards=None, shard_id=None): 1930 self.dataset_dir = os.path.realpath(dataset_dir) 1931 self.task = replace_none(task, "Classification") 1932 self.usage = replace_none(usage, "all") 1933 self.decode = replace_none(decode, False) 1934 dataset = _Flowers102Dataset(self.dataset_dir, self.task, self.usage, self.decode) 1935 super().__init__(dataset, column_names=dataset.column_names, num_samples=num_samples, 1936 num_parallel_workers=num_parallel_workers, shuffle=shuffle, sampler=sampler, 1937 num_shards=num_shards, shard_id=shard_id) 1938 1939 def get_class_indexing(self): 1940 """ 1941 Get the class index. 1942 1943 Returns: 1944 dict, a str-to-int mapping from label name to index. 1945 """ 1946 class_names = [ 1947 "pink primrose", "hard-leaved pocket orchid", "canterbury bells", 1948 "sweet pea", "english marigold", "tiger lily", "moon orchid", 1949 "bird of paradise", "monkshood", "globe thistle", "snapdragon", 1950 "colt's foot", "king protea", "spear thistle", "yellow iris", 1951 "globe-flower", "purple coneflower", "peruvian lily", "balloon flower", 1952 "giant white arum lily", "fire lily", "pincushion flower", "fritillary", 1953 "red ginger", "grape hyacinth", "corn poppy", "prince of wales feathers", 1954 "stemless gentian", "artichoke", "sweet william", "carnation", 1955 "garden phlox", "love in the mist", "mexican aster", "alpine sea holly", 1956 "ruby-lipped cattleya", "cape flower", "great masterwort", "siam tulip", 1957 "lenten rose", "barbeton daisy", "daffodil", "sword lily", "poinsettia", 1958 "bolero deep blue", "wallflower", "marigold", "buttercup", "oxeye daisy", 1959 "common dandelion", "petunia", "wild pansy", "primula", "sunflower", 1960 "pelargonium", "bishop of llandaff", "gaura", "geranium", "orange dahlia", 1961 "pink-yellow dahlia?", "cautleya spicata", "japanese anemone", 1962 "black-eyed susan", "silverbush", "californian poppy", "osteospermum", 1963 "spring crocus", "bearded iris", "windflower", "tree poppy", "gazania", 1964 "azalea", "water lily", "rose", "thorn apple", "morning glory", 1965 "passion flower", "lotus", "toad lily", "anthurium", "frangipani", 1966 "clematis", "hibiscus", "columbine", "desert-rose", "tree mallow", 1967 "magnolia", "cyclamen", "watercress", "canna lily", "hippeastrum", 1968 "bee balm", "ball moss", "foxglove", "bougainvillea", "camellia", "mallow", 1969 "mexican petunia", "bromelia", "blanket flower", "trumpet creeper", 1970 "blackberry lily" 1971 ] 1972 1973 class_dict = {} 1974 for i, class_name in enumerate(class_names): 1975 class_dict[class_name] = i 1976 1977 return class_dict 1978 1979 1980class Food101Dataset(MappableDataset, VisionBaseDataset): 1981 """ 1982 Food101 dataset. 1983 1984 The generated dataset has two columns :py:obj:`[image, label]` . 1985 The tensor of column :py:obj:`image` is of the uint8 type. 1986 The tensor of column :py:obj:`label` is of the string type. 1987 1988 Args: 1989 dataset_dir (str): Path to the root directory that contains the dataset. 1990 usage (str, optional): Usage of this dataset, can be ``'train'`` , ``'test'`` , or ``'all'`` . 1991 ``'train'`` will read from 75,750 samples, ``'test'`` will read from 25,250 samples, and ``'all'`` 1992 will read all ``'train'`` and ``'test'`` samples. Default: ``None`` , will be set to ``'all'``. 1993 num_samples (int, optional): The number of images to be included in the dataset. 1994 Default: ``None`` , will read all images. 1995 num_parallel_workers (int, optional): Number of worker threads to read the data. 1996 Default: ``None`` , will use global default workers(8), it can be set 1997 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 1998 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 1999 Default: ``None`` , expected order behavior shown in the table below. 2000 decode (bool, optional): Decode the images after reading. Default: ``False``. 2001 sampler (Sampler, optional): Object used to choose samples from the dataset. 2002 Default: ``None`` , expected order behavior shown in the table below. 2003 num_shards (int, optional): Number of shards that the dataset will be divided into. When this argument 2004 is specified, `num_samples` reflects the maximum sample number of per shard. Default: ``None`` . 2005 shard_id (int, optional): The shard ID within `num_shards` . This argument can only be specified 2006 when `num_shards` is also specified. Default: ``None`` . 2007 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 2008 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 2009 Default: ``None`` , which means no cache is used. 2010 2011 Raises: 2012 RuntimeError: If `dataset_dir` does not contain data files. 2013 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 2014 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 2015 RuntimeError: If `num_shards` is specified but `shard_id` is None. 2016 RuntimeError: If `shard_id` is specified but `num_shards` is None. 2017 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 2018 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 2019 ValueError: If the value of `usage` is not ``'train'``, ``'test'``, or ``'all'``. 2020 ValueError: If `dataset_dir` is not exist. 2021 2022 Tutorial Examples: 2023 - `Load & Process Data With Dataset Pipeline 2024 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 2025 2026 Note: 2027 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 2028 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 2029 2030 .. include:: mindspore.dataset.sampler.txt 2031 2032 Examples: 2033 >>> import mindspore.dataset as ds 2034 >>> food101_dataset_dir = "/path/to/food101_dataset_directory" 2035 >>> 2036 >>> # Read 3 samples from Food101 dataset 2037 >>> dataset = ds.Food101Dataset(dataset_dir=food101_dataset_dir, num_samples=3) 2038 2039 About Food101 dataset: 2040 2041 The Food101 is a dataset of 101 food categories, with 101,000 images. 2042 There are 250 test imgaes and 750 training images in each class. All images were rescaled 2043 to have a maximum side length of 512 pixels. 2044 2045 The following is the original Food101 dataset structure. 2046 You can unzip the dataset files into this directory structure and read by MindSpore's API. 2047 2048 .. code-block:: 2049 2050 . 2051 └── food101_dir 2052 ├── images 2053 │ ├── apple_pie 2054 │ │ ├── 1005649.jpg 2055 │ │ ├── 1014775.jpg 2056 │ │ ├──... 2057 │ ├── baby_back_rips 2058 │ │ ├── 1005293.jpg 2059 │ │ ├── 1007102.jpg 2060 │ │ ├──... 2061 │ └──... 2062 └── meta 2063 ├── train.txt 2064 ├── test.txt 2065 ├── classes.txt 2066 ├── train.json 2067 ├── test.json 2068 └── train.txt 2069 2070 Citation: 2071 2072 .. code-block:: 2073 2074 @inproceedings{bossard14, 2075 title = {Food-101 -- Mining Discriminative Components with Random Forests}, 2076 author = {Bossard, Lukas and Guillaumin, Matthieu and Van Gool, Luc}, 2077 booktitle = {European Conference on Computer Vision}, 2078 year = {2014} 2079 } 2080 """ 2081 2082 @check_food101_dataset 2083 def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=None, shuffle=None, 2084 decode=False, sampler=None, num_shards=None, shard_id=None, cache=None): 2085 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 2086 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 2087 2088 self.dataset_dir = dataset_dir 2089 self.usage = replace_none(usage, "all") 2090 self.decode = replace_none(decode, False) 2091 2092 def parse(self, children=None): 2093 return cde.Food101Node(self.dataset_dir, self.usage, self.decode, self.sampler) 2094 2095 2096class ImageFolderDataset(MappableDataset, VisionBaseDataset): 2097 """ 2098 A source dataset that reads images from a tree of directories. 2099 All images within one folder have the same label. 2100 2101 The generated dataset has two columns: :py:obj:`[image, label]` . 2102 The tensor of column :py:obj:`image` is of the uint8 type. 2103 The tensor of column :py:obj:`label` is of a scalar of uint32 type. 2104 2105 Args: 2106 dataset_dir (str): Path to the root directory that contains the dataset. 2107 num_samples (int, optional): The number of images to be included in the dataset. 2108 Default: ``None`` , all images. 2109 num_parallel_workers (int, optional): Number of worker threads to read the data. 2110 Default: ``None`` , will use global default workers(8), it can be set 2111 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 2112 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 2113 Default: ``None`` , expected order behavior shown in the table below. 2114 sampler (Sampler, optional): Object used to choose samples from the 2115 dataset. Default: ``None`` , expected order behavior shown in the table below. 2116 extensions (list[str], optional): List of file extensions to be 2117 included in the dataset. Default: ``None`` . 2118 class_indexing (dict, optional): A str-to-int mapping from folder name to index 2119 Default: ``None`` , the folder names will be sorted 2120 alphabetically and each class will be given a 2121 unique index starting from 0. 2122 decode (bool, optional): Decode the images after reading. Default: ``False``. 2123 num_shards (int, optional): Number of shards that the dataset will be divided 2124 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 2125 the maximum sample number of per shard. 2126 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 2127 argument can only be specified when `num_shards` is also specified. 2128 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 2129 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 2130 Default: ``None`` , which means no cache is used. 2131 decrypt (callable, optional): Image decryption function, which accepts the path of the encrypted image file 2132 and returns the decrypted bytes data. Default: ``None`` , no decryption. 2133 2134 Raises: 2135 RuntimeError: If `dataset_dir` does not contain data files. 2136 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 2137 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 2138 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 2139 RuntimeError: If `num_shards` is specified but `shard_id` is None. 2140 RuntimeError: If `shard_id` is specified but `num_shards` is None. 2141 RuntimeError: If `class_indexing` is not a dictionary. 2142 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 2143 2144 Tutorial Examples: 2145 - `Load & Process Data With Dataset Pipeline 2146 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 2147 2148 Note: 2149 - The shape of the image column is [image_size] if `decode` flag is ``False``, or [H,W,C] otherwise. 2150 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 2151 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 2152 2153 .. include:: mindspore.dataset.sampler.txt 2154 2155 Examples: 2156 >>> import mindspore.dataset as ds 2157 >>> image_folder_dataset_dir = "/path/to/image_folder_dataset_directory" 2158 >>> 2159 >>> # 1) Read all samples (image files) in image_folder_dataset_dir with 8 threads 2160 >>> dataset = ds.ImageFolderDataset(dataset_dir=image_folder_dataset_dir, 2161 ... num_parallel_workers=8) 2162 >>> 2163 >>> # 2) Read all samples (image files) from folder cat and folder dog with label 0 and 1 2164 >>> dataset = ds.ImageFolderDataset(dataset_dir=image_folder_dataset_dir, 2165 ... class_indexing={"cat":0, "dog":1}) 2166 >>> 2167 >>> # 3) Read all samples (image files) in image_folder_dataset_dir with extensions .JPEG 2168 >>> # and .png (case sensitive) 2169 >>> dataset = ds.ImageFolderDataset(dataset_dir=image_folder_dataset_dir, 2170 ... extensions=[".JPEG", ".png"]) 2171 2172 About ImageFolderDataset: 2173 2174 You can construct the following directory structure from your dataset files and read by MindSpore's API. 2175 2176 .. code-block:: 2177 2178 . 2179 └── image_folder_dataset_directory 2180 ├── class1 2181 │ ├── 000000000001.jpg 2182 │ ├── 000000000002.jpg 2183 │ ├── ... 2184 ├── class2 2185 │ ├── 000000000001.jpg 2186 │ ├── 000000000002.jpg 2187 │ ├── ... 2188 ├── class3 2189 │ ├── 000000000001.jpg 2190 │ ├── 000000000002.jpg 2191 │ ├── ... 2192 ├── classN 2193 ├── ... 2194 """ 2195 2196 @check_imagefolderdataset 2197 def __init__(self, dataset_dir, num_samples=None, num_parallel_workers=None, shuffle=None, sampler=None, 2198 extensions=None, class_indexing=None, decode=False, num_shards=None, shard_id=None, cache=None, 2199 decrypt=None): 2200 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 2201 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 2202 2203 self.dataset_dir = dataset_dir 2204 self.extensions = replace_none(extensions, []) 2205 self.class_indexing = replace_none(class_indexing, {}) 2206 self.decode = replace_none(decode, False) 2207 self.decrypt = decrypt 2208 2209 def parse(self, children=None): 2210 return cde.ImageFolderNode(self.dataset_dir, self.decode, self.sampler, self.extensions, self.class_indexing, 2211 self.decrypt) 2212 2213 def get_class_indexing(self): 2214 """ 2215 Get the class index. 2216 2217 Returns: 2218 dict, a str-to-int mapping from label name to index. 2219 2220 Examples: 2221 >>> import mindspore.dataset as ds 2222 >>> image_folder_dataset_dir = "/path/to/image_folder_dataset_directory" 2223 >>> 2224 >>> dataset = ds.ImageFolderDataset(dataset_dir=image_folder_dataset_dir) 2225 >>> class_indexing = dataset.get_class_indexing() 2226 """ 2227 if self.class_indexing is None or not self.class_indexing: 2228 runtime_getter = self._init_tree_getters() 2229 _class_indexing = runtime_getter[0].GetClassIndexing() 2230 for pair in _class_indexing: 2231 self.class_indexing[pair[0]] = pair[1][0] 2232 return self.class_indexing 2233 2234 2235class KITTIDataset(MappableDataset, VisionBaseDataset): 2236 """ 2237 KITTI dataset. 2238 2239 When `usage` is ``"train"``, the generated dataset has multiple columns: :py:obj:`[image, label, truncated, 2240 occluded, alpha, bbox, dimensions, location, rotation_y]` ; When `usage` is "test", the generated dataset 2241 has only one column: :py:obj:`[image]` . 2242 The tensor of column :py:obj:`image` is of the uint8 type. 2243 The tensor of column :py:obj:`label` is of the uint32 type. 2244 The tensor of column :py:obj:`truncated` is of the float32 type. 2245 The tensor of column :py:obj:`occluded` is of the uint32 type. 2246 The tensor of column :py:obj:`alpha` is of the float32 type. 2247 The tensor of column :py:obj:`bbox` is of the float32 type. 2248 The tensor of column :py:obj:`dimensions` is of the float32 type. 2249 The tensor of column :py:obj:`location` is of the float32 type. 2250 The tensor of column :py:obj:`rotation_y` is of the float32 type. 2251 2252 Args: 2253 dataset_dir (str): Path to the root directory that contains the dataset. 2254 usage (str, optional): Usage of this dataset, can be ``"train"`` or ``"test"`` . 2255 ``"train"`` will read 7481 train samples, ``"test"`` will read from 7518 test samples 2256 without label. Default: ``None`` , will use ``"train"`` . 2257 num_samples (int, optional): The number of images to be included in the dataset. 2258 Default: ``None`` , will include all images. 2259 num_parallel_workers (int, optional): Number of worker threads to read the data. 2260 Default: ``None`` , will use global default workers(8), it can be set 2261 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 2262 shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected 2263 order behavior shown in the table below. 2264 decode (bool, optional): Decode the images after reading. Default: ``False``. 2265 sampler (Sampler, optional): Object used to choose samples from the dataset. 2266 Default: ``None`` , expected order behavior shown in the table below. 2267 num_shards (int, optional): Number of shards that the dataset will be divided 2268 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 2269 the max sample number of per shard. 2270 shard_id (int, optional): The shard ID within `num_shards`. Default: ``None`` . This 2271 argument can only be specified when `num_shards` is also specified. 2272 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 2273 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 2274 Default: ``None`` , which means no cache is used. 2275 2276 Raises: 2277 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 2278 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 2279 RuntimeError: If `num_shards` is specified but `shard_id` is None. 2280 RuntimeError: If `shard_id` is specified but `num_shards` is None. 2281 ValueError: If `dataset_dir` is not exist. 2282 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 2283 2284 Tutorial Examples: 2285 - `Load & Process Data With Dataset Pipeline 2286 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 2287 2288 Note: 2289 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 2290 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 2291 2292 .. include:: mindspore.dataset.sampler.txt 2293 2294 Examples: 2295 >>> import mindspore.dataset as ds 2296 >>> kitti_dataset_dir = "/path/to/kitti_dataset_directory" 2297 >>> 2298 >>> # 1) Read all KITTI train dataset samples in kitti_dataset_dir in sequence 2299 >>> dataset = ds.KITTIDataset(dataset_dir=kitti_dataset_dir, usage="train") 2300 >>> 2301 >>> # 2) Read then decode all KITTI test dataset samples in kitti_dataset_dir in sequence 2302 >>> dataset = ds.KITTIDataset(dataset_dir=kitti_dataset_dir, usage="test", 2303 ... decode=True, shuffle=False) 2304 2305 About KITTI dataset: 2306 2307 KITTI (Karlsruhe Institute of Technology and Toyota Technological Institute) is one of the most popular 2308 datasets for use in mobile robotics and autonomous driving. It consists of hours of traffic scenarios 2309 recorded with a variety of sensor modalities, including high-resolution RGB, grayscale stereo cameras, 2310 and a 3D laser scanner. Despite its popularity, the dataset itself does not contain ground truth for 2311 semantic segmentation. However, various researchers have manually annotated parts of the dataset to fit 2312 their necessities. Álvarez et al. generated ground truth for 323 images from the road detection challenge 2313 with three classes: road, vehicles and sky. Zhang et al. annotated 252 (140 for training and 112 for testing) 2314 acquisitions – RGB and Velodyne scans – from the tracking challenge for ten object categories: building, sky, 2315 road, vegetation, sidewalk, car, pedestrian, cyclist, sign/pole, and fence. 2316 2317 You can unzip the original KITTI dataset files into this directory structure and read by MindSpore's API. 2318 2319 .. code-block:: 2320 2321 . 2322 └── kitti_dataset_directory 2323 ├── data_object_image_2 2324 │ ├──training 2325 │ │ ├──image_2 2326 │ │ │ ├── 000000000001.jpg 2327 │ │ │ ├── 000000000002.jpg 2328 │ │ │ ├── ... 2329 │ ├──testing 2330 │ │ ├── image_2 2331 │ │ │ ├── 000000000001.jpg 2332 │ │ │ ├── 000000000002.jpg 2333 │ │ │ ├── ... 2334 ├── data_object_label_2 2335 │ ├──training 2336 │ │ ├──label_2 2337 │ │ │ ├── 000000000001.jpg 2338 │ │ │ ├── 000000000002.jpg 2339 │ │ │ ├── ... 2340 2341 Citation: 2342 2343 .. code-block:: 2344 2345 @INPROCEEDINGS{Geiger2012CVPR, 2346 author={Andreas Geiger and Philip Lenz and Raquel Urtasun}, 2347 title={Are we ready for Autonomous Driving? The KITTI Vision Benchmark Suite}, 2348 booktitle={Conference on Computer Vision and Pattern Recognition (CVPR)}, 2349 year={2012} 2350 } 2351 """ 2352 2353 @check_kittidataset 2354 def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=None, shuffle=None, 2355 decode=False, sampler=None, num_shards=None, shard_id=None, cache=None): 2356 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 2357 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 2358 self.dataset_dir = dataset_dir 2359 self.usage = replace_none(usage, "train") 2360 self.decode = replace_none(decode, False) 2361 2362 def parse(self, children=None): 2363 return cde.KITTINode(self.dataset_dir, self.usage, self.decode, self.sampler) 2364 2365 2366class KMnistDataset(MappableDataset, VisionBaseDataset): 2367 """ 2368 KMNIST(Kuzushiji-MNIST) dataset. 2369 2370 The generated dataset has two columns :py:obj:`[image, label]` . 2371 The tensor of column :py:obj:`image` is of the uint8 type. 2372 The column :py:obj:`label` is a scalar of the uint32 type. 2373 2374 Args: 2375 dataset_dir (str): Path to the root directory that contains the dataset. 2376 usage (str, optional): Usage of this dataset, can be ``'train'`` , ``'test'`` or ``'all'`` . 2377 ``'train'`` will read from 60,000 train samples, ``'test'`` will read from 10,000 test samples, 2378 ``'all'`` will read from all 70,000 samples. Default: ``None`` , will read all samples. 2379 num_samples (int, optional): The number of images to be included in the dataset. 2380 Default: ``None`` , will read all images. 2381 num_parallel_workers (int, optional): Number of worker threads to read the data. 2382 Default: ``None`` , will use global default workers(8), it can be set 2383 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 2384 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 2385 Default: ``None`` , expected order behavior shown in the table below. 2386 sampler (Sampler, optional): Object used to choose samples from the dataset. 2387 Default: ``None`` , expected order behavior shown in the table below. 2388 num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` . 2389 When this argument is specified, `num_samples` reflects the maximum sample number of per shard. 2390 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 2391 argument can only be specified when `num_shards` is also specified. 2392 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 2393 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 2394 Default: ``None`` , which means no cache is used. 2395 2396 Raises: 2397 RuntimeError: If `dataset_dir` does not contain data files. 2398 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 2399 RuntimeError: If `sampler` and sharding are specified at the same time. 2400 RuntimeError: If `num_shards` is specified but `shard_id` is None. 2401 RuntimeError: If `shard_id` is specified but `num_shards` is None. 2402 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 2403 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 2404 2405 Tutorial Examples: 2406 - `Load & Process Data With Dataset Pipeline 2407 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 2408 2409 Note: 2410 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 2411 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 2412 2413 .. include:: mindspore.dataset.sampler.txt 2414 2415 Examples: 2416 >>> import mindspore.dataset as ds 2417 >>> kmnist_dataset_dir = "/path/to/kmnist_dataset_directory" 2418 >>> 2419 >>> # Read 3 samples from KMNIST dataset 2420 >>> dataset = ds.KMnistDataset(dataset_dir=kmnist_dataset_dir, num_samples=3) 2421 2422 About KMNIST dataset: 2423 2424 KMNIST is a dataset, adapted from Kuzushiji Dataset, as a drop-in replacement for MNIST dataset, 2425 which is the most famous dataset in the machine learning community. 2426 2427 Here is the original KMNIST dataset structure. 2428 You can unzip the dataset files into this directory structure and read by MindSpore's API. 2429 2430 .. code-block:: 2431 2432 . 2433 └── kmnist_dataset_dir 2434 ├── t10k-images-idx3-ubyte 2435 ├── t10k-labels-idx1-ubyte 2436 ├── train-images-idx3-ubyte 2437 └── train-labels-idx1-ubyte 2438 2439 Citation: 2440 2441 .. code-block:: 2442 2443 @online{clanuwat2018deep, 2444 author = {Tarin Clanuwat and Mikel Bober-Irizar and Asanobu Kitamoto and 2445 Alex Lamb and Kazuaki Yamamoto and David Ha}, 2446 title = {Deep Learning for Classical Japanese Literature}, 2447 date = {2018-12-03}, 2448 year = {2018}, 2449 eprintclass = {cs.CV}, 2450 eprinttype = {arXiv}, 2451 eprint = {cs.CV/1812.01718}, 2452 } 2453 """ 2454 2455 @check_mnist_cifar_dataset 2456 def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=None, shuffle=None, 2457 sampler=None, num_shards=None, shard_id=None, cache=None): 2458 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 2459 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 2460 2461 self.dataset_dir = dataset_dir 2462 self.usage = replace_none(usage, "all") 2463 2464 def parse(self, children=None): 2465 return cde.KMnistNode(self.dataset_dir, self.usage, self.sampler) 2466 2467 2468class LFWDataset(MappableDataset, VisionBaseDataset): 2469 """ 2470 LFW(Labeled Faces in the Wild) dataset. 2471 2472 When `task` is 'people', the generated dataset has two columns: :py:obj:`[image, label]`; 2473 When `task` is 'pairs', the generated dataset has three columns: :py:obj:`[image1, image2, label]` . 2474 The tensor of column :py:obj:`image` is of the uint8 type. 2475 The tensor of column :py:obj:`image1` is of the uint8 type. 2476 The tensor of column :py:obj:`image2` is of the uint8 type. 2477 The tensor of column :py:obj:`label` is a scalar of the uint32 type. 2478 2479 Args: 2480 dataset_dir (str): Path to the root directory that contains the dataset. 2481 task (str, optional): Set the task type of reading lfw data, support ``'people'`` and ``'pairs'``. 2482 Default: ``None`` , means ``'people'``. 2483 usage (str, optional): The image split to use, support '``10fold'``, ``'train'``, ``'test'`` and ``'all'``. 2484 Default: ``None`` , will read samples including ``'train'`` and ``'test'``. 2485 image_set (str, optional): Type of image funneling to use, support ``'original'``, ``'funneled'`` or 2486 ``'deepfunneled'``. Default: ``None`` , will use ``'funneled'``. 2487 num_samples (int, optional): The number of images to be included in the dataset. 2488 Default: ``None`` , all images. 2489 num_parallel_workers (int, optional): Number of worker threads to read the data. 2490 Default: ``None`` , will use global default workers(8), it can be set 2491 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 2492 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 2493 Default: ``None`` , expected order behavior shown in the table below. 2494 decode (bool, optional): Decode the images after reading. Default: ``False``. 2495 sampler (Sampler, optional): Object used to choose samples from the 2496 dataset. Default: ``None`` , expected order behavior shown in the table below. 2497 num_shards (int, optional): Number of shards that the dataset will be divided 2498 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 2499 the max sample number of per shard. 2500 shard_id (int, optional): The shard ID within `num_shards`. Default: ``None`` . This 2501 argument can only be specified when `num_shards` is also specified. 2502 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 2503 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 2504 Default: ``None`` , which means no cache is used. 2505 2506 Raises: 2507 RuntimeError: If `dataset_dir` does not contain data files. 2508 RuntimeError: If sampler and shuffle are specified at the same time. 2509 RuntimeError: If sampler and sharding are specified at the same time. 2510 RuntimeError: If `num_shards` is specified but shard_id is None. 2511 RuntimeError: If `shard_id` is specified but num_shards is None. 2512 ValueError: If `shard_id` is invalid (< 0 or >= `num_shards` ). 2513 2514 Tutorial Examples: 2515 - `Load & Process Data With Dataset Pipeline 2516 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 2517 2518 Note: 2519 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 2520 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 2521 2522 .. include:: mindspore.dataset.sampler.txt 2523 2524 Examples: 2525 >>> import mindspore.dataset as ds 2526 >>> # 1) Read LFW People dataset 2527 >>> lfw_people_dataset_dir = "/path/to/lfw_people_dataset_directory" 2528 >>> dataset = ds.LFWDataset(dataset_dir=lfw_people_dataset_dir, task="people", usage="10fold", 2529 ... image_set="original") 2530 >>> 2531 >>> # 2) Read LFW Pairs dataset 2532 >>> lfw_pairs_dataset_dir = "/path/to/lfw_pairs_dataset_directory" 2533 >>> dataset = ds.LFWDataset(dataset_dir=lfw_pairs_dataset_dir, task="pairs", usage="test", image_set="funneled") 2534 2535 About LFW dataset: 2536 2537 LFW (Labelled Faces in the Wild) dataset is one of the most commonly used and widely open datasets in 2538 the field of face recognition. It was released by Gary B. Huang and his team at Massachusetts Institute 2539 of Technology in 2007. The dataset includes nearly 50,000 images of 13,233 individuals, which are sourced 2540 from various internet platforms and contain diverse environmental factors such as different poses, lighting 2541 conditions, and angles. Most of the images in the dataset are frontal and cover a wide range of ages, genders, 2542 and ethnicities. 2543 2544 You can unzip the original LFW dataset files into this directory structure and read by MindSpore's API. 2545 2546 .. code-block:: 2547 2548 . 2549 └── lfw_dataset_directory 2550 ├── lfw 2551 │ ├──Aaron_Eckhart 2552 │ │ ├──Aaron_Eckhart_0001.jpg 2553 │ │ ├──... 2554 │ ├──Abbas_Kiarostami 2555 │ │ ├── Abbas_Kiarostami_0001.jpg 2556 │ │ ├──... 2557 │ ├──... 2558 ├── lfw-deepfunneled 2559 │ ├──Aaron_Eckhart 2560 │ │ ├──Aaron_Eckhart_0001.jpg 2561 │ │ ├──... 2562 │ ├──Abbas_Kiarostami 2563 │ │ ├── Abbas_Kiarostami_0001.jpg 2564 │ │ ├──... 2565 │ ├──... 2566 ├── lfw_funneled 2567 │ ├──Aaron_Eckhart 2568 │ │ ├──Aaron_Eckhart_0001.jpg 2569 │ │ ├──... 2570 │ ├──Abbas_Kiarostami 2571 │ │ ├── Abbas_Kiarostami_0001.jpg 2572 │ │ ├──... 2573 │ ├──... 2574 ├── lfw-names.txt 2575 ├── pairs.txt 2576 ├── pairsDevTest.txt 2577 ├── pairsDevTrain.txt 2578 ├── people.txt 2579 ├── peopleDevTest.txt 2580 ├── peopleDevTrain.txt 2581 2582 Citation: 2583 2584 .. code-block:: 2585 2586 @TechReport{LFWTech, 2587 title={LFW: A Database for Studying Recognition in Unconstrained Environments}, 2588 author={Gary B. Huang and Manu Ramesh and Tamara Berg and Erik Learned-Miller}, 2589 institution ={University of Massachusetts, Amherst}, 2590 year={2007} 2591 number={07-49}, 2592 month={October}, 2593 howpublished = {http://vis-www.cs.umass.edu/lfw} 2594 } 2595 """ 2596 2597 @check_lfw_dataset 2598 def __init__(self, dataset_dir, task=None, usage=None, image_set=None, num_samples=None, num_parallel_workers=None, 2599 shuffle=None, decode=False, sampler=None, num_shards=None, shard_id=None, cache=None): 2600 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 2601 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 2602 self.dataset_dir = dataset_dir 2603 self.task = replace_none(task, "people") 2604 self.usage = replace_none(usage, "all") 2605 self.image_set = replace_none(image_set, "funneled") 2606 self.decode = replace_none(decode, False) 2607 2608 def parse(self, children=None): 2609 return cde.LFWNode(self.dataset_dir, self.task, self.usage, self.image_set, self.decode, self.sampler) 2610 2611 2612class LSUNDataset(MappableDataset, VisionBaseDataset): 2613 """ 2614 LSUN(Large-scale Scene UNderstarding) dataset. 2615 2616 The generated dataset has two columns: :py:obj:`[image, label]` . 2617 The tensor of column :py:obj:`image` is of the uint8 type. 2618 The tensor of column :py:obj:`label` is of a scalar of uint32 type. 2619 2620 Args: 2621 dataset_dir (str): Path to the root directory that contains the dataset. 2622 usage (str, optional): Usage of this dataset, can be ``"train"`` , ``"test"`` , ``"valid"`` or ``"all"`` 2623 Default: ``None`` , will be set to ``"all"`` . 2624 classes (Union[str, list[str]], optional): Choose the specific classes to load. Default: ``None`` , 2625 means loading all classes in root directory. 2626 num_samples (int, optional): The number of images to be included in the dataset. 2627 Default: ``None`` , all images. 2628 num_parallel_workers (int, optional): Number of worker threads to read the data. 2629 Default: ``None`` , will use global default workers(8), it can be set 2630 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 2631 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 2632 Default: ``None`` , expected order behavior shown in the table below. 2633 decode (bool, optional): Decode the images after reading. Default: ``False``. 2634 sampler (Sampler, optional): Object used to choose samples from the 2635 dataset. Default: ``None`` , expected order behavior shown in the table below. 2636 num_shards (int, optional): Number of shards that the dataset will be divided 2637 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 2638 the max sample number of per shard. 2639 shard_id (int, optional): The shard ID within `num_shards`. Default: ``None`` . This 2640 argument can only be specified when `num_shards` is also specified. 2641 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 2642 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 2643 Default: ``None`` , which means no cache is used. 2644 2645 Raises: 2646 RuntimeError: If `dataset_dir` does not contain data files. 2647 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 2648 RuntimeError: If `sampler` and sharding are specified at the same time. 2649 RuntimeError: If `num_shards` is specified but `shard_id` is None. 2650 RuntimeError: If `shard_id` is specified but `num_shards` is None. 2651 ValueError: If `shard_id` is invalid (< 0 or >= `num_shards` ). 2652 ValueError: If `usage` or `classes` is invalid (not in specific types). 2653 2654 Tutorial Examples: 2655 - `Load & Process Data With Dataset Pipeline 2656 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 2657 2658 Note: 2659 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 2660 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 2661 2662 .. include:: mindspore.dataset.sampler.txt 2663 2664 Examples: 2665 >>> import mindspore.dataset as ds 2666 >>> lsun_dataset_dir = "/path/to/lsun_dataset_directory" 2667 >>> 2668 >>> # 1) Read all samples (image files) in lsun_dataset_dir with 8 threads 2669 >>> dataset = ds.LSUNDataset(dataset_dir=lsun_dataset_dir, 2670 ... num_parallel_workers=8) 2671 >>> 2672 >>> # 2) Read all train samples (image files) from folder "bedroom" and "classroom" 2673 >>> dataset = ds.LSUNDataset(dataset_dir=lsun_dataset_dir, usage="train", 2674 ... classes=["bedroom", "classroom"]) 2675 2676 About LSUN dataset: 2677 2678 The LSUN (Large-Scale Scene Understanding) is a large-scale dataset used for indoors scene 2679 understanding. It was originally launched by Stanford University in 2015 with the aim of 2680 providing a challenging and diverse dataset for research in computer vision and machine 2681 learning. The main application of this dataset for research is indoor scene analysis. 2682 2683 This dataset contains ten different categories of scenes, including bedrooms, living rooms, 2684 restaurants, lounges, studies, kitchens, bathrooms, corridors, children's room, and outdoors. 2685 Each category contains tens of thousands of images from different perspectives, and these 2686 images are high-quality, high-resolusion real-world images. 2687 2688 You can unzip the dataset files into this directory structure and read by MindSpore's API. 2689 2690 .. code-block:: 2691 2692 . 2693 └── lsun_dataset_directory 2694 ├── test 2695 │ ├── ... 2696 ├── bedroom_train 2697 │ ├── 1_1.jpg 2698 │ ├── 1_2.jpg 2699 ├── bedroom_val 2700 │ ├── ... 2701 ├── classroom_train 2702 │ ├── ... 2703 ├── classroom_val 2704 │ ├── ... 2705 2706 Citation: 2707 2708 .. code-block:: 2709 2710 article{yu15lsun, 2711 title={LSUN: Construction of a Large-scale Image Dataset using Deep Learning with Humans in the Loop}, 2712 author={Yu, Fisher and Zhang, Yinda and Song, Shuran and Seff, Ari and Xiao, Jianxiong}, 2713 journal={arXiv preprint arXiv:1506.03365}, 2714 year={2015} 2715 } 2716 """ 2717 2718 @check_lsun_dataset 2719 def __init__(self, dataset_dir, usage=None, classes=None, num_samples=None, num_parallel_workers=None, 2720 shuffle=None, decode=False, sampler=None, num_shards=None, shard_id=None, cache=None): 2721 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 2722 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 2723 self.dataset_dir = dataset_dir 2724 self.usage = replace_none(usage, "all") 2725 self.classes = replace_none(classes, []) 2726 self.decode = replace_none(decode, False) 2727 2728 def parse(self, children=None): 2729 return cde.LSUNNode(self.dataset_dir, self.usage, self.classes, self.decode, self.sampler) 2730 2731 2732class ManifestDataset(MappableDataset, VisionBaseDataset): 2733 """ 2734 A source dataset for reading images from a Manifest file. 2735 2736 The generated dataset has two columns: :py:obj:`[image, label]` . 2737 The tensor of column :py:obj:`image` is of the uint8 type. 2738 The tensor of column :py:obj:`label` is of a scalar of uint64 type. 2739 2740 Args: 2741 dataset_file (str): File to be read. 2742 usage (str, optional): Acceptable usages include ``'train'``, ``'eval'`` and ``'inference'``. 2743 Default: ``'train'``. 2744 num_samples (int, optional): The number of images to be included in the dataset. 2745 Default: ``None`` , will include all images. 2746 num_parallel_workers (int, optional): Number of worker threads to read the data. 2747 Default: ``None`` , will use global default workers(8), it can be set 2748 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 2749 shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected 2750 order behavior shown in the table below. 2751 sampler (Sampler, optional): Object used to choose samples from the 2752 dataset. Default: ``None`` , expected order behavior shown in the table below. 2753 class_indexing (dict, optional): A str-to-int mapping from label name to index. 2754 Default: ``None`` , the folder names will be sorted alphabetically and each 2755 class will be given a unique index starting from 0. 2756 decode (bool, optional): decode the images after reading. Default: ``False``. 2757 num_shards (int, optional): Number of shards that the dataset will be divided 2758 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 2759 the max number of samples per shard. 2760 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 2761 argument can only be specified when `num_shards` is also specified. 2762 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 2763 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 2764 Default: ``None`` , which means no cache is used. 2765 2766 Raises: 2767 RuntimeError: If dataset_files are not valid or do not exist. 2768 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 2769 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 2770 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 2771 RuntimeError: If `num_shards` is specified but `shard_id` is None. 2772 RuntimeError: If `shard_id` is specified but `num_shards` is None. 2773 RuntimeError: If class_indexing is not a dictionary. 2774 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 2775 2776 Tutorial Examples: 2777 - `Load & Process Data With Dataset Pipeline 2778 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 2779 2780 Note: 2781 - If `decode` is ``False`` , the "image" column will get the 1D raw bytes of the image. 2782 Otherwise, a decoded image with shape :math:`[H,W,C]` will be returned. 2783 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 2784 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 2785 2786 .. include:: mindspore.dataset.sampler.txt 2787 2788 Examples: 2789 >>> import mindspore.dataset as ds 2790 >>> manifest_dataset_dir = "/path/to/manifest_dataset_file" 2791 >>> 2792 >>> # 1) Read all samples specified in manifest_dataset_dir dataset with 8 threads for training 2793 >>> dataset = ds.ManifestDataset(dataset_file=manifest_dataset_dir, usage="train", num_parallel_workers=8) 2794 >>> 2795 >>> # 2) Read samples (specified in manifest_file.manifest) for shard 0 in a 2-way distributed training setup 2796 >>> dataset = ds.ManifestDataset(dataset_file=manifest_dataset_dir, num_shards=2, shard_id=0) 2797 2798 About Manifest dataset: 2799 2800 Manifest file contains a list of files included in a dataset, including basic file info such as File name and File 2801 ID, along with extended file metadata. Manifest is a data format file supported by Huawei Modelarts. For details, 2802 see `Specifications for Importing the Manifest File <https://support.huaweicloud.com/intl/en-us/dataprepare-modelarts/ 2803 dataprepare-modelarts-0015.html>`_ . 2804 2805 .. code-block:: 2806 2807 . 2808 └── manifest_dataset_directory 2809 ├── train 2810 │ ├── 1.JPEG 2811 │ ├── 2.JPEG 2812 │ ├── ... 2813 ├── eval 2814 │ ├── 1.JPEG 2815 │ ├── 2.JPEG 2816 │ ├── ... 2817 """ 2818 2819 @check_manifestdataset 2820 def __init__(self, dataset_file, usage="train", num_samples=None, num_parallel_workers=None, shuffle=None, 2821 sampler=None, class_indexing=None, decode=False, num_shards=None, shard_id=None, cache=None): 2822 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 2823 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 2824 2825 self.dataset_file = dataset_file 2826 self.decode = replace_none(decode, False) 2827 self.usage = replace_none(usage, "train") 2828 self.class_indexing = replace_none(class_indexing, {}) 2829 2830 def parse(self, children=None): 2831 return cde.ManifestNode(self.dataset_file, self.usage, self.sampler, self.class_indexing, self.decode) 2832 2833 def get_class_indexing(self): 2834 """ 2835 Get the class index. 2836 2837 Returns: 2838 dict, a str-to-int mapping from label name to index. 2839 2840 Examples: 2841 >>> import mindspore.dataset as ds 2842 >>> manifest_dataset_dir = "/path/to/manifest_dataset_file" 2843 >>> 2844 >>> dataset = ds.ManifestDataset(dataset_file=manifest_dataset_dir) 2845 >>> class_indexing = dataset.get_class_indexing() 2846 """ 2847 if self.class_indexing is None or not self.class_indexing: 2848 if self._class_indexing is None: 2849 runtime_getter = self._init_tree_getters() 2850 self._class_indexing = runtime_getter[0].GetClassIndexing() 2851 self.class_indexing = {} 2852 for pair in self._class_indexing: 2853 self.class_indexing[pair[0]] = pair[1][0] 2854 return self.class_indexing 2855 2856 2857class MnistDataset(MappableDataset, VisionBaseDataset): 2858 """ 2859 MNIST dataset. 2860 2861 The generated dataset has two columns :py:obj:`[image, label]` . 2862 The tensor of column :py:obj:`image` is of the uint8 type. 2863 The tensor of column :py:obj:`label` is a scalar of the uint32 type. 2864 2865 Args: 2866 dataset_dir (str): Path to the root directory that contains the dataset. 2867 usage (str, optional): Usage of this dataset, can be ``'train'`` , ``'test'`` or ``'all'`` . 2868 ``'train'`` will read from 60,000 train samples, ``'test'`` will read from 10,000 test samples, 2869 ``'all'`` will read from all 70,000 samples. Default: ``None`` , will read all samples. 2870 num_samples (int, optional): The number of images to be included in the dataset. 2871 Default: ``None`` , will read all images. 2872 num_parallel_workers (int, optional): Number of worker threads to read the data. 2873 Default: ``None`` , will use global default workers(8), it can be set 2874 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 2875 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 2876 Default: ``None`` , expected order behavior shown in the table below. 2877 sampler (Sampler, optional): Object used to choose samples from the 2878 dataset. Default: ``None`` , expected order behavior shown in the table below. 2879 num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` . 2880 When this argument is specified, `num_samples` reflects the maximum sample number of per shard. 2881 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 2882 argument can only be specified when `num_shards` is also specified. 2883 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 2884 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 2885 Default: ``None`` , which means no cache is used. 2886 2887 Raises: 2888 RuntimeError: If `dataset_dir` does not contain data files. 2889 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 2890 ValueError: If `usage` is not ``'train'``、``'test'`` or ``'all'``. 2891 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 2892 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 2893 RuntimeError: If `num_shards` is specified but shard_id is None. 2894 RuntimeError: If `shard_id` is specified but `num_shards` is None. 2895 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 2896 2897 Tutorial Examples: 2898 - `Load & Process Data With Dataset Pipeline 2899 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 2900 2901 Note: 2902 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 2903 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 2904 2905 .. include:: mindspore.dataset.sampler.txt 2906 2907 Examples: 2908 >>> import mindspore.dataset as ds 2909 >>> mnist_dataset_dir = "/path/to/mnist_dataset_directory" 2910 >>> 2911 >>> # Read 3 samples from MNIST dataset 2912 >>> dataset = ds.MnistDataset(dataset_dir=mnist_dataset_dir, num_samples=3) 2913 >>> 2914 >>> # Note: In mnist_dataset dataset, each dictionary has keys "image" and "label" 2915 2916 About MNIST dataset: 2917 2918 The MNIST database of handwritten digits has a training set of 60,000 examples, 2919 and a test set of 10,000 examples. It is a subset of a larger set available from 2920 NIST. The digits have been size-normalized and centered in a fixed-size image. 2921 2922 Here is the original MNIST dataset structure. 2923 You can unzip the dataset files into this directory structure and read by MindSpore's API. 2924 2925 .. code-block:: 2926 2927 . 2928 └── mnist_dataset_dir 2929 ├── t10k-images-idx3-ubyte 2930 ├── t10k-labels-idx1-ubyte 2931 ├── train-images-idx3-ubyte 2932 └── train-labels-idx1-ubyte 2933 2934 Citation: 2935 2936 .. code-block:: 2937 2938 @article{lecun2010mnist, 2939 title = {MNIST handwritten digit database}, 2940 author = {LeCun, Yann and Cortes, Corinna and Burges, CJ}, 2941 journal = {ATT Labs [Online]}, 2942 volume = {2}, 2943 year = {2010}, 2944 howpublished = {http://yann.lecun.com/exdb/mnist} 2945 } 2946 """ 2947 2948 @check_mnist_cifar_dataset 2949 def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=None, shuffle=None, 2950 sampler=None, num_shards=None, shard_id=None, cache=None): 2951 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 2952 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 2953 2954 self.dataset_dir = dataset_dir 2955 self.usage = replace_none(usage, "all") 2956 2957 def parse(self, children=None): 2958 return cde.MnistNode(self.dataset_dir, self.usage, self.sampler) 2959 2960 2961class OmniglotDataset(MappableDataset, VisionBaseDataset): 2962 """ 2963 Omniglot dataset. 2964 2965 The generated dataset has two columns :py:obj:`[image, label]` . 2966 The tensor of column :py:obj:`image` is of the uint8 type. 2967 The tensor of column :py:obj:`label` is a scalar of the uint32 type. 2968 2969 Args: 2970 dataset_dir (str): Path to the root directory that contains the dataset. 2971 background (bool, optional): Whether to create dataset from the "background" set. 2972 Otherwise create from the "evaluation" set. Default: ``None`` , set to ``True``. 2973 num_samples (int, optional): The number of images to be included in the dataset. 2974 Default: ``None`` , all images. 2975 num_parallel_workers (int, optional): Number of worker threads to read the data. 2976 Default: ``None`` , will use global default workers(8), it can be set 2977 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 2978 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 2979 Default: ``None`` , expected order behavior shown in the table below. 2980 decode (bool, optional): Decode the images after reading. Default: ``False``. 2981 sampler (Sampler, optional): Object used to choose samples from the 2982 dataset. Default: ``None`` , expected order behavior shown in the table below. 2983 num_shards (int, optional): Number of shards that the dataset will be divided 2984 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 2985 the max sample number of per shard. 2986 shard_id (int, optional): The shard ID within `num_shards`. Default: ``None`` . This 2987 argument can only be specified when `num_shards` is also specified. 2988 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 2989 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 2990 Default: ``None`` , which means no cache is used. 2991 2992 Raises: 2993 RuntimeError: If `dataset_dir` does not contain data files. 2994 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 2995 RuntimeError: If `sampler` and `sharding` are specified at the same time. 2996 RuntimeError: If `num_shards` is specified but `shard_id` is None. 2997 RuntimeError: If `shard_id` is specified but `num_shards` is None. 2998 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 2999 3000 Tutorial Examples: 3001 - `Load & Process Data With Dataset Pipeline 3002 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 3003 3004 Note: 3005 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 3006 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 3007 3008 .. include:: mindspore.dataset.sampler.txt 3009 3010 Examples: 3011 >>> import mindspore.dataset as ds 3012 >>> omniglot_dataset_dir = "/path/to/omniglot_dataset_directory" 3013 >>> dataset = ds.OmniglotDataset(dataset_dir=omniglot_dataset_dir, 3014 ... num_parallel_workers=8) 3015 3016 About Omniglot dataset: 3017 3018 The Omniglot dataset is designed for developing more human-like learning algorithms. It contains 1623 different 3019 handwritten characters from 50 different alphabets. Each of the 1623 characters was drawn online via Amazon's 3020 Mechanical Turk by 20 different people. Each image is paired with stroke data, a sequences of [x, y, t] coordinates 3021 with time in milliseconds. 3022 3023 You can unzip the original Omniglot dataset files into this directory structure and read by MindSpore's API. 3024 3025 .. code-block:: 3026 3027 . 3028 └── omniglot_dataset_directory 3029 ├── images_background/ 3030 │ ├── character_class1/ 3031 ├ ├──── 01.jpg 3032 │ ├──── 02.jpg 3033 │ ├── character_class2/ 3034 ├ ├──── 01.jpg 3035 │ ├──── 02.jpg 3036 │ ├── ... 3037 ├── images_evaluation/ 3038 │ ├── character_class1/ 3039 ├ ├──── 01.jpg 3040 │ ├──── 02.jpg 3041 │ ├── character_class2/ 3042 ├ ├──── 01.jpg 3043 │ ├──── 02.jpg 3044 │ ├── ... 3045 3046 Citation: 3047 3048 .. code-block:: 3049 3050 @article{lake2015human, 3051 title={Human-level concept learning through probabilistic program induction}, 3052 author={Lake, Brenden M and Salakhutdinov, Ruslan and Tenenbaum, Joshua B}, 3053 journal={Science}, 3054 volume={350}, 3055 number={6266}, 3056 pages={1332--1338}, 3057 year={2015}, 3058 publisher={American Association for the Advancement of Science} 3059 } 3060 """ 3061 3062 @check_omniglotdataset 3063 def __init__(self, dataset_dir, background=None, num_samples=None, num_parallel_workers=None, shuffle=None, 3064 decode=False, sampler=None, num_shards=None, shard_id=None, cache=None): 3065 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 3066 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 3067 self.dataset_dir = dataset_dir 3068 self.background = replace_none(background, True) 3069 self.decode = replace_none(decode, False) 3070 3071 def parse(self, children=None): 3072 return cde.OmniglotNode(self.dataset_dir, self.background, self.decode, self.sampler) 3073 3074 3075class PhotoTourDataset(MappableDataset, VisionBaseDataset): 3076 """ 3077 PhotoTour dataset. 3078 3079 According to the given `usage` configuration, the generated dataset has different output columns: 3080 3081 - `usage` = 'train', output columns: `[image, dtype=uint8]` . 3082 - `usage` ≠ 'train', output columns: `[image1, dtype=uint8]` , `[image2, dtype=uint8]` , `[matches, dtype=uint32]` . 3083 3084 Args: 3085 dataset_dir (str): Path to the root directory that contains the dataset. 3086 name (str): Name of the dataset to load, 3087 should be one of ``'notredame'``, ``'yosemite'``, ``'liberty'``, ``'notredame_harris'``, 3088 ``'yosemite_harris'`` or ``'liberty_harris'``. 3089 usage (str, optional): Usage of the dataset, can be ``'train'`` or ``'test'``. Default: ``None`` , 3090 will be set to 'train'. 3091 When usage is 'train', number of samples for each `name` is 3092 {'notredame': 468159, 'yosemite': 633587, 'liberty': 450092, 'liberty_harris': 379587, 3093 'yosemite_harris': 450912, 'notredame_harris': 325295}. 3094 When usage is 'test', will read 100,000 samples for testing. 3095 num_samples (int, optional): The number of images to be included in the dataset. 3096 Default: ``None`` , will read all images. 3097 num_parallel_workers (int, optional): Number of worker threads to read the data. 3098 Default: ``None`` , will use global default workers(8), it can be set 3099 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 3100 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 3101 Default: ``None`` , expected order behavior shown in the table below. 3102 sampler (Sampler, optional): Object used to choose samples from the dataset. 3103 Default: ``None`` , expected order behavior shown in the table below. 3104 num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` . 3105 When this argument is specified, `num_samples` reflects the max sample number of per shard. 3106 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 3107 argument can only be specified when `num_shards` is also specified. 3108 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 3109 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 3110 Default: ``None`` , which means no cache is used. 3111 3112 Raises: 3113 RuntimeError: If `dataset_dir` does not contain data files. 3114 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 3115 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 3116 RuntimeError: If `num_shards` is specified but `shard_id` is None. 3117 RuntimeError: If `shard_id` is specified but `num_shards` is None. 3118 ValueError: If `dataset_dir` is not exist. 3119 ValueError: If `usage` is not ``'train'`` or ``'test'``. 3120 ValueError: If name is not ``'notredame'``, ``'yosemite'``, ``'liberty'``, 3121 ``'notredame_harris'``, ``'yosemite_harris'`` or ``'liberty_harris'``. 3122 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 3123 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 3124 3125 Tutorial Examples: 3126 - `Load & Process Data With Dataset Pipeline 3127 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 3128 3129 Note: 3130 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 3131 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 3132 3133 .. include:: mindspore.dataset.sampler.txt 3134 3135 Examples: 3136 >>> import mindspore.dataset as ds 3137 >>> # Read 3 samples from PhotoTour dataset. 3138 >>> dataset = ds.PhotoTourDataset(dataset_dir="/path/to/photo_tour_dataset_directory", 3139 ... name='liberty', usage='train', num_samples=3) 3140 3141 About PhotoTour dataset: 3142 3143 The data is taken from Photo Tourism reconstructions from Trevi Fountain (Rome), Notre Dame (Paris) and Half 3144 Dome (Yosemite). Each dataset consists of a series of corresponding patches, which are obtained by projecting 3145 3D points from Photo Tourism reconstructions back into the original images. 3146 3147 The dataset consists of 1024 x 1024 bitmap (.bmp) images, each containing a 16 x 16 array of image patches. 3148 Each patch is sampled as 64 x 64 grayscale, with a canonical scale and orientation. For details of how the scale 3149 and orientation is established, please see the paper. An associated metadata file info.txt contains the match 3150 information. Each row of info.txt corresponds to a separate patch, with the patches ordered from left to right and 3151 top to bottom in each bitmap image. The first number on each row of info.txt is the 3D point ID from which that 3152 patch was sampled -- patches with the same 3D point ID are projected from the same 3D point (into different images). 3153 The second number in info.txt corresponds to the image from which the patch was sampled, and is not used at present. 3154 3155 You can unzip the original PhotoTour dataset files into this directory structure and read by MindSpore's API. 3156 3157 .. code-block:: 3158 3159 . 3160 └── photo_tour_dataset_directory 3161 ├── liberty/ 3162 │ ├── info.txt // two columns: 3D_point_ID, unused 3163 │ ├── m50_100000_100000_0.txt // seven columns: patch_ID1, 3D_point_ID1, unused1, 3164 │ │ // patch_ID2, 3D_point_ID2, unused2, unused3 3165 │ ├── patches0000.bmp // 1024*1024 pixels, with 16 * 16 patches. 3166 │ ├── patches0001.bmp 3167 │ ├── ... 3168 ├── yosemite/ 3169 │ ├── ... 3170 ├── notredame/ 3171 │ ├── ... 3172 ├── liberty_harris/ 3173 │ ├── ... 3174 ├── yosemite_harris/ 3175 │ ├── ... 3176 ├── notredame_harris/ 3177 │ ├── ... 3178 3179 Citation: 3180 3181 .. code-block:: 3182 3183 @INPROCEEDINGS{4269996, 3184 author={Winder, Simon A. J. and Brown, Matthew}, 3185 booktitle={2007 IEEE Conference on Computer Vision and Pattern Recognition}, 3186 title={Learning Local Image Descriptors}, 3187 year={2007}, 3188 volume={}, 3189 number={}, 3190 pages={1-8}, 3191 doi={10.1109/CVPR.2007.382971} 3192 } 3193 """ 3194 3195 @check_photo_tour_dataset 3196 def __init__(self, dataset_dir, name, usage=None, num_samples=None, num_parallel_workers=None, 3197 shuffle=None, sampler=None, num_shards=None, shard_id=None, cache=None): 3198 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 3199 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 3200 3201 self.dataset_dir = dataset_dir 3202 self.name = name 3203 self.usage = replace_none(usage, "train") 3204 3205 def parse(self, children=None): 3206 return cde.PhotoTourNode(self.dataset_dir, self.name, self.usage, self.sampler) 3207 3208 3209class Places365Dataset(MappableDataset, VisionBaseDataset): 3210 """ 3211 Places365 dataset. 3212 3213 The generated dataset has two columns :py:obj:`[image, label]` . 3214 The tensor of column :py:obj:`image` is of the uint8 type. 3215 The tensor of column :py:obj:`label` is of the uint32 type. 3216 3217 Args: 3218 dataset_dir (str): Path to the root directory that contains the dataset. 3219 usage (str, optional): Usage of this dataset, can be ``'train-standard'``, ``'train-challenge'`` 3220 or ``'val'``. Default: ``None`` , will be set to ``'train-standard'``. 3221 small (bool, optional): Use 256 * 256 images (True) or high resolution images (False). Default: ``True``. 3222 decode (bool, optional): Decode the images after reading. Default: ``False``. 3223 num_samples (int, optional): The number of images to be included in the dataset. 3224 Default: ``None`` , will read all images. 3225 num_parallel_workers (int, optional): Number of worker threads to read the data. 3226 Default: ``None`` , will use global default workers(8), it can be set 3227 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 3228 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 3229 Default: ``None`` , expected order behavior shown in the table below. 3230 sampler (Sampler, optional): Object used to choose samples from the 3231 dataset. Default: ``None`` , expected order behavior shown in the table below. 3232 num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` . 3233 When this argument is specified, `num_samples` reflects the max sample number of per shard. 3234 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 3235 argument can only be specified when `num_shards` is also specified. 3236 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 3237 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 3238 Default: ``None`` , which means no cache is used. 3239 3240 Raises: 3241 RuntimeError: If `dataset_dir` does not contain data files. 3242 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 3243 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 3244 RuntimeError: If `num_shards` is specified but `shard_id` is None. 3245 RuntimeError: If `shard_id` is specified but `num_shards` is None. 3246 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 3247 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 3248 ValueError: If `usage` is not ``"train-standard"``, ``"train-challenge"`` or ``"val"``. 3249 3250 Tutorial Examples: 3251 - `Load & Process Data With Dataset Pipeline 3252 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 3253 3254 Note: 3255 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 3256 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 3257 3258 .. include:: mindspore.dataset.sampler.txt 3259 3260 Examples: 3261 >>> import mindspore.dataset as ds 3262 >>> place365_dataset_dir = "/path/to/place365_dataset_directory" 3263 >>> 3264 >>> # Read 3 samples from Places365 dataset 3265 >>> dataset = ds.Places365Dataset(dataset_dir=place365_dataset_dir, usage='train-standard', 3266 ... small=True, decode=True, num_samples=3) 3267 3268 About Places365 dataset: 3269 3270 Convolutional neural networks (CNNs) trained on the Places2 Database can be used for scene recognition as well as 3271 generic deep scene features for visual recognition. 3272 3273 The author releases the data of Places365-Standard and the data of Places365-Challenge to the public. 3274 Places365-Standard is the core set of Places2 Database, which has been used to train the Places365-CNNs. The author 3275 will add other kinds of annotation on the Places365-Standard in the future. Places365-Challenge is the competition 3276 set of Places2 Database, which has 6.2 million extra images compared to the Places365-Standard. 3277 The Places365-Challenge will be used for the Places Challenge 2016. 3278 3279 You can unzip the original Places365 dataset files into this directory structure and read by MindSpore's API. 3280 3281 .. code-block:: 3282 3283 . 3284 └── categories_places365 3285 ├── places365_train-standard.txt 3286 ├── places365_train-challenge.txt 3287 ├── val_large/ 3288 │ ├── Places365_val_00000001.jpg 3289 │ ├── Places365_val_00000002.jpg 3290 │ ├── Places365_val_00000003.jpg 3291 │ ├── ... 3292 ├── val_256/ 3293 │ ├── ... 3294 ├── data_large_standard/ 3295 │ ├── ... 3296 ├── data_256_standard/ 3297 │ ├── ... 3298 ├── data_large_challenge/ 3299 │ ├── ... 3300 ├── data_256_challenge / 3301 │ ├── ... 3302 3303 Citation: 3304 3305 .. code-block:: 3306 3307 article{zhou2017places, 3308 title={Places: A 10 million Image Database for Scene Recognition}, 3309 author={Zhou, Bolei and Lapedriza, Agata and Khosla, Aditya and Oliva, Aude and Torralba, Antonio}, 3310 journal={IEEE Transactions on Pattern Analysis and Machine Intelligence}, 3311 year={2017}, 3312 publisher={IEEE} 3313 } 3314 """ 3315 3316 @check_places365_dataset 3317 def __init__(self, dataset_dir, usage=None, small=True, decode=False, num_samples=None, num_parallel_workers=None, 3318 shuffle=None, sampler=None, num_shards=None, shard_id=None, cache=None): 3319 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 3320 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 3321 3322 self.dataset_dir = os.path.abspath(dataset_dir) 3323 self.usage = replace_none(usage, "train-standard") 3324 self.small = small 3325 self.decode = decode 3326 3327 def parse(self, children=None): 3328 return cde.Places365Node(self.dataset_dir, self.usage, self.small, self.decode, self.sampler) 3329 3330 3331class QMnistDataset(MappableDataset, VisionBaseDataset): 3332 """ 3333 QMNIST dataset. 3334 3335 The generated dataset has two columns :py:obj:`[image, label]` . 3336 The tensor of column :py:obj:`image` is of the uint8 type. 3337 The tensor of column :py:obj:`label` is of the uint32 type. 3338 3339 Args: 3340 dataset_dir (str): Path to the root directory that contains the dataset. 3341 usage (str, optional): Usage of this dataset, can be ``'train'``, ``'test'``, ``'test10k'``, 3342 ``'test50k'``, ``'nist'`` or ``'all'``. Default: ``None`` , will read all samples. 3343 compat (bool, optional): Whether the label for each example is class number (compat= ``True`` ) 3344 or the full QMNIST information (compat= ``False`` ). Default: ``True``. 3345 num_samples (int, optional): The number of images to be included in the dataset. 3346 Default: ``None`` , will read all images. 3347 num_parallel_workers (int, optional): Number of worker threads to read the data. 3348 Default: ``None`` , will use global default workers(8), it can be set 3349 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 3350 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 3351 Default: ``None`` , expected order behavior shown in the table below. 3352 sampler (Sampler, optional): Object used to choose samples from the 3353 dataset. Default: ``None`` , expected order behavior shown in the table below. 3354 num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` . 3355 When this argument is specified, `num_samples` reflects the maximum sample number of per shard. 3356 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 3357 argument can only be specified when `num_shards` is also specified. 3358 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 3359 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 3360 Default: ``None`` , which means no cache is used. 3361 3362 Raises: 3363 RuntimeError: If `dataset_dir` does not contain data files. 3364 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 3365 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 3366 RuntimeError: If `num_shards` is specified but `shard_id` is None. 3367 RuntimeError: If `shard_id` is specified but `num_shards` is None. 3368 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 3369 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 3370 3371 Tutorial Examples: 3372 - `Load & Process Data With Dataset Pipeline 3373 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 3374 3375 Note: 3376 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 3377 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 3378 3379 .. include:: mindspore.dataset.sampler.txt 3380 3381 Examples: 3382 >>> import mindspore.dataset as ds 3383 >>> qmnist_dataset_dir = "/path/to/qmnist_dataset_directory" 3384 >>> 3385 >>> # Read 3 samples from QMNIST train dataset 3386 >>> dataset = ds.QMnistDataset(dataset_dir=qmnist_dataset_dir, num_samples=3) 3387 >>> 3388 >>> # Note: In QMNIST dataset, each dictionary has keys "image" and "label" 3389 3390 About QMNIST dataset: 3391 3392 The QMNIST dataset was generated from the original data found in the NIST Special Database 19 with the goal to 3393 match the MNIST preprocessing as closely as possible. 3394 Through an iterative process, researchers tried to generate an additional 50k images of MNIST-like data. 3395 They started with a reconstruction process given in the paper and used the Hungarian algorithm to find the best 3396 matches between the original MNIST samples and their reconstructed samples. 3397 3398 Here is the original QMNIST dataset structure. 3399 You can unzip the dataset files into this directory structure and read by MindSpore's API. 3400 3401 .. code-block:: 3402 3403 . 3404 └── qmnist_dataset_dir 3405 ├── qmnist-train-images-idx3-ubyte 3406 ├── qmnist-train-labels-idx2-int 3407 ├── qmnist-test-images-idx3-ubyte 3408 ├── qmnist-test-labels-idx2-int 3409 ├── xnist-images-idx3-ubyte 3410 └── xnist-labels-idx2-int 3411 3412 Citation: 3413 3414 .. code-block:: 3415 3416 @incollection{qmnist-2019, 3417 title = "Cold Case: The Lost MNIST Digits", 3418 author = "Chhavi Yadav and L\'{e}on Bottou",\ 3419 booktitle = {Advances in Neural Information Processing Systems 32}, 3420 year = {2019}, 3421 publisher = {Curran Associates, Inc.}, 3422 } 3423 """ 3424 3425 @check_qmnist_dataset 3426 def __init__(self, dataset_dir, usage=None, compat=True, num_samples=None, num_parallel_workers=None, 3427 shuffle=None, sampler=None, num_shards=None, shard_id=None, cache=None): 3428 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 3429 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 3430 3431 self.dataset_dir = dataset_dir 3432 self.usage = replace_none(usage, "all") 3433 self.compat = compat 3434 3435 def parse(self, children=None): 3436 return cde.QMnistNode(self.dataset_dir, self.usage, self.compat, self.sampler) 3437 3438 3439class RandomDataset(SourceDataset, VisionBaseDataset): 3440 """ 3441 A source dataset that generates random data. 3442 3443 Args: 3444 total_rows (int, optional): Number of samples for the dataset to generate. 3445 Default: ``None`` , number of samples is random. 3446 schema (Union[str, Schema], optional): Data format policy, which specifies the data types and shapes of the data 3447 column to be read. Both JSON file path and objects constructed by :class:`mindspore.dataset.Schema` are 3448 acceptable. Default: ``None`` . 3449 columns_list (list[str], optional): List of column names of the dataset. 3450 Default: ``None`` , the columns will be named like this "c0", "c1", "c2" etc. 3451 num_samples (int, optional): The number of samples to be included in the dataset. 3452 Default: ``None`` , all samples. 3453 num_parallel_workers (int, optional): Number of worker threads to read the data. 3454 Default: ``None`` , will use global default workers(8), it can be set 3455 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 3456 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 3457 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 3458 Default: ``None`` , which means no cache is used. 3459 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 3460 Default: ``None`` , expected order behavior shown in the table below. 3461 num_shards (int, optional): Number of shards that the dataset will be divided 3462 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 3463 the maximum sample number of per shard. 3464 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 3465 argument can only be specified when `num_shards` is also specified. 3466 3467 Raises: 3468 RuntimeError: If `num_shards` is specified but `shard_id` is None. 3469 RuntimeError: If `shard_id` is specified but `num_shards` is None. 3470 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 3471 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 3472 TypeError: If `total_rows` is not of type int. 3473 TypeError: If `num_shards` is not of type int. 3474 TypeError: If `num_parallel_workers` is not of type int. 3475 TypeError: If `shuffle` is not of type bool. 3476 TypeError: If `columns_list` is not of type list. 3477 3478 Tutorial Examples: 3479 - `Load & Process Data With Dataset Pipeline 3480 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 3481 3482 Examples: 3483 >>> from mindspore import dtype as mstype 3484 >>> import mindspore.dataset as ds 3485 >>> 3486 >>> schema = ds.Schema() 3487 >>> schema.add_column('image', de_type=mstype.uint8, shape=[2]) 3488 >>> schema.add_column('label', de_type=mstype.uint8, shape=[1]) 3489 >>> # apply dataset operations 3490 >>> ds1 = ds.RandomDataset(schema=schema, total_rows=50, num_parallel_workers=4) 3491 """ 3492 3493 @check_random_dataset 3494 def __init__(self, total_rows=None, schema=None, columns_list=None, num_samples=None, num_parallel_workers=None, 3495 cache=None, shuffle=None, num_shards=None, shard_id=None): 3496 super().__init__(num_parallel_workers=num_parallel_workers, num_samples=num_samples, shuffle=shuffle, 3497 num_shards=num_shards, shard_id=shard_id, cache=cache) 3498 3499 self.total_rows = replace_none(total_rows, 0) 3500 if self.total_rows != 0 and self.num_samples != 0: 3501 self.total_rows = min(self.total_rows, self.num_samples) 3502 if self.num_samples != 0: 3503 self.total_rows = self.num_samples 3504 if schema is not None: 3505 self.total_rows = replace_none(total_rows, Schema.get_num_rows(schema)) 3506 self.schema = replace_none(schema, "") 3507 self.columns_list = replace_none(columns_list, []) 3508 3509 def parse(self, children=None): 3510 schema = self.schema.cpp_schema if isinstance(self.schema, Schema) else self.schema 3511 return cde.RandomNode(self.total_rows, schema, self.columns_list) 3512 3513 3514class RenderedSST2Dataset(MappableDataset, VisionBaseDataset): 3515 """ 3516 RenderedSST2(Rendered Stanford Sentiment Treebank v2) dataset. 3517 3518 The generated dataset has two columns: :py:obj:`[image, label]`. 3519 The tensor of column :py:obj:`image` is of the uint8 type. 3520 The tensor of column :py:obj:`label` is of the uint32 type. 3521 3522 Args: 3523 dataset_dir (str): Path to the root directory that contains the dataset. 3524 usage (str, optional): Usage of this dataset, can be ``'train'``, ``'val'``, ``'test'`` 3525 or ``'all'``. Default: ``None`` , will read all samples. 3526 num_samples (int, optional): The number of images to be included in the dataset. 3527 Default: ``None`` , will include all images. 3528 num_parallel_workers (int, optional): Number of worker threads to read the data. 3529 Default: ``None`` , will use global default workers(8), it can be set 3530 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 3531 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 3532 Default: ``None`` , expected order behavior shown in the table below. 3533 decode (bool, optional): Whether or not to decode the images after reading. Default: ``False``. 3534 sampler (Sampler, optional): Object used to choose samples from the 3535 dataset. Default: ``None`` , expected order behavior shown in the table below. 3536 num_shards (int, optional): Number of shards that the dataset will be divided 3537 into. When this argument is specified, `num_samples` reflects 3538 the maximum sample number of per shard. Default: ``None`` . 3539 shard_id (int, optional): The shard ID within `num_shards` . This 3540 argument can only be specified when `num_shards` is also specified. Default: ``None`` . 3541 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 3542 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 3543 Default: ``None`` , which means no cache is used. 3544 3545 Raises: 3546 RuntimeError: If `dataset_dir` does not contain data files. 3547 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 3548 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 3549 RuntimeError: If `num_shards` is specified but `shard_id` is None. 3550 RuntimeError: If `shard_id` is specified but `num_shards` is None. 3551 ValueError: If `usage` is not ``'train'``, ``'test'``, ``'val'`` or ``'all'``. 3552 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 3553 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 3554 3555 Tutorial Examples: 3556 - `Load & Process Data With Dataset Pipeline 3557 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 3558 3559 Note: 3560 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 3561 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 3562 3563 .. include:: mindspore.dataset.sampler.txt 3564 3565 Examples: 3566 >>> import mindspore.dataset as ds 3567 >>> rendered_sst2_dataset_dir = "/path/to/rendered_sst2_dataset_directory" 3568 >>> 3569 >>> # 1) Read all samples (image files) in rendered_sst2_dataset_dir with 8 threads 3570 >>> dataset = ds.RenderedSST2Dataset(dataset_dir=rendered_sst2_dataset_dir, 3571 ... usage="all", num_parallel_workers=8) 3572 3573 About RenderedSST2Dataset: 3574 3575 Rendered SST2 is an image classification dataset which was generated by rendering sentences in the Standford 3576 Sentiment Treebank v2 dataset. There are three splits in this dataset and each split contains two classes 3577 (positive and negative): a train split containing 6920 images (3610 positive and 3310 negative), a validation 3578 split containing 872 images (444 positive and 428 negative), and a test split containing 1821 images 3579 (909 positive and 912 negative). 3580 3581 Here is the original RenderedSST2 dataset structure. 3582 You can unzip the dataset files into the following directory structure and read by MindSpore's API. 3583 3584 .. code-block:: 3585 3586 . 3587 └── rendered_sst2_dataset_directory 3588 ├── train 3589 │ ├── negative 3590 │ │ ├── 0001.jpg 3591 │ │ ├── 0002.jpg 3592 │ │ ... 3593 │ └── positive 3594 │ ├── 0001.jpg 3595 │ ├── 0002.jpg 3596 │ ... 3597 ├── test 3598 │ ├── negative 3599 │ │ ├── 0001.jpg 3600 │ │ ├── 0002.jpg 3601 │ │ ... 3602 │ └── positive 3603 │ ├── 0001.jpg 3604 │ ├── 0002.jpg 3605 │ ... 3606 └── valid 3607 ├── negative 3608 │ ├── 0001.jpg 3609 │ ├── 0002.jpg 3610 │ ... 3611 └── positive 3612 ├── 0001.jpg 3613 ├── 0002.jpg 3614 ... 3615 3616 Citation: 3617 3618 .. code-block:: 3619 3620 @inproceedings{socher-etal-2013-recursive, 3621 title = {Recursive Deep Models for Semantic Compositionality Over a Sentiment Treebank}, 3622 author = {Socher, Richard and Perelygin, Alex and Wu, Jean and Chuang, Jason and Manning, 3623 Christopher D. and Ng, Andrew and Potts, Christopher}, 3624 booktitle = {Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing}, 3625 month = oct, 3626 year = {2013}, 3627 address = {Seattle, Washington, USA}, 3628 publisher = {Association for Computational Linguistics}, 3629 url = {https://www.aclweb.org/anthology/D13-1170}, 3630 pages = {1631--1642}, 3631 } 3632 """ 3633 3634 @check_rendered_sst2_dataset 3635 def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=None, shuffle=None, 3636 decode=False, sampler=None, num_shards=None, shard_id=None, cache=None): 3637 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 3638 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 3639 3640 self.dataset_dir = dataset_dir 3641 self.usage = replace_none(usage, "all") 3642 self.decode = replace_none(decode, False) 3643 3644 def parse(self, children=None): 3645 return cde.RenderedSST2Node(self.dataset_dir, self.usage, self.decode, self.sampler) 3646 3647 3648class _SBDataset: 3649 """ 3650 Dealing with the data file with .mat extension, and return one row in tuple (image, task) each time. 3651 """ 3652 3653 def __init__(self, dataset_dir, task, usage, decode): 3654 self.column_list = ['image', 'task'] 3655 self.task = task 3656 self.images_path = os.path.join(dataset_dir, 'img') 3657 self.cls_path = os.path.join(dataset_dir, 'cls') 3658 self._loadmat = loadmat 3659 self.categories = 20 3660 self.decode = replace_none(decode, False) 3661 3662 if usage == "all": 3663 image_names = [] 3664 for item in ["train", "val"]: 3665 usage_path = os.path.join(dataset_dir, item + '.txt') 3666 if not os.path.exists(usage_path): 3667 raise FileNotFoundError("SBDataset: {0} not found".format(usage_path)) 3668 with open(usage_path, 'r') as f: 3669 image_names += [x.strip() for x in f.readlines()] 3670 else: 3671 usage_path = os.path.join(dataset_dir, usage + '.txt') 3672 if not os.path.exists(usage_path): 3673 raise FileNotFoundError("SBDataset: {0} not found".format(usage_path)) 3674 with open(usage_path, 'r') as f: 3675 image_names = [x.strip() for x in f.readlines()] 3676 3677 self.images = [os.path.join(self.images_path, i + ".jpg") for i in image_names] 3678 self.clss = [os.path.join(self.cls_path, i + ".mat") for i in image_names] 3679 3680 if len(self.images) != len(self.clss): 3681 raise ValueError("SBDataset: images count not equal to cls count") 3682 3683 self._get_data = self._get_boundaries_data if self.task == "Boundaries" else self._get_segmentation_data 3684 self._get_item = self._get_decode_item if self.decode else self._get_undecode_item 3685 3686 def _get_boundaries_data(self, mat_path): 3687 mat_data = self._loadmat(mat_path) 3688 return np.concatenate([np.expand_dims(mat_data['GTcls'][0][self.task][0][i][0].toarray(), axis=0) 3689 for i in range(self.categories)], axis=0) 3690 3691 def _get_segmentation_data(self, mat_path): 3692 mat_data = self._loadmat(mat_path) 3693 return Image.fromarray(mat_data['GTcls'][0][self.task][0]) 3694 3695 def _get_decode_item(self, idx): 3696 return Image.open(self.images[idx]).convert('RGB'), self._get_data(self.clss[idx]) 3697 3698 def _get_undecode_item(self, idx): 3699 return np.fromfile(self.images[idx], dtype=np.uint8), self._get_data(self.clss[idx]) 3700 3701 def __len__(self): 3702 return len(self.images) 3703 3704 def __getitem__(self, idx): 3705 return self._get_item(idx) 3706 3707 3708class SBDataset(GeneratorDataset): 3709 """ 3710 SB(Semantic Boundaries) Dataset. 3711 3712 By configuring the `task` parameter, the generated dataset has different output columns. 3713 3714 - `task` is ``'Boundaries'`` , there are two output columns: the 'image' column has the data type uint8 and 3715 the 'label' column contains one image of the data type uint8. 3716 - `task` is ``'Segmentation'`` , there are two output columns: the 'image' column has the data type uint8 and 3717 the 'label' column contains 20 images of the data type uint8. 3718 3719 Args: 3720 dataset_dir (str): Path to the root directory that contains the dataset. 3721 task (str, optional): Acceptable tasks include ``'Boundaries'`` or ``'Segmentation'``. 3722 Default: ``'Boundaries'``. 3723 usage (str, optional): Acceptable usages include ``'train'``, ``'val'``, ``'train_noval'`` 3724 and ``'all'``. Default: ``'all'``. 3725 num_samples (int, optional): The number of images to be included in the dataset. 3726 Default: ``None`` , all images. 3727 num_parallel_workers (int, optional): Number of worker subprocesses to read the data. Default: ``1``. 3728 shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected 3729 order behavior shown in the table below. 3730 decode (bool, optional): Decode the images after reading. Default: ``None`` , means ``False``. 3731 sampler (Sampler, optional): Object used to choose samples from the 3732 dataset. Default: ``None`` , expected order behavior shown in the table below. 3733 num_shards (int, optional): Number of shards that the dataset will be divided 3734 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 3735 the max sample number of per shard. 3736 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 3737 argument can only be specified when `num_shards` is also specified. 3738 3739 Raises: 3740 RuntimeError: If `dataset_dir` is not valid or does not contain data files. 3741 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 3742 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 3743 RuntimeError: If `num_shards` is specified but `shard_id` is None. 3744 RuntimeError: If `shard_id` is specified but `num_shards` is None. 3745 ValueError: If `dataset_dir` is not exist. 3746 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 3747 ValueError: If `task` is not ``'Boundaries'`` or ``'Segmentation'``. 3748 ValueError: If `usage` is not ``'train'``, ``'val'``, ``'train_noval'`` or ``'all'``. 3749 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 3750 3751 Tutorial Examples: 3752 - `Load & Process Data With Dataset Pipeline 3753 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 3754 3755 Note: 3756 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 3757 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 3758 3759 .. include:: mindspore.dataset.sampler.txt 3760 3761 Examples: 3762 >>> import mindspore.dataset as ds 3763 >>> sb_dataset_dir = "/path/to/sb_dataset_directory" 3764 >>> 3765 >>> # 1) Get all samples from Semantic Boundaries Dataset in sequence 3766 >>> dataset = ds.SBDataset(dataset_dir=sb_dataset_dir, shuffle=False) 3767 >>> 3768 >>> # 2) Randomly select 350 samples from Semantic Boundaries Dataset 3769 >>> dataset = ds.SBDataset(dataset_dir=sb_dataset_dir, num_samples=350, shuffle=True) 3770 >>> 3771 >>> # 3) Get samples from Semantic Boundaries Dataset for shard 0 in a 2-way distributed training 3772 >>> dataset = ds.SBDataset(dataset_dir=sb_dataset_dir, num_shards=2, shard_id=0) 3773 >>> 3774 >>> # In Semantic Boundaries Dataset, each dictionary has keys "image" and "task" 3775 3776 About Semantic Boundaries Dataset: 3777 3778 The Semantic Boundaries Dataset consists of 11355 color images. There are 8498 images' name in the train.txt, 3779 2857 images' name in the val.txt and 5623 images' name in the train_noval.txt. The category cls/ 3780 contains the Segmentation and Boundaries results of category-level, the category inst/ contains the 3781 Segmentation and Boundaries results of instance-level. 3782 3783 You can unzip the dataset files into the following structure and read by MindSpore's API: 3784 3785 .. code-block:: 3786 3787 . 3788 └── benchmark_RELEASE 3789 ├── dataset 3790 ├── img 3791 │ ├── 2008_000002.jpg 3792 │ ├── 2008_000003.jpg 3793 │ ├── ... 3794 ├── cls 3795 │ ├── 2008_000002.mat 3796 │ ├── 2008_000003.mat 3797 │ ├── ... 3798 ├── inst 3799 │ ├── 2008_000002.mat 3800 │ ├── 2008_000003.mat 3801 │ ├── ... 3802 ├── train.txt 3803 └── val.txt 3804 3805 .. code-block:: 3806 3807 @InProceedings{BharathICCV2011, 3808 author = "Bharath Hariharan and Pablo Arbelaez and Lubomir Bourdev and 3809 Subhransu Maji and Jitendra Malik", 3810 title = "Semantic Contours from Inverse Detectors", 3811 booktitle = "International Conference on Computer Vision (ICCV)", 3812 year = "2011", 3813 } 3814 """ 3815 3816 @check_sb_dataset 3817 def __init__(self, dataset_dir, task='Boundaries', usage='all', num_samples=None, num_parallel_workers=1, 3818 shuffle=None, decode=None, sampler=None, num_shards=None, shard_id=None): 3819 dataset = _SBDataset(dataset_dir, task, usage, decode) 3820 super().__init__(dataset, column_names=dataset.column_list, num_samples=num_samples, 3821 num_parallel_workers=num_parallel_workers, shuffle=shuffle, sampler=sampler, 3822 num_shards=num_shards, shard_id=shard_id) 3823 3824 3825class SBUDataset(MappableDataset, VisionBaseDataset): 3826 """ 3827 SBU(SBU Captioned Photo) dataset. 3828 3829 The generated dataset has two columns :py:obj:`[image, caption]` . 3830 The tensor of column :py:obj:`image` is of the uint8 type. 3831 The tensor of column :py:obj:`caption` is of the string type. 3832 3833 Args: 3834 dataset_dir (str): Path to the root directory that contains the dataset. 3835 num_samples (int, optional): The number of images to be included in the dataset. 3836 Default: ``None`` , will read all images. 3837 num_parallel_workers (int, optional): Number of worker threads to read the data. 3838 Default: ``None`` , will use global default workers(8), it can be set 3839 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 3840 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 3841 Default: ``None`` , expected order behavior shown in the table below. 3842 decode (bool, optional): Decode the images after reading. Default: ``False``. 3843 sampler (Sampler, optional): Object used to choose samples from the 3844 dataset. Default: ``None`` , expected order behavior shown in the table below. 3845 num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` . 3846 When this argument is specified, `num_samples` reflects the max sample number of per shard. 3847 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 3848 argument can only be specified when `num_shards` is also specified. 3849 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 3850 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 3851 Default: ``None`` , which means no cache is used. 3852 3853 Raises: 3854 RuntimeError: If `dataset_dir` does not contain data files. 3855 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 3856 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 3857 RuntimeError: If `num_shards` is specified but `shard_id` is None. 3858 RuntimeError: If `shard_id` is specified but `num_shards` is None. 3859 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 3860 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 3861 3862 Tutorial Examples: 3863 - `Load & Process Data With Dataset Pipeline 3864 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 3865 3866 Note: 3867 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 3868 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 3869 3870 .. include:: mindspore.dataset.sampler.txt 3871 3872 Examples: 3873 >>> import mindspore.dataset as ds 3874 >>> sbu_dataset_dir = "/path/to/sbu_dataset_directory" 3875 >>> # Read 3 samples from SBU dataset 3876 >>> dataset = ds.SBUDataset(dataset_dir=sbu_dataset_dir, num_samples=3) 3877 3878 About SBU dataset: 3879 3880 SBU dataset is a large captioned photo collection. 3881 It contains one million images with associated visually relevant captions. 3882 3883 You should manually download the images using official download.m by replacing 'urls{i}(24, end)' with 3884 'urls{i}(24:1:end)' and keep the directory as below. 3885 3886 .. code-block:: 3887 3888 . 3889 └─ dataset_dir 3890 ├── SBU_captioned_photo_dataset_captions.txt 3891 ├── SBU_captioned_photo_dataset_urls.txt 3892 └── sbu_images 3893 ├── m_3326_3596303505_3ce4c20529.jpg 3894 ├── ...... 3895 └── m_2522_4182181099_c3c23ab1cc.jpg 3896 3897 Citation: 3898 3899 .. code-block:: 3900 3901 @inproceedings{Ordonez:2011:im2text, 3902 Author = {Vicente Ordonez and Girish Kulkarni and Tamara L. Berg}, 3903 Title = {Im2Text: Describing Images Using 1 Million Captioned Photographs}, 3904 Booktitle = {Neural Information Processing Systems ({NIPS})}, 3905 Year = {2011}, 3906 } 3907 """ 3908 3909 @check_sbu_dataset 3910 def __init__(self, dataset_dir, num_samples=None, num_parallel_workers=None, shuffle=None, decode=False, 3911 sampler=None, num_shards=None, shard_id=None, cache=None): 3912 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 3913 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 3914 3915 self.dataset_dir = dataset_dir 3916 self.decode = replace_none(decode, False) 3917 3918 def parse(self, children=None): 3919 return cde.SBUNode(self.dataset_dir, self.decode, self.sampler) 3920 3921 3922class SemeionDataset(MappableDataset, VisionBaseDataset): 3923 """ 3924 Semeion dataset. 3925 3926 The generated dataset has two columns :py:obj:`[image, label]` . 3927 The tensor of column :py:obj:`image` is of the uint8 type. 3928 The tensor of column :py:obj:`label` is a scalar of the uint32 type. 3929 3930 Args: 3931 dataset_dir (str): Path to the root directory that contains the dataset. 3932 num_samples (int, optional): The number of samples to be included in the dataset. 3933 Default: ``None`` , will read all images. 3934 num_parallel_workers (int, optional): Number of worker threads to read the data. 3935 Default: ``None`` , will use global default workers(8), it can be set 3936 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 3937 shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected 3938 order behavior shown in the table below. 3939 sampler (Sampler, optional): Object used to choose samples from the 3940 dataset. Default: ``None`` , expected order behavior shown in the table below. 3941 num_shards (int, optional): Number of shards that the dataset will be divided 3942 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 3943 the maximum sample number of per shard. 3944 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 3945 argument can only be specified when `num_shards` is also specified. 3946 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 3947 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 3948 Default: ``None`` , which means no cache is used. 3949 3950 Raises: 3951 RuntimeError: If `dataset_dir` does not contain data files. 3952 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 3953 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 3954 RuntimeError: If `num_shards` is specified but `shard_id` is None. 3955 RuntimeError: If `shard_id` is specified but `num_shards` is None. 3956 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 3957 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 3958 3959 Tutorial Examples: 3960 - `Load & Process Data With Dataset Pipeline 3961 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 3962 3963 Note: 3964 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 3965 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 3966 3967 .. include:: mindspore.dataset.sampler.txt 3968 3969 Examples: 3970 >>> import mindspore.dataset as ds 3971 >>> semeion_dataset_dir = "/path/to/semeion_dataset_directory" 3972 >>> 3973 >>> # 1) Get all samples from SEMEION dataset in sequence 3974 >>> dataset = ds.SemeionDataset(dataset_dir=semeion_dataset_dir, shuffle=False) 3975 >>> 3976 >>> # 2) Randomly select 10 samples from SEMEION dataset 3977 >>> dataset = ds.SemeionDataset(dataset_dir=semeion_dataset_dir, num_samples=10, shuffle=True) 3978 >>> 3979 >>> # 3) Get samples from SEMEION dataset for shard 0 in a 2-way distributed training 3980 >>> dataset = ds.SemeionDataset(dataset_dir=semeion_dataset_dir, num_shards=2, shard_id=0) 3981 >>> 3982 >>> # In SEMEION dataset, each dictionary has keys: image, label. 3983 3984 About SEMEION dataset: 3985 3986 The dataset was created by Tactile Srl, Brescia, Italy (http://www.tattile.it) and donated in 1994 3987 to Semeion Research Center of Sciences of Communication, Rome, Italy (http://www.semeion.it), 3988 for machine learning research. 3989 3990 This dataset consists of 1593 records (rows) and 256 attributes (columns). Each record represents 3991 a handwritten digit, originally scanned with a resolution of 256 grey scale. Each pixel of the each 3992 original scanned image was first stretched, and after scaled between 0 and 1 3993 (setting to 0 every pixel whose value was under the value 127 of the grey scale (127 included) 3994 and setting to 1 each pixel whose original value in the grey scale was over 127). Finally, each binary image 3995 was scaled again into a 16x16 square box (the final 256 binary attributes). 3996 3997 .. code-block:: 3998 3999 . 4000 └── semeion_dataset_dir 4001 └──semeion.data 4002 └──semeion.names 4003 4004 Citation: 4005 4006 .. code-block:: 4007 4008 @article{ 4009 title={The Theory of Independent Judges, in Substance Use & Misuse 33(2)1998, pp 439-461}, 4010 author={M Buscema, MetaNet}, 4011 } 4012 """ 4013 4014 @check_semeion_dataset 4015 def __init__(self, dataset_dir, num_samples=None, num_parallel_workers=None, shuffle=None, 4016 sampler=None, num_shards=None, shard_id=None, cache=None): 4017 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 4018 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 4019 4020 self.dataset_dir = dataset_dir 4021 4022 def parse(self, children=None): 4023 return cde.SemeionNode(self.dataset_dir, self.sampler) 4024 4025 4026class STL10Dataset(MappableDataset, VisionBaseDataset): 4027 """ 4028 STL-10 dataset. 4029 4030 The generated dataset has two columns: :py:obj:`[image, label]` . 4031 The tensor of column :py:obj:`image` is of the uint8 type. 4032 The tensor of column :py:obj:`label` is of a scalar of int32 type. 4033 4034 Args: 4035 dataset_dir (str): Path to the root directory that contains the dataset. 4036 usage (str, optional): Usage of this dataset, can be ``'train'``, ``'test'``, 4037 ``'unlabeled'``, ``'train+unlabeled'`` or ``'all'`` . ``'train'`` will read from 5,000 4038 train samples, ``'test'`` will read from 8,000 test samples, 4039 ``'unlabeled'`` will read from all 100,000 samples, and ``'train+unlabeled'`` 4040 will read from 105000 samples, ``'all'`` will read all the samples 4041 Default: ``None`` , all samples. 4042 num_samples (int, optional): The number of images to be included in the dataset. 4043 Default: ``None`` , all images. 4044 num_parallel_workers (int, optional): Number of worker threads to read the data. 4045 Default: ``None`` , will use global default workers(8), it can be set 4046 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 4047 shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected 4048 order behavior shown in the table below. 4049 sampler (Sampler, optional): Object used to choose samples from the 4050 dataset. Default: ``None`` , expected order behavior shown in the table below. 4051 num_shards (int, optional): Number of shards that the dataset will be divided 4052 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 4053 the max sample number of per shard. 4054 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 4055 argument can only be specified when `num_shards` is also specified. 4056 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 4057 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 4058 Default: ``None`` , which means no cache is used. 4059 4060 Raises: 4061 RuntimeError: If `dataset_dir` is not valid or does not exist or does not contain data files. 4062 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 4063 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 4064 RuntimeError: If `num_shards` is specified but `shard_id` is None. 4065 RuntimeError: If `shard_id` is specified but `num_shards` is None. 4066 ValueError: If `usage` is invalid. 4067 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 4068 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 4069 4070 Tutorial Examples: 4071 - `Load & Process Data With Dataset Pipeline 4072 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 4073 4074 Note: 4075 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 4076 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 4077 4078 .. include:: mindspore.dataset.sampler.txt 4079 4080 Examples: 4081 >>> import mindspore.dataset as ds 4082 >>> stl10_dataset_dir = "/path/to/stl10_dataset_directory" 4083 >>> 4084 >>> # 1) Get all samples from STL10 dataset in sequence 4085 >>> dataset = ds.STL10Dataset(dataset_dir=stl10_dataset_dir, shuffle=False) 4086 >>> 4087 >>> # 2) Randomly select 350 samples from STL10 dataset 4088 >>> dataset = ds.STL10Dataset(dataset_dir=stl10_dataset_dir, num_samples=350, shuffle=True) 4089 >>> 4090 >>> # 3) Get samples from STL10 dataset for shard 0 in a 2-way distributed training 4091 >>> dataset = ds.STL10Dataset(dataset_dir=stl10_dataset_dir, num_shards=2, shard_id=0) 4092 4093 About STL10 dataset: 4094 4095 STL10 dataset consists of 10 classes: airplane, bird, car, cat, deer, dog, horse, monkey, ship, truck. 4096 Images are 96x96 pixels, color. 4097 500 training images, 800 test images per class and 100000 unlabeled images. 4098 Labels are 0-indexed, and unlabeled images have -1 as their labels. 4099 4100 Here is the original STL10 dataset structure. 4101 You can unzip the dataset files into this directory structure and read by MindSpore's API. 4102 4103 .. code-block:: 4104 4105 . 4106 └── stl10_dataset_dir 4107 ├── train_X.bin 4108 ├── train_y.bin 4109 ├── test_X.bin 4110 ├── test_y.bin 4111 └── unlabeled_X.bin 4112 4113 Citation of STL10 dataset: 4114 4115 .. code-block:: 4116 4117 @techreport{Coates10, 4118 author = {Adam Coates}, 4119 title = {Learning multiple layers of features from tiny images}, 4120 year = {20010}, 4121 howpublished = {https://cs.stanford.edu/~acoates/stl10/}, 4122 description = {The STL-10 dataset consists of 96x96 RGB images in 10 classes, 4123 with 500 training images and 800 testing images per class. 4124 There are 5000 training images and 8000 test images. 4125 It also has 100000 unlabeled images for unsupervised learning. 4126 These examples are extracted from a similar but broader distribution of images. 4127 } 4128 } 4129 """ 4130 4131 @check_stl10_dataset 4132 def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=None, shuffle=None, 4133 sampler=None, num_shards=None, shard_id=None, cache=None): 4134 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 4135 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 4136 4137 self.dataset_dir = dataset_dir 4138 self.usage = replace_none(usage, "all") 4139 4140 def parse(self, children=None): 4141 return cde.STL10Node(self.dataset_dir, self.usage, self.sampler) 4142 4143 4144class SUN397Dataset(MappableDataset, VisionBaseDataset): 4145 """ 4146 SUN397(Scene UNderstanding) dataset. 4147 4148 The generated dataset has two columns: :py:obj:`[image, label]`. 4149 The tensor of column :py:obj:`image` is of the uint8 type. 4150 The tensor of column :py:obj:`label` is of the uint32 type. 4151 4152 Args: 4153 dataset_dir (str): Path to the root directory that contains the dataset. 4154 num_samples (int, optional): The number of images to be included in the dataset. 4155 Default: ``None`` , all images. 4156 num_parallel_workers (int, optional): Number of worker threads to read the data. 4157 Default: ``None`` , will use global default workers(8), it can be set 4158 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 4159 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 4160 Default: ``None`` , expected order behavior shown in the table below. 4161 decode (bool, optional): Whether or not to decode the images after reading. Default: ``False``. 4162 sampler (Sampler, optional): Object used to choose samples from the 4163 dataset. Default: ``None`` , expected order behavior shown in the table below. 4164 num_shards (int, optional): Number of shards that the dataset will be divided 4165 into. When this argument is specified, `num_samples` reflects 4166 the maximum sample number of per shard. Default: ``None`` . 4167 shard_id (int, optional): The shard ID within `num_shards` . This 4168 argument can only be specified when `num_shards` is also specified. Default: ``None`` . 4169 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 4170 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 4171 Default: ``None`` , which means no cache is used. 4172 4173 Raises: 4174 RuntimeError: If `dataset_dir` does not contain data files. 4175 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 4176 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 4177 RuntimeError: If `num_shards` is specified but `shard_id` is None. 4178 RuntimeError: If `shard_id` is specified but `num_shards` is None. 4179 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 4180 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 4181 4182 Tutorial Examples: 4183 - `Load & Process Data With Dataset Pipeline 4184 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 4185 4186 Note: 4187 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 4188 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 4189 4190 .. include:: mindspore.dataset.sampler.txt 4191 4192 Examples: 4193 >>> import mindspore.dataset as ds 4194 >>> sun397_dataset_dir = "/path/to/sun397_dataset_directory" 4195 >>> 4196 >>> # 1) Read all samples (image files) in sun397_dataset_dir with 8 threads 4197 >>> dataset = ds.SUN397Dataset(dataset_dir=sun397_dataset_dir, num_parallel_workers=8) 4198 4199 About SUN397Dataset: 4200 4201 The SUN397 or Scene UNderstanding (SUN) is a dataset for scene recognition consisting of 397 categories with 4202 108,754 images. The number of images varies across categories, but there are at least 100 images per category. 4203 Images are in jpg, png, or gif format. 4204 4205 Here is the original SUN397 dataset structure. 4206 You can unzip the dataset files into this directory structure and read by MindSpore's API. 4207 4208 .. code-block:: 4209 4210 . 4211 └── sun397_dataset_directory 4212 ├── ClassName.txt 4213 ├── README.txt 4214 ├── a 4215 │ ├── abbey 4216 │ │ ├── sun_aaaulhwrhqgejnyt.jpg 4217 │ │ ├── sun_aacphuqehdodwawg.jpg 4218 │ │ ├── ... 4219 │ ├── apartment_building 4220 │ │ └── outdoor 4221 │ │ ├── sun_aamyhslnsnomjzue.jpg 4222 │ │ ├── sun_abbjzfrsalhqivis.jpg 4223 │ │ ├── ... 4224 │ ├── ... 4225 ├── b 4226 │ ├── badlands 4227 │ │ ├── sun_aabtemlmesogqbbp.jpg 4228 │ │ ├── sun_afbsfeexggdhzshd.jpg 4229 │ │ ├── ... 4230 │ ├── balcony 4231 │ │ ├── exterior 4232 │ │ │ ├── sun_aaxzaiuznwquburq.jpg 4233 │ │ │ ├── sun_baajuldidvlcyzhv.jpg 4234 │ │ │ ├── ... 4235 │ │ └── interior 4236 │ │ ├── sun_babkzjntjfarengi.jpg 4237 │ │ ├── sun_bagjvjynskmonnbv.jpg 4238 │ │ ├── ... 4239 │ └── ... 4240 ├── ... 4241 4242 4243 Citation: 4244 4245 .. code-block:: 4246 4247 @inproceedings{xiao2010sun, 4248 title = {Sun database: Large-scale scene recognition from abbey to zoo}, 4249 author = {Xiao, Jianxiong and Hays, James and Ehinger, Krista A and Oliva, Aude and Torralba, Antonio}, 4250 booktitle = {2010 IEEE computer society conference on computer vision and pattern recognition}, 4251 pages = {3485--3492}, 4252 year = {2010}, 4253 organization = {IEEE} 4254 } 4255 """ 4256 4257 @check_sun397_dataset 4258 def __init__(self, dataset_dir, num_samples=None, num_parallel_workers=None, shuffle=None, decode=False, 4259 sampler=None, num_shards=None, shard_id=None, cache=None): 4260 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 4261 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 4262 4263 self.dataset_dir = dataset_dir 4264 self.decode = replace_none(decode, False) 4265 4266 def parse(self, children=None): 4267 return cde.SUN397Node(self.dataset_dir, self.decode, self.sampler) 4268 4269 4270class _SVHNDataset: 4271 """ 4272 Mainly for loading SVHN Dataset, and return two rows each time. 4273 """ 4274 4275 def __init__(self, dataset_dir, usage): 4276 self.dataset_dir = os.path.realpath(dataset_dir) 4277 self.usage = usage 4278 self.column_names = ["image", "label"] 4279 self.usage_all = ["train", "test", "extra"] 4280 self.data = np.array([], dtype=np.uint8) 4281 self.labels = np.array([], dtype=np.uint32) 4282 4283 if self.usage == "all": 4284 for _usage in self.usage_all: 4285 data, label = self._load_mat(_usage) 4286 self.data = np.concatenate((self.data, data)) if self.data.size else data 4287 self.labels = np.concatenate((self.labels, label)) if self.labels.size else label 4288 else: 4289 self.data, self.labels = self._load_mat(self.usage) 4290 4291 def _load_mat(self, mode): 4292 filename = mode + "_32x32.mat" 4293 mat_data = loadmat(os.path.join(self.dataset_dir, filename)) 4294 data = np.transpose(mat_data['X'], [3, 0, 1, 2]) 4295 label = mat_data['y'].astype(np.uint32).squeeze() 4296 np.place(label, label == 10, 0) 4297 return data, label 4298 4299 def __getitem__(self, index): 4300 return self.data[index], self.labels[index] 4301 4302 def __len__(self): 4303 return len(self.data) 4304 4305 4306class SVHNDataset(GeneratorDataset): 4307 """ 4308 SVHN(Street View House Numbers) dataset. 4309 4310 The generated dataset has two columns: :py:obj:`[image, label]` . 4311 The tensor of column :py:obj:`image` is of the uint8 type. 4312 The tensor of column :py:obj:`label` is of a scalar of uint32 type. 4313 4314 Args: 4315 dataset_dir (str): Path to the root directory that contains the dataset. 4316 usage (str, optional): Specify the ``'train'``, ``'test'``, ``'extra'`` or ``'all'`` parts of dataset. 4317 Default: ``None`` , will read all samples. 4318 num_samples (int, optional): The number of samples to be included in the dataset. Default: ``None`` , 4319 all images. 4320 num_parallel_workers (int, optional): Number of worker subprocesses used to 4321 fetch the dataset in parallel. Default: ``1``. 4322 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 4323 Default: ``None`` , expected order behavior shown in the table below. 4324 sampler (Sampler, optional): Object used to choose samples from the dataset. Random accessible 4325 input is required. Default: ``None`` , expected order behavior shown in the table below. 4326 num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` . 4327 When this argument is specified, `num_samples` reflects the max sample number of per shard. 4328 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . 4329 This argument must be specified only when `num_shards` is also specified. 4330 4331 Raises: 4332 RuntimeError: If `dataset_dir` is not valid or does not exist or does not contain data files. 4333 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 4334 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 4335 RuntimeError: If `num_shards` is specified but `shard_id` is None. 4336 RuntimeError: If `shard_id` is specified but `num_shards` is None. 4337 ValueError: If `usage` is invalid. 4338 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 4339 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 4340 4341 Tutorial Examples: 4342 - `Load & Process Data With Dataset Pipeline 4343 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 4344 4345 Note: 4346 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 4347 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 4348 4349 .. include:: mindspore.dataset.sampler.txt 4350 4351 Examples: 4352 >>> import mindspore.dataset as ds 4353 >>> svhn_dataset_dir = "/path/to/svhn_dataset_directory" 4354 >>> dataset = ds.SVHNDataset(dataset_dir=svhn_dataset_dir, usage="train") 4355 4356 About SVHN dataset: 4357 4358 SVHN dataset consists of 10 digit classes and is obtained from house numbers in Google Street View images. 4359 4360 Here is the original SVHN dataset structure. 4361 You can unzip the dataset files into this directory structure and read by MindSpore's API. 4362 4363 .. code-block:: 4364 4365 . 4366 └── svhn_dataset_dir 4367 ├── train_32x32.mat 4368 ├── test_32x32.mat 4369 └── extra_32x32.mat 4370 4371 Citation: 4372 4373 .. code-block:: 4374 4375 @article{ 4376 title={Reading Digits in Natural Images with Unsupervised Feature Learning}, 4377 author={Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco, Bo Wu, Andrew Y. Ng}, 4378 conference={NIPS Workshop on Deep Learning and Unsupervised Feature Learning 2011.}, 4379 year={2011}, 4380 publisher={NIPS} 4381 url={http://ufldl.stanford.edu/housenumbers} 4382 } 4383 4384 """ 4385 4386 @check_svhn_dataset 4387 def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=1, shuffle=None, 4388 sampler=None, num_shards=None, shard_id=None): 4389 self.dataset_dir = os.path.realpath(dataset_dir) 4390 self.usage = replace_none(usage, "all") 4391 dataset = _SVHNDataset(self.dataset_dir, self.usage) 4392 4393 super().__init__(dataset, column_names=dataset.column_names, num_samples=num_samples, 4394 num_parallel_workers=num_parallel_workers, shuffle=shuffle, sampler=sampler, 4395 num_shards=num_shards, shard_id=shard_id) 4396 4397 4398class USPSDataset(SourceDataset, VisionBaseDataset): 4399 """ 4400 USPS(U.S. Postal Service) dataset. 4401 4402 The generated dataset has two columns: :py:obj:`[image, label]` . 4403 The tensor of column :py:obj:`image` is of the uint8 type. 4404 The tensor of column :py:obj:`label` is of the uint32 type. 4405 4406 Args: 4407 dataset_dir (str): Path to the root directory that contains the dataset. 4408 usage (str, optional): Usage of this dataset, can be ``'train'``, ``'test'`` or ``'all'`` . 4409 ``'train'`` will read from 7,291 train samples, ``'test'`` will read from 2,007 test samples, 4410 ``'all'`` will read from all 9,298 samples. Default: ``None`` , will read all samples. 4411 num_samples (int, optional): The number of images to be included in the dataset. 4412 Default: ``None`` , will read all images. 4413 num_parallel_workers (int, optional): Number of worker threads to read the data. 4414 Default: ``None`` , will use global default workers(8), it can be set 4415 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 4416 shuffle (Union[bool, Shuffle], optional): Perform reshuffling of the data every epoch. 4417 Bool type and Shuffle enum are both supported to pass in. Default: ``Shuffle.GLOBAL`` . 4418 If `shuffle` is ``False`` , no shuffling will be performed. 4419 If `shuffle` is ``True`` , it is equivalent to setting `shuffle` to ``mindspore.dataset.Shuffle.GLOBAL``. 4420 Set the mode of data shuffling by passing in enumeration variables: 4421 4422 - ``Shuffle.GLOBAL`` : Shuffle both the files and samples. 4423 4424 - ``Shuffle.FILES`` : Shuffle files only. 4425 4426 num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` . 4427 When this argument is specified, `num_samples` reflects the max sample number of per shard. 4428 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 4429 argument can only be specified when `num_shards` is also specified. 4430 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 4431 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 4432 Default: ``None`` , which means no cache is used. 4433 4434 Raises: 4435 RuntimeError: If `dataset_dir` is not valid or does not exist or does not contain data files. 4436 RuntimeError: If `num_shards` is specified but `shard_id` is None. 4437 RuntimeError: If `shard_id` is specified but `num_shards` is None. 4438 ValueError: If `usage` is invalid. 4439 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 4440 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 4441 4442 Tutorial Examples: 4443 - `Load & Process Data With Dataset Pipeline 4444 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 4445 4446 Examples: 4447 >>> import mindspore.dataset as ds 4448 >>> usps_dataset_dir = "/path/to/usps_dataset_directory" 4449 >>> 4450 >>> # Read 3 samples from USPS dataset 4451 >>> dataset = ds.USPSDataset(dataset_dir=usps_dataset_dir, num_samples=3) 4452 4453 About USPS dataset: 4454 4455 USPS is a digit dataset automatically scanned from envelopes by the U.S. Postal Service 4456 containing a total of 9,298 16×16 pixel grayscale samples. 4457 The images are centered, normalized and show a broad range of font styles. 4458 4459 Here is the original USPS dataset structure. 4460 You can download and unzip the dataset files into this directory structure and read by MindSpore's API. 4461 4462 .. code-block:: 4463 4464 . 4465 └── usps_dataset_dir 4466 ├── usps 4467 ├── usps.t 4468 4469 Citation: 4470 4471 .. code-block:: 4472 4473 @article{hull1994database, 4474 title={A database for handwritten text recognition research}, 4475 author={Hull, Jonathan J.}, 4476 journal={IEEE Transactions on pattern analysis and machine intelligence}, 4477 volume={16}, 4478 number={5}, 4479 pages={550--554}, 4480 year={1994}, 4481 publisher={IEEE} 4482 } 4483 """ 4484 4485 @check_usps_dataset 4486 def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=None, shuffle=Shuffle.GLOBAL, 4487 num_shards=None, shard_id=None, cache=None): 4488 super().__init__(num_parallel_workers=num_parallel_workers, num_samples=num_samples, shuffle=shuffle, 4489 num_shards=num_shards, shard_id=shard_id, cache=cache) 4490 4491 self.dataset_dir = dataset_dir 4492 self.usage = replace_none(usage, "all") 4493 4494 def parse(self, children=None): 4495 return cde.USPSNode(self.dataset_dir, self.usage, self.num_samples, self.shuffle_flag, self.num_shards, 4496 self.shard_id) 4497 4498 4499class VOCDataset(MappableDataset, VisionBaseDataset): 4500 """ 4501 VOC(Visual Object Classes) dataset. 4502 4503 The generated dataset with different `task` setting has different output columns: 4504 4505 - `task` = :py:obj:`Detection` , output columns: :py:obj:`[image, dtype=uint8]` , 4506 :py:obj:`[bbox, dtype=float32]` , :py:obj:`[label, dtype=uint32]` , 4507 :py:obj:`[difficult, dtype=uint32]` , :py:obj:`[truncate, dtype=uint32]` . 4508 - `task` = :py:obj:`Segmentation` , output columns: :py:obj:`[image, dtype=uint8]` , 4509 :py:obj:`[target,dtype=uint8]` . 4510 4511 Args: 4512 dataset_dir (str): Path to the root directory that contains the dataset. 4513 task (str, optional): Set the task type of reading voc data, now only support ``'Segmentation'`` or 4514 ``'Detection'``. Default: ``'Segmentation'``. 4515 usage (str, optional): Set the task type of ImageSets. Default: ``'train'``. If `task` is ``'Segmentation'``, 4516 image and annotation list will be loaded in ./ImageSets/Segmentation/usage + ".txt"; 4517 If `task` is 'Detection', image and annotation list will be loaded in ./ImageSets/Main/usage + ".txt"; 4518 if `task` and `usage` are not set, image and annotation list will be loaded in 4519 ./ImageSets/Segmentation/train.txt as default. 4520 class_indexing (dict, optional): A str-to-int mapping from label name to index, only valid in 4521 'Detection' task. Default: ``None`` , the folder names will be sorted alphabetically and each 4522 class will be given a unique index starting from 0. 4523 num_samples (int, optional): The number of images to be included in the dataset. 4524 Default: ``None`` , all images. 4525 num_parallel_workers (int, optional): Number of worker threads to read the data. 4526 Default: ``None`` , will use global default workers(8), it can be set 4527 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 4528 shuffle (bool, optional): Whether to perform shuffle on the dataset. Default: ``None`` , expected 4529 order behavior shown in the table below. 4530 decode (bool, optional): Decode the images after reading. Default: ``False``. 4531 sampler (Sampler, optional): Object used to choose samples from the dataset. 4532 Default: ``None`` , expected order behavior shown in the table below. 4533 num_shards (int, optional): Number of shards that the dataset will be divided 4534 into. Default: ``None`` . When this argument is specified, `num_samples` reflects 4535 the maximum sample number of per shard. 4536 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . This 4537 argument can only be specified when `num_shards` is also specified. 4538 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 4539 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 4540 Default: ``None`` , which means no cache is used. 4541 extra_metadata(bool, optional): Flag to add extra meta-data to row. If True, an additional column named 4542 :py:obj:`[_meta-filename, dtype=string]` will be output at the end. Default: ``False``. 4543 decrypt (callable, optional): Image decryption function, which accepts the path of the encrypted image file 4544 and returns the decrypted bytes data. Default: ``None`` , no decryption. 4545 4546 Raises: 4547 RuntimeError: If `dataset_dir` does not contain data files. 4548 RuntimeError: If xml of Annotations is an invalid format. 4549 RuntimeError: If xml of Annotations loss attribution of `object` . 4550 RuntimeError: If xml of Annotations loss attribution of `bndbox` . 4551 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 4552 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 4553 RuntimeError: If `num_shards` is specified but `shard_id` is None. 4554 RuntimeError: If `shard_id` is specified but `num_shards` is None. 4555 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 4556 ValueError: If `task` is not equal ``'Segmentation'`` or ``'Detection'``. 4557 ValueError: If `task` is ``'Segmentation'`` but `class_indexing` is not ``None``. 4558 ValueError: If txt related to mode is not exist. 4559 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 4560 4561 Tutorial Examples: 4562 - `Load & Process Data With Dataset Pipeline 4563 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 4564 4565 Note: 4566 - Column '[_meta-filename, dtype=string]' won't be output unless an explicit rename dataset op 4567 is added to remove the prefix('_meta-'). 4568 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 4569 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 4570 4571 .. include:: mindspore.dataset.sampler.txt 4572 4573 Examples: 4574 >>> import mindspore.dataset as ds 4575 >>> voc_dataset_dir = "/path/to/voc_dataset_directory" 4576 >>> 4577 >>> # 1) Read VOC data for segmentation training 4578 >>> dataset = ds.VOCDataset(dataset_dir=voc_dataset_dir, task="Segmentation", usage="train") 4579 >>> 4580 >>> # 2) Read VOC data for detection training 4581 >>> dataset = ds.VOCDataset(dataset_dir=voc_dataset_dir, task="Detection", usage="train") 4582 >>> 4583 >>> # 3) Read all VOC dataset samples in voc_dataset_dir with 8 threads in random order 4584 >>> dataset = ds.VOCDataset(dataset_dir=voc_dataset_dir, task="Detection", usage="train", 4585 ... num_parallel_workers=8) 4586 >>> 4587 >>> # 4) Read then decode all VOC dataset samples in voc_dataset_dir in sequence 4588 >>> dataset = ds.VOCDataset(dataset_dir=voc_dataset_dir, task="Detection", usage="train", 4589 ... decode=True, shuffle=False) 4590 >>> 4591 >>> # In VOC dataset, if task='Segmentation', each dictionary has keys "image" and "target" 4592 >>> # In VOC dataset, if task='Detection', each dictionary has keys "image" and "annotation" 4593 4594 About VOC dataset: 4595 4596 The PASCAL Visual Object Classes (VOC) challenge is a benchmark in visual 4597 object category recognition and detection, providing the vision and machine 4598 learning communities with a standard dataset of images and annotation, and 4599 standard evaluation procedures. 4600 4601 You can unzip the original VOC-2012 dataset files into this directory structure and read by MindSpore's API. 4602 4603 .. code-block:: 4604 4605 . 4606 └── voc2012_dataset_dir 4607 ├── Annotations 4608 │ ├── 2007_000027.xml 4609 │ ├── 2007_000032.xml 4610 │ ├── ... 4611 ├── ImageSets 4612 │ ├── Action 4613 │ ├── Layout 4614 │ ├── Main 4615 │ └── Segmentation 4616 ├── JPEGImages 4617 │ ├── 2007_000027.jpg 4618 │ ├── 2007_000032.jpg 4619 │ ├── ... 4620 ├── SegmentationClass 4621 │ ├── 2007_000032.png 4622 │ ├── 2007_000033.png 4623 │ ├── ... 4624 └── SegmentationObject 4625 ├── 2007_000032.png 4626 ├── 2007_000033.png 4627 ├── ... 4628 4629 Citation: 4630 4631 .. code-block:: 4632 4633 @article{Everingham10, 4634 author = {Everingham, M. and Van~Gool, L. and Williams, C. K. I. and Winn, J. and Zisserman, A.}, 4635 title = {The Pascal Visual Object Classes (VOC) Challenge}, 4636 journal = {International Journal of Computer Vision}, 4637 volume = {88}, 4638 year = {2012}, 4639 number = {2}, 4640 month = {jun}, 4641 pages = {303--338}, 4642 biburl = {http://host.robots.ox.ac.uk/pascal/VOC/pubs/everingham10.html#bibtex}, 4643 howpublished = {http://host.robots.ox.ac.uk/pascal/VOC/voc2012/index.html} 4644 } 4645 """ 4646 4647 @check_vocdataset 4648 def __init__(self, dataset_dir, task="Segmentation", usage="train", class_indexing=None, num_samples=None, 4649 num_parallel_workers=None, shuffle=None, decode=False, sampler=None, num_shards=None, shard_id=None, 4650 cache=None, extra_metadata=False, decrypt=None): 4651 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 4652 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 4653 self.dataset_dir = dataset_dir 4654 self.task = replace_none(task, "Segmentation") 4655 self.usage = replace_none(usage, "train") 4656 self.class_indexing = replace_none(class_indexing, {}) 4657 self.decode = replace_none(decode, False) 4658 self.extra_metadata = extra_metadata 4659 self.decrypt = decrypt 4660 4661 def parse(self, children=None): 4662 return cde.VOCNode(self.dataset_dir, self.task, self.usage, self.class_indexing, self.decode, self.sampler, 4663 self.extra_metadata, self.decrypt) 4664 4665 def get_class_indexing(self): 4666 """ 4667 Get the class index. 4668 4669 Returns: 4670 dict, a str-to-int mapping from label name to index. 4671 4672 Examples: 4673 >>> import mindspore.dataset as ds 4674 >>> voc_dataset_dir = "/path/to/voc_dataset_directory" 4675 >>> 4676 >>> dataset = ds.VOCDataset(dataset_dir=voc_dataset_dir, task="Detection") 4677 >>> class_indexing = dataset.get_class_indexing() 4678 """ 4679 if self.task != "Detection": 4680 raise NotImplementedError("Only 'Detection' support get_class_indexing.") 4681 if self.class_indexing is None or not self.class_indexing: 4682 if self._class_indexing is None: 4683 runtime_getter = self._init_tree_getters() 4684 self._class_indexing = runtime_getter[0].GetClassIndexing() 4685 self.class_indexing = {} 4686 for pair in self._class_indexing: 4687 self.class_indexing[pair[0]] = pair[1][0] 4688 return self.class_indexing 4689 4690 4691class WIDERFaceDataset(MappableDataset, VisionBaseDataset): 4692 """ 4693 WIDERFace dataset. 4694 4695 When usage is "train", "valid" or "all", the generated dataset has eight columns ["image", "bbox", "blur", 4696 "expression", "illumination", "occlusion", "pose", "invalid"]. The data type of the `image` column is uint8, 4697 and all other columns are uint32. When usage is "test", it only has one column 4698 ["image"], with uint8 data type. 4699 4700 Args: 4701 dataset_dir (str): Path to the root directory that contains the dataset. 4702 usage (str, optional): Usage of this dataset, can be ``'train'``, ``'test'``, ``'valid'`` or 4703 ``'all'``. ``'train'`` will read from 12,880 samples, ``'test'`` will read from 16,097 samples, 4704 ``'valid'`` will read from 3,226 test samples and ``'all'`` will read all ``'train'`` 4705 and ``'valid'`` samples. Default: ``None`` , will be set to ``'all'``. 4706 num_samples (int, optional): The number of images to be included in the dataset. 4707 Default: ``None`` , will read all images. 4708 num_parallel_workers (int, optional): Number of worker threads to read the data. 4709 Default: ``None`` , will use global default workers(8), it can be set 4710 by :func:`mindspore.dataset.config.set_num_parallel_workers` . 4711 shuffle (bool, optional): Whether or not to perform shuffle on the dataset. 4712 Default: ``None`` , expected order behavior shown in the table below. 4713 decode (bool, optional): Decode the images after reading. Default: ``False``. 4714 sampler (Sampler, optional): Object used to choose samples from the dataset. 4715 Default: ``None`` , expected order behavior shown in the table below. 4716 num_shards (int, optional): Number of shards that the dataset will be divided into. Default: ``None`` . 4717 When this argument is specified, `num_samples` reflects the maximum sample number of per shard. 4718 shard_id (int, optional): The shard ID within `num_shards` . Default: ``None`` . 4719 This argument can only be specified when `num_shards` is also specified. 4720 cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. More details: 4721 `Single-Node Data Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ . 4722 Default: ``None`` , which means no cache is used. 4723 4724 Raises: 4725 RuntimeError: If `dataset_dir` does not contain data files. 4726 RuntimeError: If `sampler` and `shuffle` are specified at the same time. 4727 RuntimeError: If `sampler` and `num_shards`/`shard_id` are specified at the same time. 4728 RuntimeError: If `num_shards` is specified but `shard_id` is None. 4729 RuntimeError: If `shard_id` is specified but `num_shards` is None. 4730 ValueError: If `shard_id` is not in range of [0, `num_shards` ). 4731 ValueError: If `usage` is not ``'train'``, ``'test'``, ``'valid'``, ``'all'``. 4732 ValueError: If `num_parallel_workers` exceeds the max thread numbers. 4733 ValueError: If `annotation_file` is not exist. 4734 ValueError: If `dataset_dir` is not exist. 4735 4736 Tutorial Examples: 4737 - `Load & Process Data With Dataset Pipeline 4738 <https://www.mindspore.cn/docs/en/master/api_python/samples/dataset/dataset_gallery.html>`_ 4739 4740 Note: 4741 - The parameters `num_samples` , `shuffle` , `num_shards` , `shard_id` can be used to control the sampler 4742 used in the dataset, and their effects when combined with parameter `sampler` are as follows. 4743 4744 .. include:: mindspore.dataset.sampler.txt 4745 4746 Examples: 4747 >>> import mindspore.dataset as ds 4748 >>> wider_face_dir = "/path/to/wider_face_dataset" 4749 >>> 4750 >>> # Read 3 samples from WIDERFace dataset 4751 >>> dataset = ds.WIDERFaceDataset(dataset_dir=wider_face_dir, num_samples=3) 4752 4753 About WIDERFace dataset: 4754 4755 The WIDERFace database has a training set of 12,880 samples, a testing set of 16,097 examples 4756 and a validating set of 3,226 examples. It is a subset of a larger set available from WIDER. The digits have 4757 been size-normalized and centered in a fixed-size image. 4758 4759 The following is the original WIDERFace dataset structure. 4760 You can unzip the dataset files into this directory structure and read by MindSpore's API. 4761 4762 .. code-block:: 4763 4764 . 4765 └── wider_face_dir 4766 ├── WIDER_test 4767 │ └── images 4768 │ ├── 0--Parade 4769 │ │ ├── 0_Parade_marchingband_1_9.jpg 4770 │ │ ├── ... 4771 │ ├──1--Handshaking 4772 │ ├──... 4773 ├── WIDER_train 4774 │ └── images 4775 │ ├── 0--Parade 4776 │ │ ├── 0_Parade_marchingband_1_11.jpg 4777 │ │ ├── ... 4778 │ ├──1--Handshaking 4779 │ ├──... 4780 ├── WIDER_val 4781 │ └── images 4782 │ ├── 0--Parade 4783 │ │ ├── 0_Parade_marchingband_1_102.jpg 4784 │ │ ├── ... 4785 │ ├──1--Handshaking 4786 │ ├──... 4787 └── wider_face_split 4788 ├── wider_face_test_filelist.txt 4789 ├── wider_face_train_bbx_gt.txt 4790 └── wider_face_val_bbx_gt.txt 4791 4792 Citation: 4793 4794 .. code-block:: 4795 4796 @inproceedings{2016WIDER, 4797 title={WIDERFACE: A Detection Benchmark}, 4798 author={Yang, S. and Luo, P. and Loy, C. C. and Tang, X.}, 4799 booktitle={IEEE}, 4800 pages={5525-5533}, 4801 year={2016}, 4802 } 4803 """ 4804 4805 @check_wider_face_dataset 4806 def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=None, shuffle=None, 4807 decode=False, sampler=None, num_shards=None, shard_id=None, cache=None): 4808 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, 4809 shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) 4810 4811 self.dataset_dir = dataset_dir 4812 self.usage = replace_none(usage, "all") 4813 self.decode = replace_none(decode, False) 4814 4815 def parse(self, children=None): 4816 return cde.WIDERFaceNode(self.dataset_dir, self.usage, self.decode, self.sampler) 4817