• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15""" create train dataset. """
16
17from functools import partial
18
19import mindspore.common.dtype as mstype
20import mindspore.dataset as ds
21import mindspore.dataset.transforms.c_transforms as C2
22import mindspore.dataset.vision.c_transforms as C
23
24
25def create_dataset(dataset_path, config, repeat_num=1, batch_size=32):
26    """
27    create a train dataset
28
29    Args:
30        dataset_path(string): the path of dataset.
31        config(EasyDict)：the basic config for training
32        repeat_num(int): the repeat times of dataset. Default: 1.
33        batch_size(int): the batch size of dataset. Default: 32.
34
35    Returns:
36        dataset
37    """
38
39    load_func = partial(ds.Cifar10Dataset, dataset_path)
40    data_set = load_func(num_parallel_workers=8, shuffle=False)
41
42    resize_height = config.image_height
43    resize_width = config.image_width
44
45    mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
46    std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
47
48    # define map operations
49    resize_op = C.Resize((resize_height, resize_width))
50    normalize_op = C.Normalize(mean=mean, std=std)
51    changeswap_op = C.HWC2CHW()
52    c_trans = [resize_op, normalize_op, changeswap_op]
53
54    type_cast_op = C2.TypeCast(mstype.int32)
55
56    data_set = data_set.map(operations=c_trans, input_columns="image",
57                            num_parallel_workers=8)
58    data_set = data_set.map(operations=type_cast_op,
59                            input_columns="label", num_parallel_workers=8)
60
61    # apply batch operations
62    data_set = data_set.batch(batch_size, drop_remainder=True)
63
64    # apply dataset repeat operation
65    data_set = data_set.repeat(repeat_num)
66
67    return data_set
68