1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Reuters topic classification dataset.""" 16 17import json 18 19import numpy as np 20 21from tensorflow.python.keras.preprocessing.sequence import _remove_long_seq 22from tensorflow.python.keras.utils.data_utils import get_file 23from tensorflow.python.platform import tf_logging as logging 24from tensorflow.python.util.tf_export import keras_export 25 26 27@keras_export('keras.datasets.reuters.load_data') 28def load_data(path='reuters.npz', 29 num_words=None, 30 skip_top=0, 31 maxlen=None, 32 test_split=0.2, 33 seed=113, 34 start_char=1, 35 oov_char=2, 36 index_from=3, 37 **kwargs): 38 """Loads the Reuters newswire classification dataset. 39 40 This is a dataset of 11,228 newswires from Reuters, labeled over 46 topics. 41 42 This was originally generated by parsing and preprocessing the classic 43 Reuters-21578 dataset, but the preprocessing code is no longer packaged 44 with Keras. See this 45 [github discussion](https://github.com/keras-team/keras/issues/12072) 46 for more info. 47 48 Each newswire is encoded as a list of word indexes (integers). 49 For convenience, words are indexed by overall frequency in the dataset, 50 so that for instance the integer "3" encodes the 3rd most frequent word in 51 the data. This allows for quick filtering operations such as: 52 "only consider the top 10,000 most 53 common words, but eliminate the top 20 most common words". 54 55 As a convention, "0" does not stand for a specific word, but instead is used 56 to encode any unknown word. 57 58 Args: 59 path: where to cache the data (relative to `~/.keras/dataset`). 60 num_words: integer or None. Words are 61 ranked by how often they occur (in the training set) and only 62 the `num_words` most frequent words are kept. Any less frequent word 63 will appear as `oov_char` value in the sequence data. If None, 64 all words are kept. Defaults to None, so all words are kept. 65 skip_top: skip the top N most frequently occurring words 66 (which may not be informative). These words will appear as 67 `oov_char` value in the dataset. Defaults to 0, so no words are 68 skipped. 69 maxlen: int or None. Maximum sequence length. 70 Any longer sequence will be truncated. Defaults to None, which 71 means no truncation. 72 test_split: Float between 0 and 1. Fraction of the dataset to be used 73 as test data. Defaults to 0.2, meaning 20% of the dataset is used as 74 test data. 75 seed: int. Seed for reproducible data shuffling. 76 start_char: int. The start of a sequence will be marked with this 77 character. Defaults to 1 because 0 is usually the padding character. 78 oov_char: int. The out-of-vocabulary character. 79 Words that were cut out because of the `num_words` or 80 `skip_top` limits will be replaced with this character. 81 index_from: int. Index actual words with this index and higher. 82 **kwargs: Used for backwards compatibility. 83 84 Returns: 85 Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`. 86 87 **x_train, x_test**: lists of sequences, which are lists of indexes 88 (integers). If the num_words argument was specific, the maximum 89 possible index value is `num_words - 1`. If the `maxlen` argument was 90 specified, the largest possible sequence length is `maxlen`. 91 92 **y_train, y_test**: lists of integer labels (1 or 0). 93 94 Note: The 'out of vocabulary' character is only used for 95 words that were present in the training set but are not included 96 because they're not making the `num_words` cut here. 97 Words that were not seen in the training set but are in the test set 98 have simply been skipped. 99 """ 100 # Legacy support 101 if 'nb_words' in kwargs: 102 logging.warning('The `nb_words` argument in `load_data` ' 103 'has been renamed `num_words`.') 104 num_words = kwargs.pop('nb_words') 105 if kwargs: 106 raise TypeError('Unrecognized keyword arguments: ' + str(kwargs)) 107 108 origin_folder = 'https://storage.googleapis.com/tensorflow/tf-keras-datasets/' 109 path = get_file( 110 path, 111 origin=origin_folder + 'reuters.npz', 112 file_hash= 113 'd6586e694ee56d7a4e65172e12b3e987c03096cb01eab99753921ef915959916') 114 with np.load(path, allow_pickle=True) as f: # pylint: disable=unexpected-keyword-arg 115 xs, labels = f['x'], f['y'] 116 117 rng = np.random.RandomState(seed) 118 indices = np.arange(len(xs)) 119 rng.shuffle(indices) 120 xs = xs[indices] 121 labels = labels[indices] 122 123 if start_char is not None: 124 xs = [[start_char] + [w + index_from for w in x] for x in xs] 125 elif index_from: 126 xs = [[w + index_from for w in x] for x in xs] 127 128 if maxlen: 129 xs, labels = _remove_long_seq(maxlen, xs, labels) 130 131 if not num_words: 132 num_words = max(max(x) for x in xs) 133 134 # by convention, use 2 as OOV word 135 # reserve 'index_from' (=3 by default) characters: 136 # 0 (padding), 1 (start), 2 (OOV) 137 if oov_char is not None: 138 xs = [[w if skip_top <= w < num_words else oov_char for w in x] for x in xs] 139 else: 140 xs = [[w for w in x if skip_top <= w < num_words] for x in xs] 141 142 idx = int(len(xs) * (1 - test_split)) 143 x_train, y_train = np.array(xs[:idx], dtype='object'), np.array(labels[:idx]) 144 x_test, y_test = np.array(xs[idx:], dtype='object'), np.array(labels[idx:]) 145 146 return (x_train, y_train), (x_test, y_test) 147 148 149@keras_export('keras.datasets.reuters.get_word_index') 150def get_word_index(path='reuters_word_index.json'): 151 """Retrieves a dict mapping words to their index in the Reuters dataset. 152 153 Args: 154 path: where to cache the data (relative to `~/.keras/dataset`). 155 156 Returns: 157 The word index dictionary. Keys are word strings, values are their index. 158 """ 159 origin_folder = 'https://storage.googleapis.com/tensorflow/tf-keras-datasets/' 160 path = get_file( 161 path, 162 origin=origin_folder + 'reuters_word_index.json', 163 file_hash='4d44cc38712099c9e383dc6e5f11a921') 164 with open(path) as f: 165 return json.load(f) 166