Home
last modified time | relevance | path

Searched refs:compression_type (Results 1 – 25 of 129) sorted by relevance

123456

/external/tensorflow/tensorflow/python/kernel_tests/
Ddecode_compressed_op_test.py35 def _compress(self, bytes_in, compression_type): argument
36 if not compression_type:
38 elif compression_type == "ZLIB":
48 for compression_type in ["ZLIB", "GZIP", ""]:
52 in_bytes, compression_type=compression_type)
56 for compression_type in ["ZLIB", "GZIP", ""]:
59 def decode(in_bytes, compression_type=compression_type): argument
61 in_bytes, compression_type=compression_type)
63 in_val = [self._compress(b"AaAA", compression_type),
64 self._compress(b"bBbb", compression_type)]
[all …]
Drecord_input_test.py36 compression_type=tf_record.TFRecordCompressionType.NONE): argument
37 options = tf_record.TFRecordOptions(compression_type)
66 compression_type=tf_record.TFRecordCompressionType.GZIP)
74 compression_type=tf_record.TFRecordCompressionType.GZIP).get_yield_op(
85 compression_type=tf_record.TFRecordCompressionType.ZLIB)
93 compression_type=tf_record.TFRecordCompressionType.ZLIB).get_yield_op(
/external/tensorflow/tensorflow/core/lib/io/
Drecord_writer.cc27 return options.compression_type == RecordWriterOptions::ZLIB_COMPRESSION; in IsZlibCompressed()
31 return options.compression_type == RecordWriterOptions::SNAPPY_COMPRESSION; in IsSnappyCompressed()
36 const string& compression_type) { in CreateRecordWriterOptions() argument
39 if (compression_type != compression::kNone) { in CreateRecordWriterOptions()
44 if (compression_type == compression::kZlib) { in CreateRecordWriterOptions()
45 options.compression_type = io::RecordWriterOptions::ZLIB_COMPRESSION; in CreateRecordWriterOptions()
47 } else if (compression_type == compression::kGzip) { in CreateRecordWriterOptions()
48 options.compression_type = io::RecordWriterOptions::ZLIB_COMPRESSION; in CreateRecordWriterOptions()
50 } else if (compression_type == compression::kSnappy) { in CreateRecordWriterOptions()
51 options.compression_type = io::RecordWriterOptions::SNAPPY_COMPRESSION; in CreateRecordWriterOptions()
[all …]
Drecord_reader.cc32 const string& compression_type) { in CreateRecordReaderOptions() argument
36 if (compression_type != compression::kNone) { in CreateRecordReaderOptions()
41 if (compression_type == compression::kZlib) { in CreateRecordReaderOptions()
42 options.compression_type = io::RecordReaderOptions::ZLIB_COMPRESSION; in CreateRecordReaderOptions()
44 } else if (compression_type == compression::kGzip) { in CreateRecordReaderOptions()
45 options.compression_type = io::RecordReaderOptions::ZLIB_COMPRESSION; in CreateRecordReaderOptions()
47 } else if (compression_type == compression::kSnappy) { in CreateRecordReaderOptions()
48 options.compression_type = io::RecordReaderOptions::SNAPPY_COMPRESSION; in CreateRecordReaderOptions()
49 } else if (compression_type != compression::kNone) { in CreateRecordReaderOptions()
50 LOG(ERROR) << "Unsupported compression_type:" << compression_type in CreateRecordReaderOptions()
[all …]
Drecord_reader_writer_test.cc41 if (options.compression_type == io::RecordWriterOptions::ZLIB_COMPRESSION) { in GetMatchingReaderOptions()
110 options.compression_type = io::RecordWriterOptions::ZLIB_COMPRESSION; in TEST()
244 options.compression_type = io::RecordWriterOptions::SNAPPY_COMPRESSION; in TEST()
257 options.compression_type = io::RecordReaderOptions::SNAPPY_COMPRESSION; in TEST()
282 options.compression_type = io::RecordWriterOptions::ZLIB_COMPRESSION; in TEST()
295 options.compression_type = io::RecordReaderOptions::ZLIB_COMPRESSION; in TEST()
317 options.compression_type = io::RecordWriterOptions::ZLIB_COMPRESSION; in TEST()
/external/tensorflow/tensorflow/python/data/kernel_tests/
Dtext_line_dataset_test.py53 compression_type=None): argument
67 if not compression_type:
70 elif compression_type == "GZIP":
73 elif compression_type == "ZLIB":
78 raise ValueError("Unsupported compression_type", compression_type)
88 combinations.combine(compression_type=[None, "GZIP", "ZLIB"])))
89 def testTextLineDataset(self, compression_type): argument
91 2, 5, crlf=True, compression_type=compression_type)
95 filenames, compression_type=compression_type).repeat(num_epochs)
192 def _build_iterator_graph(self, test_filenames, compression_type=None): argument
[all …]
Dfixed_length_record_dataset_test.py50 def _createFiles(self, compression_type=None): argument
63 if not compression_type:
66 elif compression_type == "GZIP":
69 elif compression_type == "ZLIB":
74 raise ValueError("Unsupported compression_type", compression_type)
82 def _testFixedLengthRecordDataset(self, compression_type=None): argument
83 test_filenames = self._createFiles(compression_type=compression_type)
91 compression_type=compression_type).repeat(num_epochs)
143 self._testFixedLengthRecordDataset(compression_type="GZIP")
147 self._testFixedLengthRecordDataset(compression_type="ZLIB")
[all …]
Dtf_record_dataset_test.py42 compression_type="", argument
47 filenames, compression_type).repeat(num_epochs)
120 dataset = self._dataset_factory(zlib_files, compression_type="ZLIB")
136 dataset = self._dataset_factory(gzip_files, compression_type="GZIP")
189 compression_type=None, argument
192 if compression_type == "ZLIB":
203 elif compression_type == "GZIP":
214 filenames, compression_type,
245 combinations.combine(compression_type=[None, "GZIP", "ZLIB"])))
246 def testCompressionTypes(self, verify_fn, compression_type): argument
[all …]
/external/tensorflow/tensorflow/python/data/ops/
Dreaders.py111 def __init__(self, filenames, compression_type=None, buffer_size=None): argument
125 compression_type,
180 compression_type=None, argument
206 self._compression_type = compression_type
210 return _TextLineDataset(filename, compression_type, buffer_size)
229 compression_type=None, argument
232 wrapped = TextLineDatasetV2(filenames, compression_type, buffer_size,
250 def __init__(self, filenames, compression_type=None, buffer_size=None): argument
263 compression_type,
381 compression_type=None, argument
[all …]
/external/tensorflow/tensorflow/core/kernels/data/
Dtext_line_dataset_op_test.cc31 CompressionType compression_type, int64_t buffer_size, in TextLineDatasetParams() argument
36 compression_type_(compression_type),
76 CompressionType compression_type) { in CreateTestFiles() argument
83 params.compression_type = compression_type; in CreateTestFiles()
97 CompressionType compression_type = CompressionType::ZLIB; in TextLineDatasetParams1() local
98 if (!CreateTestFiles(filenames, contents, compression_type).ok()) { in TextLineDatasetParams1()
103 /*compression_type=*/compression_type, in TextLineDatasetParams1()
114 CompressionType compression_type = CompressionType::GZIP; in TextLineDatasetParams2() local
115 if (!CreateTestFiles(filenames, contents, compression_type).ok()) { in TextLineDatasetParams2()
120 /*compression_type=*/compression_type, in TextLineDatasetParams2()
[all …]
Dfixed_length_record_dataset_op_test.cc34 CompressionType compression_type, in FixedLengthRecordDatasetParams() argument
43 compression_type_(compression_type) {
91 CompressionType compression_type) { in CreateTestFiles() argument
96 if (compression_type == CompressionType::UNCOMPRESSED) { in CreateTestFiles()
103 params.compression_type = compression_type; in CreateTestFiles()
118 CompressionType compression_type = CompressionType::ZLIB; in FixedLengthRecordDatasetParams1() local
119 if (!CreateTestFiles(filenames, contents, compression_type).ok()) { in FixedLengthRecordDatasetParams1()
129 /*compression_type=*/compression_type, in FixedLengthRecordDatasetParams1()
139 CompressionType compression_type = CompressionType::GZIP; in FixedLengthRecordDatasetParams2() local
140 if (!CreateTestFiles(filenames, contents, compression_type).ok()) { in FixedLengthRecordDatasetParams2()
[all …]
Dtf_record_dataset_op_test.cc25 CompressionType compression_type, int64_t buffer_size, in TFRecordDatasetParams() argument
30 compression_type_(compression_type),
70 CompressionType compression_type) { in CreateTestFiles() argument
78 params.compression_type = compression_type; in CreateTestFiles()
93 CompressionType compression_type = CompressionType::ZLIB; in TFRecordDatasetParams1() local
94 if (!CreateTestFiles(filenames, contents, compression_type).ok()) { in TFRecordDatasetParams1()
99 /*compression_type=*/compression_type, in TFRecordDatasetParams1()
111 CompressionType compression_type = CompressionType::GZIP; in TFRecordDatasetParams2() local
112 if (!CreateTestFiles(filenames, contents, compression_type).ok()) { in TFRecordDatasetParams2()
117 /*compression_type=*/compression_type, in TFRecordDatasetParams2()
[all …]
Dtext_line_dataset_op.cc43 const string& compression_type, in Dataset() argument
47 compression_type_(compression_type), in Dataset()
48 use_compression_(!compression_type.empty()), in Dataset()
84 Node* compression_type = nullptr; in AsGraphDefInternal() local
87 TF_RETURN_IF_ERROR(b->AddScalar(compression_type_, &compression_type)); in AsGraphDefInternal()
90 this, {filenames, compression_type, buffer_size}, output)); in AsGraphDefInternal()
247 tstring compression_type; in MakeDataset() local
249 &compression_type)); in MakeDataset()
260 if (compression_type == kZLIB) { in MakeDataset()
262 } else if (compression_type == kGZIP) { in MakeDataset()
[all …]
Dtf_record_dataset_op.cc57 const string& compression_type, int64_t buffer_size) in Dataset() argument
60 compression_type_(compression_type), in Dataset()
62 compression_type)) { in Dataset()
101 Node* compression_type = nullptr; in AsGraphDefInternal() local
102 TF_RETURN_IF_ERROR(b->AddScalar(compression_type_, &compression_type)); in AsGraphDefInternal()
106 this, {filenames, compression_type, buffer_size}, output)); in AsGraphDefInternal()
299 tstring compression_type; in MakeDataset() local
301 &compression_type)); in MakeDataset()
326 new Dataset(ctx, std::move(filenames), compression_type, buffer_size); in MakeDataset()
Dfixed_length_record_dataset_op.cc55 const string& compression_type, int op_version) in Dataset() argument
62 compression_type_(compression_type), in Dataset()
111 Node* compression_type = nullptr; in AsGraphDefInternal() local
117 TF_RETURN_IF_ERROR(b->AddScalar(compression_type_, &compression_type)); in AsGraphDefInternal()
121 buffer_size, compression_type}, in AsGraphDefInternal()
503 tstring compression_type; in MakeDataset() local
506 &compression_type)); in MakeDataset()
508 compression_type.empty() || compression_type == kZLIB || in MakeDataset()
509 compression_type == kGZIP, in MakeDataset()
514 footer_bytes, buffer_size, compression_type, op_version_); in MakeDataset()
/external/tensorflow/tensorflow/python/lib/io/
Dtf_record.py52 compression_type=None, argument
90 self.get_compression_type_string(compression_type)
91 self.compression_type = compression_type
117 return cls.get_compression_type_string(options.compression_type)
131 self.get_compression_type_string(self.compression_type)))
170 compression_type = TFRecordOptions.get_compression_type_string(options)
171 return _pywrap_record_io.RecordIterator(path, compression_type)
295 options = TFRecordOptions(compression_type=options)
/external/bsdiff/
Dpatch_reader.cc39 std::vector<CompressorType> compression_type; in Init() local
42 compression_type = {CompressorType::kBZ2, CompressorType::kBZ2, in Init()
54 compression_type.push_back(CompressorType::kBZ2); in Init()
57 compression_type.push_back(CompressorType::kBrotli); in Init()
86 ctrl_stream_ = CreateDecompressor(compression_type[0]); in Init()
87 diff_stream_ = CreateDecompressor(compression_type[1]); in Init()
88 extra_stream_ = CreateDecompressor(compression_type[2]); in Init()
/external/tensorflow/tensorflow/core/kernels/
Dtf_record_reader_op.cc30 TFRecordReader(const string& node_name, const string& compression_type, in TFRecordReader() argument
35 compression_type_(compression_type) {} in TFRecordReader()
89 string compression_type; in TFRecordReaderOp() local
91 context->GetAttr("compression_type", &compression_type)); in TFRecordReaderOp()
93 SetReaderFactory([this, compression_type, env]() { in TFRecordReaderOp()
94 return new TFRecordReader(name(), compression_type, env); in TFRecordReaderOp()
Drecord_input_op.cc39 GETATTR(string, compression_type); in RecordInputOp()
42 OP_REQUIRES_OK(ctx, ctx->GetAttr("compression_type", &compression_type)); in RecordInputOp()
50 yopts.compression_type = compression_type; in RecordInputOp()
/external/tensorflow/tensorflow/core/data/
Dsnapshot_utils_test.cc41 void SnapshotRoundTrip(std::string compression_type, int version) { in SnapshotRoundTrip() argument
52 compression_type, version, dtypes, &writer)); in SnapshotRoundTrip()
60 TF_ASSERT_OK(Reader::Create(Env::Default(), filename, compression_type, in SnapshotRoundTrip()
95 std::string compression_type, int version) { in SnapshotReaderBenchmarkLoop() argument
105 compression_type, version, dtypes, &writer)); in SnapshotReaderBenchmarkLoop()
113 TF_ASSERT_OK(Reader::Create(Env::Default(), filename, compression_type, in SnapshotReaderBenchmarkLoop()
151 std::string compression_type, int version) { in SnapshotWriterBenchmarkLoop() argument
161 compression_type, version, dtypes, &writer)); in SnapshotWriterBenchmarkLoop()
Dsnapshot_utils.h78 const std::string& compression_type, int version,
102 const std::string& compression_type);
133 CustomWriter(const std::string& filename, const std::string& compression_type,
204 const string& compression_type, int version,
215 const string& compression_type, int version,
240 TFRecordReader(const std::string& filename, const string& compression_type,
278 CustomReader(const std::string& filename, const string& compression_type,
/external/tensorflow/tensorflow/python/data/experimental/kernel_tests/
Dcsv_dataset_test.py43 def _setup_files(self, inputs, linebreak='\n', compression_type=None): argument
48 if compression_type is None:
51 elif compression_type == 'GZIP':
54 elif compression_type == 'ZLIB':
59 raise ValueError('Unsupported compression_type', compression_type)
84 compression_type=None, # Used for both setup and parsing argument
88 filenames = self._setup_files(inputs, linebreak, compression_type)
89 kwargs['compression_type'] = compression_type
470 compression_type=None, argument
478 compression_type=compression_type,
[all …]
Dtf_record_writer_test.py45 def writer_fn(self, filename, compression_type=""): argument
46 input_dataset = readers.TFRecordDataset([filename], compression_type)
48 compression_type).write(input_dataset)
77 self.writer_fn(self._createFile(options), compression_type="ZLIB"))
86 self.writer_fn(self._createFile(options), compression_type="GZIP"))
/external/tensorflow/tensorflow/python/data/experimental/ops/
Dreaders.py341 compression_type=None, argument
489 if compression_type is not None:
490 compression_type_value = tensor_util.constant_value(compression_type)
498 compression_type)
501 compression_type)
550 compression_type=compression_type
630 compression_type=None, argument
638 compression_type, ignore_errors))
692 compression_type=None, argument
749 compression_type,
[all …]
/external/tensorflow/tensorflow/core/kernels/data/experimental/
Dcsv_dataset_op.cc45 tstring compression_type; in MakeDataset() local
47 &compression_type)); in MakeDataset()
115 if (compression_type == "ZLIB") { in MakeDataset()
117 } else if (compression_type == "GZIP") { in MakeDataset()
120 OP_REQUIRES(ctx, compression_type.empty(), in MakeDataset()
122 "Unsupported compression_type: ", compression_type, ".")); in MakeDataset()
156 std::move(compression_type), zlib_compression_options, in MakeDataset()
167 string compression_type, io::ZlibCompressionOptions options, in Dataset() argument
185 use_compression_(!compression_type.empty()), in Dataset()
186 compression_type_(std::move(compression_type)), in Dataset()
[all …]

123456