Home
last modified time | relevance | path

Searched refs:compression_type (Results 1 – 25 of 134) sorted by relevance

123456

/external/tensorflow/tensorflow/python/kernel_tests/
Ddecode_compressed_op_test.py35 def _compress(self, bytes_in, compression_type): argument
36 if not compression_type:
38 elif compression_type == "ZLIB":
48 for compression_type in ["ZLIB", "GZIP", ""]:
52 in_bytes, compression_type=compression_type)
56 for compression_type in ["ZLIB", "GZIP", ""]:
59 def decode(in_bytes, compression_type=compression_type): argument
61 in_bytes, compression_type=compression_type)
63 in_val = [self._compress(b"AaAA", compression_type),
64 self._compress(b"bBbb", compression_type)]
[all …]
Drecord_input_test.py36 compression_type=tf_record.TFRecordCompressionType.NONE): argument
37 options = tf_record.TFRecordOptions(compression_type)
66 compression_type=tf_record.TFRecordCompressionType.GZIP)
74 compression_type=tf_record.TFRecordCompressionType.GZIP).get_yield_op(
85 compression_type=tf_record.TFRecordCompressionType.ZLIB)
93 compression_type=tf_record.TFRecordCompressionType.ZLIB).get_yield_op(
/external/tensorflow/tensorflow/core/lib/io/
Drecord_writer.cc27 return options.compression_type == RecordWriterOptions::ZLIB_COMPRESSION; in IsZlibCompressed()
31 return options.compression_type == RecordWriterOptions::SNAPPY_COMPRESSION; in IsSnappyCompressed()
36 const string& compression_type) { in CreateRecordWriterOptions() argument
39 if (compression_type != compression::kNone) { in CreateRecordWriterOptions()
44 if (compression_type == compression::kZlib) { in CreateRecordWriterOptions()
45 options.compression_type = io::RecordWriterOptions::ZLIB_COMPRESSION; in CreateRecordWriterOptions()
47 } else if (compression_type == compression::kGzip) { in CreateRecordWriterOptions()
48 options.compression_type = io::RecordWriterOptions::ZLIB_COMPRESSION; in CreateRecordWriterOptions()
50 } else if (compression_type == compression::kSnappy) { in CreateRecordWriterOptions()
51 options.compression_type = io::RecordWriterOptions::SNAPPY_COMPRESSION; in CreateRecordWriterOptions()
[all …]
Drecord_reader.cc32 const string& compression_type) { in CreateRecordReaderOptions() argument
36 if (compression_type != compression::kNone) { in CreateRecordReaderOptions()
41 if (compression_type == compression::kZlib) { in CreateRecordReaderOptions()
42 options.compression_type = io::RecordReaderOptions::ZLIB_COMPRESSION; in CreateRecordReaderOptions()
44 } else if (compression_type == compression::kGzip) { in CreateRecordReaderOptions()
45 options.compression_type = io::RecordReaderOptions::ZLIB_COMPRESSION; in CreateRecordReaderOptions()
47 } else if (compression_type == compression::kSnappy) { in CreateRecordReaderOptions()
48 options.compression_type = io::RecordReaderOptions::SNAPPY_COMPRESSION; in CreateRecordReaderOptions()
49 } else if (compression_type != compression::kNone) { in CreateRecordReaderOptions()
50 LOG(ERROR) << "Unsupported compression_type:" << compression_type in CreateRecordReaderOptions()
[all …]
/external/tensorflow/tensorflow/python/data/ops/
Dreaders.py111 def __init__(self, filenames, compression_type=None, buffer_size=None): argument
125 compression_type,
148 compression_type=None, argument
174 self._compression_type = compression_type
178 return _TextLineDataset(filename, compression_type, buffer_size)
197 compression_type=None, argument
200 wrapped = TextLineDatasetV2(filenames, compression_type, buffer_size,
218 def __init__(self, filenames, compression_type=None, buffer_size=None): argument
231 compression_type,
308 compression_type=None, argument
[all …]
/external/tensorflow/tensorflow/core/kernels/data/
Dtext_line_dataset_op_test.cc31 CompressionType compression_type, int64 buffer_size, in TextLineDatasetParams() argument
36 compression_type_(compression_type),
76 CompressionType compression_type) { in CreateTestFiles() argument
83 params.compression_type = compression_type; in CreateTestFiles()
97 CompressionType compression_type = CompressionType::ZLIB; in TextLineDatasetParams1() local
98 if (!CreateTestFiles(filenames, contents, compression_type).ok()) { in TextLineDatasetParams1()
103 /*compression_type=*/compression_type, in TextLineDatasetParams1()
114 CompressionType compression_type = CompressionType::GZIP; in TextLineDatasetParams2() local
115 if (!CreateTestFiles(filenames, contents, compression_type).ok()) { in TextLineDatasetParams2()
120 /*compression_type=*/compression_type, in TextLineDatasetParams2()
[all …]
Dfixed_length_record_dataset_op_test.cc34 CompressionType compression_type, in FixedLengthRecordDatasetParams() argument
43 compression_type_(compression_type) {
91 CompressionType compression_type) { in CreateTestFiles() argument
96 if (compression_type == CompressionType::UNCOMPRESSED) { in CreateTestFiles()
103 params.compression_type = compression_type; in CreateTestFiles()
118 CompressionType compression_type = CompressionType::ZLIB; in FixedLengthRecordDatasetParams1() local
119 if (!CreateTestFiles(filenames, contents, compression_type).ok()) { in FixedLengthRecordDatasetParams1()
129 /*compression_type=*/compression_type, in FixedLengthRecordDatasetParams1()
139 CompressionType compression_type = CompressionType::GZIP; in FixedLengthRecordDatasetParams2() local
140 if (!CreateTestFiles(filenames, contents, compression_type).ok()) { in FixedLengthRecordDatasetParams2()
[all …]
Dtf_record_dataset_op_test.cc25 CompressionType compression_type, int64 buffer_size, in TFRecordDatasetParams() argument
30 compression_type_(compression_type),
70 CompressionType compression_type) { in CreateTestFiles() argument
78 params.compression_type = compression_type; in CreateTestFiles()
93 CompressionType compression_type = CompressionType::ZLIB; in TFRecordDatasetParams1() local
94 if (!CreateTestFiles(filenames, contents, compression_type).ok()) { in TFRecordDatasetParams1()
99 /*compression_type=*/compression_type, in TFRecordDatasetParams1()
111 CompressionType compression_type = CompressionType::GZIP; in TFRecordDatasetParams2() local
112 if (!CreateTestFiles(filenames, contents, compression_type).ok()) { in TFRecordDatasetParams2()
117 /*compression_type=*/compression_type, in TFRecordDatasetParams2()
[all …]
Dtext_line_dataset_op.cc43 const string& compression_type, in Dataset() argument
47 compression_type_(compression_type), in Dataset()
48 use_compression_(!compression_type.empty()), in Dataset()
84 Node* compression_type = nullptr; in AsGraphDefInternal() local
87 TF_RETURN_IF_ERROR(b->AddScalar(compression_type_, &compression_type)); in AsGraphDefInternal()
90 this, {filenames, compression_type, buffer_size}, output)); in AsGraphDefInternal()
247 tstring compression_type; in MakeDataset() local
249 &compression_type)); in MakeDataset()
260 if (compression_type == kZLIB) { in MakeDataset()
262 } else if (compression_type == kGZIP) { in MakeDataset()
[all …]
Dtf_record_dataset_op.cc56 const string& compression_type, int64 buffer_size) in Dataset() argument
59 compression_type_(compression_type), in Dataset()
61 compression_type)) { in Dataset()
100 Node* compression_type = nullptr; in AsGraphDefInternal() local
101 TF_RETURN_IF_ERROR(b->AddScalar(compression_type_, &compression_type)); in AsGraphDefInternal()
105 this, {filenames, compression_type, buffer_size}, output)); in AsGraphDefInternal()
298 tstring compression_type; in MakeDataset() local
300 &compression_type)); in MakeDataset()
325 new Dataset(ctx, std::move(filenames), compression_type, buffer_size); in MakeDataset()
Dfixed_length_record_dataset_op.cc54 int64 buffer_size, const string& compression_type, in Dataset() argument
62 compression_type_(compression_type), in Dataset()
111 Node* compression_type = nullptr; in AsGraphDefInternal() local
117 TF_RETURN_IF_ERROR(b->AddScalar(compression_type_, &compression_type)); in AsGraphDefInternal()
121 buffer_size, compression_type}, in AsGraphDefInternal()
503 tstring compression_type; in MakeDataset() local
506 &compression_type)); in MakeDataset()
508 compression_type.empty() || compression_type == kZLIB || in MakeDataset()
509 compression_type == kGZIP, in MakeDataset()
514 footer_bytes, buffer_size, compression_type, op_version_); in MakeDataset()
/external/tensorflow/tensorflow/python/data/experimental/kernel_tests/serialization/
Dtextline_dataset_serialization_test.py35 def _build_iterator_graph(self, test_filenames, compression_type=None): argument
37 test_filenames, compression_type=compression_type, buffer_size=10)
45 for compression_type in compression_types:
50 compression_type=compression_type)
53 lambda: self._build_iterator_graph(test_filenames, compression_type),
Dtf_record_dataset_serialization_test.py42 compression_type=None, argument
45 if compression_type == "ZLIB":
56 elif compression_type == "GZIP":
67 filenames, compression_type,
97 lambda: self._build_iterator_graph(num_epochs, compression_type="ZLIB"),
100 lambda: self._build_iterator_graph(num_epochs, compression_type="GZIP"),
Dcsv_dataset_serialization_test.py59 compression_type = kwargs.get("compression_type", None)
60 if compression_type == "GZIP":
62 elif compression_type is None:
65 raise ValueError("Invalid compression type:", compression_type)
/external/tensorflow/tensorflow/python/lib/io/
Dtf_record.py52 compression_type=None, argument
90 self.get_compression_type_string(compression_type)
91 self.compression_type = compression_type
117 return cls.get_compression_type_string(options.compression_type)
131 self.get_compression_type_string(self.compression_type)))
170 compression_type = TFRecordOptions.get_compression_type_string(options)
171 return _pywrap_record_io.RecordIterator(path, compression_type)
295 options = TFRecordOptions(compression_type=options)
/external/tensorflow/tensorflow/python/data/kernel_tests/
Dtext_line_dataset_test.py51 compression_type=None): argument
65 if not compression_type:
68 elif compression_type == "GZIP":
71 elif compression_type == "ZLIB":
76 raise ValueError("Unsupported compression_type", compression_type)
83 combinations.combine(compression_type=[None, "GZIP", "ZLIB"])))
84 def testTextLineDataset(self, compression_type): argument
86 2, 5, crlf=True, compression_type=compression_type)
90 filenames, compression_type=compression_type).repeat(num_epochs)
Dfixed_length_record_dataset_test.py49 def _createFiles(self, compression_type=None): argument
62 if not compression_type:
65 elif compression_type == "GZIP":
68 elif compression_type == "ZLIB":
73 raise ValueError("Unsupported compression_type", compression_type)
77 def _testFixedLengthRecordDataset(self, compression_type=None): argument
78 test_filenames = self._createFiles(compression_type=compression_type)
86 compression_type=compression_type).repeat(num_epochs)
138 self._testFixedLengthRecordDataset(compression_type="GZIP")
142 self._testFixedLengthRecordDataset(compression_type="ZLIB")
/external/bsdiff/
Dpatch_reader.cc39 std::vector<CompressorType> compression_type; in Init() local
42 compression_type = {CompressorType::kBZ2, CompressorType::kBZ2, in Init()
54 compression_type.push_back(CompressorType::kBZ2); in Init()
57 compression_type.push_back(CompressorType::kBrotli); in Init()
86 ctrl_stream_ = CreateDecompressor(compression_type[0]); in Init()
87 diff_stream_ = CreateDecompressor(compression_type[1]); in Init()
88 extra_stream_ = CreateDecompressor(compression_type[2]); in Init()
/external/tensorflow/tensorflow/core/kernels/
Dtf_record_reader_op.cc30 TFRecordReader(const string& node_name, const string& compression_type, in TFRecordReader() argument
35 compression_type_(compression_type) {} in TFRecordReader()
89 string compression_type; in TFRecordReaderOp() local
91 context->GetAttr("compression_type", &compression_type)); in TFRecordReaderOp()
93 SetReaderFactory([this, compression_type, env]() { in TFRecordReaderOp()
94 return new TFRecordReader(name(), compression_type, env); in TFRecordReaderOp()
Drecord_input_op.cc39 GETATTR(string, compression_type); in RecordInputOp()
42 OP_REQUIRES_OK(ctx, ctx->GetAttr("compression_type", &compression_type)); in RecordInputOp()
50 yopts.compression_type = compression_type; in RecordInputOp()
/external/tensorflow/tensorflow/core/kernels/data/experimental/
Dsnapshot_util_test.cc41 void SnapshotRoundTrip(std::string compression_type, int version) { in SnapshotRoundTrip() argument
52 compression_type, version, dtypes, &writer)); in SnapshotRoundTrip()
60 TF_ASSERT_OK(Reader::Create(Env::Default(), filename, compression_type, in SnapshotRoundTrip()
95 std::string compression_type, int version) { in SnapshotReaderBenchmarkLoop() argument
105 compression_type, version, dtypes, &writer)); in SnapshotReaderBenchmarkLoop()
113 TF_ASSERT_OK(Reader::Create(Env::Default(), filename, compression_type, in SnapshotReaderBenchmarkLoop()
151 std::string compression_type, int version) { in SnapshotWriterBenchmarkLoop() argument
161 compression_type, version, dtypes, &writer)); in SnapshotWriterBenchmarkLoop()
Dsnapshot_util.h77 const std::string& compression_type, int version,
101 const std::string& compression_type);
132 CustomWriter(const std::string& filename, const std::string& compression_type,
173 const string& compression_type, int version,
184 const string& compression_type, int version,
209 TFRecordReader(const std::string& filename, const string& compression_type,
247 CustomReader(const std::string& filename, const string& compression_type,
/external/tensorflow/tensorflow/python/data/experimental/kernel_tests/
Dcsv_dataset_test.py42 def _setup_files(self, inputs, linebreak='\n', compression_type=None): argument
47 if compression_type is None:
50 elif compression_type == 'GZIP':
53 elif compression_type == 'ZLIB':
58 raise ValueError('Unsupported compression_type', compression_type)
83 compression_type=None, # Used for both setup and parsing argument
87 filenames = self._setup_files(inputs, linebreak, compression_type)
88 kwargs['compression_type'] = compression_type
484 compression_type=None, argument
492 compression_type=compression_type,
[all …]
Dtf_record_writer_test.py45 def writer_fn(self, filename, compression_type=""): argument
46 input_dataset = readers.TFRecordDataset([filename], compression_type)
48 compression_type).write(input_dataset)
77 self.writer_fn(self._createFile(options), compression_type="ZLIB"))
86 self.writer_fn(self._createFile(options), compression_type="GZIP"))
/external/tensorflow/tensorflow/python/data/experimental/ops/
Dreaders.py341 compression_type=None, argument
489 if compression_type is not None:
490 compression_type_value = tensor_util.constant_value(compression_type)
498 compression_type)
501 compression_type)
550 compression_type=compression_type
630 compression_type=None, argument
638 compression_type, ignore_errors))
652 compression_type=None, argument
758 compression_type,
[all …]

123456