1 /*
2 * Copyright 2021 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8 #include "src/codec/SkJpegxlCodec.h"
9
10 #include "include/codec/SkCodec.h"
11 #include "include/core/SkColorType.h"
12 #include "include/core/SkData.h"
13 #include "include/core/SkImageInfo.h"
14 #include "include/core/SkRefCnt.h"
15 #include "include/core/SkStream.h"
16 #include "include/core/SkTypes.h"
17 #include "include/private/SkEncodedInfo.h"
18 #include "include/private/base/SkTFitsIn.h"
19 #include "include/private/base/SkTemplates.h"
20 #include "include/private/base/SkTo.h"
21 #include "modules/skcms/skcms.h"
22 #include "src/codec/SkFrameHolder.h"
23 #include "src/core/SkOpts.h"
24 #include "src/core/SkStreamPriv.h"
25
26 #include "jxl/codestream_header.h"
27 #include "jxl/decode.h"
28 #include "jxl/decode_cxx.h"
29 #include "jxl/types.h"
30
31 #include <cstdint>
32 #include <cstring>
33 #include <limits>
34 #include <utility>
35 #include <vector>
36
37 namespace {
38
39 class Frame : public SkFrame {
40 public:
Frame(int i,SkEncodedInfo::Alpha alpha)41 explicit Frame(int i, SkEncodedInfo::Alpha alpha) : INHERITED(i), fReportedAlpha(alpha) {}
onReportedAlpha() const42 SkEncodedInfo::Alpha onReportedAlpha() const override { return fReportedAlpha; }
43
44 private:
45 const SkEncodedInfo::Alpha fReportedAlpha;
46
47 using INHERITED = SkFrame;
48 };
49
50 } // namespace
51
IsJpegxl(const void * buffer,size_t bytesRead)52 bool SkJpegxlCodec::IsJpegxl(const void* buffer, size_t bytesRead) {
53 JxlSignature result = JxlSignatureCheck(reinterpret_cast<const uint8_t*>(buffer), bytesRead);
54 return (result == JXL_SIG_CODESTREAM) || (result == JXL_SIG_CONTAINER);
55 }
56
57 class SkJpegxlCodecPriv : public SkFrameHolder {
58 public:
SkJpegxlCodecPriv()59 SkJpegxlCodecPriv() : fDecoder(JxlDecoderMake(/* memory_manager= */ nullptr)) {}
60 JxlDecoderPtr fDecoder; // unique_ptr with custom destructor
61 JxlBasicInfo fInfo;
62 bool fSeenAllFrames = false;
63 std::vector<Frame> fFrames;
64 int fLastProcessedFrame = SkCodec::kNoFrame;
65 void* fDst;
66 size_t fPixelShift;
67 size_t fRowBytes;
68 SkColorType fDstColorType;
69
70 protected:
onGetFrame(int i) const71 const SkFrame* onGetFrame(int i) const override {
72 SkASSERT(i >= 0 && static_cast<size_t>(i) < fFrames.size());
73 return static_cast<const SkFrame*>(&fFrames[i]);
74 }
75 };
76
SkJpegxlCodec(std::unique_ptr<SkJpegxlCodecPriv> codec,SkEncodedInfo && info,std::unique_ptr<SkStream> stream,sk_sp<SkData> data)77 SkJpegxlCodec::SkJpegxlCodec(std::unique_ptr<SkJpegxlCodecPriv> codec,
78 SkEncodedInfo&& info,
79 std::unique_ptr<SkStream> stream,
80 sk_sp<SkData> data)
81 : INHERITED(std::move(info), skcms_PixelFormat_RGBA_16161616LE, std::move(stream))
82 , fCodec(std::move(codec))
83 , fData(std::move(data)) {}
84
MakeFromStream(std::unique_ptr<SkStream> stream,Result * result)85 std::unique_ptr<SkCodec> SkJpegxlCodec::MakeFromStream(std::unique_ptr<SkStream> stream,
86 Result* result) {
87 *result = kInternalError;
88 // Either wrap or copy stream data.
89 sk_sp<SkData> data = nullptr;
90 if (stream->getMemoryBase()) {
91 data = SkData::MakeWithoutCopy(stream->getMemoryBase(), stream->getLength());
92 } else {
93 data = SkCopyStreamToData(stream.get());
94 // Data is copied; stream can be released now.
95 stream.reset(nullptr);
96 }
97
98 auto priv = std::make_unique<SkJpegxlCodecPriv>();
99 JxlDecoder* dec = priv->fDecoder.get();
100
101 // Only query metadata this time.
102 auto status = JxlDecoderSubscribeEvents(dec, JXL_DEC_BASIC_INFO | JXL_DEC_COLOR_ENCODING);
103 if (status != JXL_DEC_SUCCESS) {
104 // Fresh instance must accept request for subscription.
105 SkDEBUGFAIL("libjxl returned unexpected status");
106 return nullptr;
107 }
108
109 status = JxlDecoderSetInput(dec, data->bytes(), data->size());
110 if (status != JXL_DEC_SUCCESS) {
111 // Fresh instance must accept first chunk of input.
112 SkDEBUGFAIL("libjxl returned unexpected status");
113 return nullptr;
114 }
115
116 status = JxlDecoderProcessInput(dec);
117 if (status == JXL_DEC_NEED_MORE_INPUT) {
118 *result = kIncompleteInput;
119 return nullptr;
120 }
121 if (status != JXL_DEC_BASIC_INFO) {
122 *result = kInvalidInput;
123 return nullptr;
124 }
125 JxlBasicInfo& info = priv->fInfo;
126 status = JxlDecoderGetBasicInfo(dec, &info);
127 if (status != JXL_DEC_SUCCESS) {
128 // Current event is "JXL_DEC_BASIC_INFO" -> can't fail.
129 SkDEBUGFAIL("libjxl returned unexpected status");
130 return nullptr;
131 }
132
133 // Check that image dimensions are not too large.
134 if (!SkTFitsIn<int32_t>(info.xsize) || !SkTFitsIn<int32_t>(info.ysize)) {
135 *result = kInvalidInput;
136 return nullptr;
137 }
138 int32_t width = SkTo<int32_t>(info.xsize);
139 int32_t height = SkTo<int32_t>(info.ysize);
140
141 bool hasAlpha = (info.alpha_bits != 0);
142 bool isGray = (info.num_color_channels == 1);
143 SkEncodedInfo::Alpha alpha =
144 hasAlpha ? SkEncodedInfo::kUnpremul_Alpha : SkEncodedInfo::kOpaque_Alpha;
145 SkEncodedInfo::Color color;
146 if (hasAlpha) {
147 color = isGray ? SkEncodedInfo::kGrayAlpha_Color : SkEncodedInfo::kRGBA_Color;
148 } else {
149 color = isGray ? SkEncodedInfo::kGray_Color : SkEncodedInfo::kRGB_Color;
150 }
151
152 status = JxlDecoderProcessInput(dec);
153 if (status != JXL_DEC_COLOR_ENCODING) {
154 *result = kInvalidInput;
155 return nullptr;
156 }
157
158 size_t iccSize = 0;
159 // TODO(eustas): format field is currently ignored by decoder.
160 status = JxlDecoderGetICCProfileSize(
161 dec, /* format = */ nullptr, JXL_COLOR_PROFILE_TARGET_DATA, &iccSize);
162 if (status != JXL_DEC_SUCCESS) {
163 // Likely incompatible colorspace.
164 iccSize = 0;
165 }
166 std::unique_ptr<SkEncodedInfo::ICCProfile> profile = nullptr;
167 if (iccSize) {
168 auto icc = SkData::MakeUninitialized(iccSize);
169 // TODO(eustas): format field is currently ignored by decoder.
170 status = JxlDecoderGetColorAsICCProfile(dec,
171 /* format = */ nullptr,
172 JXL_COLOR_PROFILE_TARGET_DATA,
173 reinterpret_cast<uint8_t*>(icc->writable_data()),
174 iccSize);
175 if (status != JXL_DEC_SUCCESS) {
176 // Current event is JXL_DEC_COLOR_ENCODING -> can't fail.
177 SkDEBUGFAIL("libjxl returned unexpected status");
178 return nullptr;
179 }
180 profile = SkEncodedInfo::ICCProfile::Make(std::move(icc));
181 }
182
183 int bitsPerChannel = 16;
184
185 *result = kSuccess;
186 SkEncodedInfo encodedInfo =
187 SkEncodedInfo::Make(width, height, color, alpha, bitsPerChannel, std::move(profile));
188
189 return std::unique_ptr<SkCodec>(new SkJpegxlCodec(
190 std::move(priv), std::move(encodedInfo), std::move(stream), std::move(data)));
191 }
192
onGetPixels(const SkImageInfo & dstInfo,void * dst,size_t rowBytes,const Options & options,int * rowsDecodedPtr)193 SkCodec::Result SkJpegxlCodec::onGetPixels(const SkImageInfo& dstInfo, void* dst, size_t rowBytes,
194 const Options& options, int* rowsDecodedPtr) {
195 // TODO(eustas): implement
196 if (options.fSubset) {
197 return kUnimplemented;
198 }
199 auto& codec = *fCodec.get();
200 const int index = options.fFrameIndex;
201 SkASSERT(0 == index || static_cast<size_t>(index) < codec.fFrames.size());
202 auto* dec = codec.fDecoder.get();
203 JxlDecoderStatus status;
204
205 if ((codec.fLastProcessedFrame >= index) || (codec.fLastProcessedFrame = SkCodec::kNoFrame)) {
206 codec.fLastProcessedFrame = SkCodec::kNoFrame;
207 JxlDecoderRewind(dec);
208 status = JxlDecoderSubscribeEvents(dec, JXL_DEC_FRAME | JXL_DEC_FULL_IMAGE);
209 if (status != JXL_DEC_SUCCESS) {
210 // Fresh decoder instance (after rewind) must accept subscription request.
211 SkDEBUGFAIL("libjxl returned unexpected status");
212 return kInternalError;
213 }
214 status = JxlDecoderSetInput(dec, fData->bytes(), fData->size());
215 if (status != JXL_DEC_SUCCESS) {
216 // Fresh decoder instance (after rewind) must accept first data chunk.
217 SkDEBUGFAIL("libjxl returned unexpected status");
218 return kInternalError;
219 }
220 SkASSERT(codec.fLastProcessedFrame + 1 == 0);
221 }
222
223 int nextFrame = codec.fLastProcessedFrame + 1;
224 if (nextFrame < index) {
225 JxlDecoderSkipFrames(dec, index - nextFrame);
226 }
227
228 // Decode till the frame start.
229 status = JxlDecoderProcessInput(dec);
230 // TODO(eustas): actually, frame is not completely processed; for streaming / partial decoding
231 // we should also add a flag that "last processed frame" is still incomplete, and
232 // flip that flag when frame decoding is over.
233 codec.fLastProcessedFrame = index;
234 if (status != JXL_DEC_FRAME) {
235 // TODO(eustas): check status: it might be either corrupted or incomplete input.
236 return kInternalError;
237 }
238
239 codec.fDst = dst;
240 codec.fRowBytes = rowBytes;
241
242 // TODO(eustas): consider grayscale.
243 uint32_t numColorChannels = 3;
244 // TODO(eustas): consider no-alpha.
245 uint32_t numAlphaChannels = 1;
246 // NB: SKIA works with little-endian F16s.
247 auto endianness = JXL_LITTLE_ENDIAN;
248
249 // Internally JXL does most processing in floats. By "default" we request
250 // output data type to be U8; it takes less memory, but results in some precision loss.
251 // We request F16 in two cases:
252 // - destination type is F16
253 // - color transformation is required; in this case values are remapped,
254 // and with 8-bit precision it is likely that visual artefact will appear
255 // (like banding, etc.)
256 bool halfFloatOutput = false;
257 if (fCodec->fDstColorType == kRGBA_F16_SkColorType) halfFloatOutput = true;
258 if (colorXform()) halfFloatOutput = true;
259 auto dataType = halfFloatOutput ? JXL_TYPE_FLOAT16 : JXL_TYPE_UINT8;
260
261 JxlPixelFormat format =
262 {numColorChannels + numAlphaChannels, dataType, endianness, /* align = */ 0};
263 status = JxlDecoderSetImageOutCallback(dec, &format, SkJpegxlCodec::imageOutCallback, this);
264 if (status != JXL_DEC_SUCCESS) {
265 // Current event is JXL_DEC_FRAME -> decoder must accept callback.
266 SkDEBUGFAIL("libjxl returned unexpected status");
267 return kInternalError;
268 }
269
270 // Decode till the frame start.
271 status = JxlDecoderProcessInput(dec);
272 if (status != JXL_DEC_FULL_IMAGE) {
273 // TODO(eustas): check status: it might be either corrupted or incomplete input.
274 return kInternalError;
275 }
276 // TODO(eustas): currently it is supposed that complete input is accessible;
277 // when streaming support is added JXL_DEC_NEED_MORE_INPUT would also
278 // become a legal outcome; amount of decoded scanlines should be calculated
279 // based on callback invocations / render-pipeline API.
280 *rowsDecodedPtr = dstInfo.height();
281
282 return kSuccess;
283 }
284
onRewind()285 bool SkJpegxlCodec::onRewind() {
286 JxlDecoderRewind(fCodec->fDecoder.get());
287 return true;
288 }
289
conversionSupported(const SkImageInfo & dstInfo,bool srcIsOpaque,bool needsColorXform)290 bool SkJpegxlCodec::conversionSupported(const SkImageInfo& dstInfo, bool srcIsOpaque,
291 bool needsColorXform) {
292 fCodec->fDstColorType = dstInfo.colorType();
293 switch (dstInfo.colorType()) {
294 case kRGBA_8888_SkColorType:
295 return true; // memcpy
296 case kBGRA_8888_SkColorType:
297 return true; // rgba->bgra
298
299 case kRGBA_F16_SkColorType:
300 SkASSERT(needsColorXform); // TODO(eustas): not necessary for JXL.
301 return true; // memcpy
302
303 // TODO(eustas): implement
304 case kRGB_565_SkColorType:
305 return false;
306 case kGray_8_SkColorType:
307 return false;
308 case kAlpha_8_SkColorType:
309 return false;
310
311 default:
312 return false;
313 }
314 return true;
315 }
316
imageOutCallback(void * opaque,size_t x,size_t y,size_t num_pixels,const void * pixels)317 void SkJpegxlCodec::imageOutCallback(void* opaque, size_t x, size_t y,
318 size_t num_pixels, const void* pixels) {
319 SkJpegxlCodec* instance = reinterpret_cast<SkJpegxlCodec*>(opaque);
320 auto& codec = *instance->fCodec.get();
321 size_t offset = y * codec.fRowBytes + (x << codec.fPixelShift);
322 void* dst = SkTAddOffset<void>(codec.fDst, offset);
323 if (instance->colorXform()) {
324 instance->applyColorXform(dst, pixels, num_pixels);
325 return;
326 }
327 switch (codec.fDstColorType) {
328 case kRGBA_8888_SkColorType:
329 memcpy(dst, pixels, 4 * num_pixels);
330 return;
331 case kBGRA_8888_SkColorType:
332 SkOpts::RGBA_to_bgrA((uint32_t*) dst, (const uint32_t*)(pixels), num_pixels);
333 return;
334 case kRGBA_F16_SkColorType:
335 memcpy(dst, pixels, 8 * num_pixels);
336 return;
337 default:
338 SK_ABORT("Selected output format is not supported yet");
339 return;
340 }
341 }
342
scanFrames()343 bool SkJpegxlCodec::scanFrames() {
344 auto decoder = JxlDecoderMake(/* memory_manager = */ nullptr);
345 JxlDecoder* dec = decoder.get();
346 auto* frameHolder = fCodec.get();
347 auto& frames = frameHolder->fFrames;
348 const auto& info = fCodec->fInfo;
349 frames.clear();
350
351 auto alpha = (info.alpha_bits != 0) ? SkEncodedInfo::Alpha::kUnpremul_Alpha
352 : SkEncodedInfo::Alpha::kOpaque_Alpha;
353
354 auto status = JxlDecoderSubscribeEvents(dec, JXL_DEC_FRAME);
355 if (status != JXL_DEC_SUCCESS) {
356 // Fresh instance must accept request for subscription.
357 SkDEBUGFAIL("libjxl returned unexpected status");
358 return true;
359 }
360
361 status = JxlDecoderSetInput(dec, fData->bytes(), fData->size());
362 if (status != JXL_DEC_SUCCESS) {
363 // Fresh instance must accept first input chunk.
364 SkDEBUGFAIL("libjxl returned unexpected status");
365 return true;
366 }
367
368 while (true) {
369 status = JxlDecoderProcessInput(dec);
370 switch (status) {
371 case JXL_DEC_FRAME: {
372 size_t frameId = frames.size();
373 JxlFrameHeader frameHeader;
374 if (JxlDecoderGetFrameHeader(dec, &frameHeader) != JXL_DEC_SUCCESS) {
375 return true;
376 }
377 frames.emplace_back(static_cast<int>(frameId), alpha);
378 auto& frame = frames.back();
379 // TODO(eustas): for better consistency we need to track total duration and report
380 // frame duration as delta to previous frame.
381 int duration = (1000 * frameHeader.duration * info.animation.tps_denominator) /
382 info.animation.tps_numerator;
383 frame.setDuration(duration);
384 frameHolder->setAlphaAndRequiredFrame(&frame);
385 break;
386 }
387 case JXL_DEC_SUCCESS: {
388 return true;
389 }
390 default: {
391 return false;
392 }
393 }
394 }
395 }
396
onGetFrameCount()397 int SkJpegxlCodec::onGetFrameCount() {
398 if (!fCodec->fInfo.have_animation) {
399 return 1;
400 }
401
402 if (!fCodec->fSeenAllFrames) {
403 fCodec->fSeenAllFrames = scanFrames();
404 }
405
406 return fCodec->fFrames.size();
407 }
408
onGetFrameInfo(int index,FrameInfo * frameInfo) const409 bool SkJpegxlCodec::onGetFrameInfo(int index, FrameInfo* frameInfo) const {
410 if (index < 0) {
411 return false;
412 }
413 if (static_cast<size_t>(index) >= fCodec->fFrames.size()) {
414 return false;
415 }
416 fCodec->fFrames[index].fillIn(frameInfo, true);
417 return true;
418 }
419
onGetRepetitionCount()420 int SkJpegxlCodec::onGetRepetitionCount() {
421 JxlBasicInfo& info = fCodec->fInfo;
422 if (!info.have_animation) {
423 return 0;
424 }
425
426 if (info.animation.num_loops == 0) {
427 return kRepetitionCountInfinite;
428 }
429
430 if (SkTFitsIn<int>(info.animation.num_loops)) {
431 return info.animation.num_loops - 1;
432 }
433
434 // Largest "non-infinite" value.
435 return std::numeric_limits<int>::max();
436 }
437
getFrameHolder() const438 const SkFrameHolder* SkJpegxlCodec::getFrameHolder() const {
439 return fCodec.get();
440 }
441
442 // TODO(eustas): implement
443 // SkCodec::Result SkJpegxlCodec::onStartScanlineDecode(
444 // const SkImageInfo& /*dstInfo*/, const Options& /*options*/) { return kUnimplemented; }
445
446 // TODO(eustas): implement
447 // SkCodec::Result SkJpegxlCodec::onStartIncrementalDecode(
448 // const SkImageInfo& /*dstInfo*/, void*, size_t, const Options&) { return kUnimplemented; }
449
450 // TODO(eustas): implement
451 // SkCodec::Result SkJpegxlCodec::onIncrementalDecode(int*) { return kUnimplemented; }
452
453 // TODO(eustas): implement
454 // bool SkJpegxlCodec::onSkipScanlines(int /*countLines*/) { return false; }
455
456 // TODO(eustas): implement
457 // int SkJpegxlCodec::onGetScanlines(
458 // void* /*dst*/, int /*countLines*/, size_t /*rowBytes*/) { return 0; }
459
460 // TODO(eustas): implement
461 // SkSampler* SkJpegxlCodec::getSampler(bool /*createIfNecessary*/) { return nullptr; }
462