1 /*
2 * Copyright 2021 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8 #include "src/codec/SkJpegxlCodec.h"
9
10 #include <limits>
11 #include <vector>
12
13 #include "include/codec/SkEncodedOrigin.h"
14 #include "include/core/SkData.h"
15 #include "include/core/SkEncodedImageFormat.h"
16 #include "include/core/SkImageInfo.h"
17 #include "include/core/SkRefCnt.h"
18 #include "include/core/SkStream.h"
19 #include "include/private/SkEncodedInfo.h"
20 #include "include/private/SkTFitsIn.h"
21 #include "include/private/SkTo.h"
22 #include "jxl/decode.h"
23 #include "jxl/decode_cxx.h"
24 #include "src/codec/SkFrameHolder.h"
25 #include "src/codec/SkSampler.h"
26 #include "src/core/SkOpts.h"
27 #include "src/core/SkStreamPriv.h"
28
29 namespace {
30
31 class Frame : public SkFrame {
32 public:
Frame(int i,SkEncodedInfo::Alpha alpha)33 explicit Frame(int i, SkEncodedInfo::Alpha alpha) : INHERITED(i), fReportedAlpha(alpha) {}
onReportedAlpha() const34 SkEncodedInfo::Alpha onReportedAlpha() const override { return fReportedAlpha; }
35
36 private:
37 const SkEncodedInfo::Alpha fReportedAlpha;
38
39 using INHERITED = SkFrame;
40 };
41
42 } // namespace
43
IsJpegxl(const void * buffer,size_t bytesRead)44 bool SkJpegxlCodec::IsJpegxl(const void* buffer, size_t bytesRead) {
45 JxlSignature result = JxlSignatureCheck(reinterpret_cast<const uint8_t*>(buffer), bytesRead);
46 return (result == JXL_SIG_CODESTREAM) || (result == JXL_SIG_CONTAINER);
47 }
48
49 class SkJpegxlCodecPriv : public SkFrameHolder {
50 public:
SkJpegxlCodecPriv()51 SkJpegxlCodecPriv() : fDecoder(JxlDecoderMake(/* memory_manager= */ nullptr)) {}
52 JxlDecoderPtr fDecoder; // unique_ptr with custom destructor
53 JxlBasicInfo fInfo;
54 bool fSeenAllFrames = false;
55 std::vector<Frame> fFrames;
56 int fLastProcessedFrame = SkCodec::kNoFrame;
57 void* fDst;
58 size_t fPixelShift;
59 size_t fRowBytes;
60 SkColorType fDstColorType;
61
62 protected:
onGetFrame(int i) const63 const SkFrame* onGetFrame(int i) const override {
64 SkASSERT(i >= 0 && static_cast<size_t>(i) < fFrames.size());
65 return static_cast<const SkFrame*>(&fFrames[i]);
66 }
67 };
68
SkJpegxlCodec(std::unique_ptr<SkJpegxlCodecPriv> codec,SkEncodedInfo && info,std::unique_ptr<SkStream> stream,sk_sp<SkData> data)69 SkJpegxlCodec::SkJpegxlCodec(std::unique_ptr<SkJpegxlCodecPriv> codec,
70 SkEncodedInfo&& info,
71 std::unique_ptr<SkStream> stream,
72 sk_sp<SkData> data)
73 : INHERITED(std::move(info), skcms_PixelFormat_RGBA_16161616LE, std::move(stream))
74 , fCodec(std::move(codec))
75 , fData(std::move(data)) {}
76
MakeFromStream(std::unique_ptr<SkStream> stream,Result * result)77 std::unique_ptr<SkCodec> SkJpegxlCodec::MakeFromStream(std::unique_ptr<SkStream> stream,
78 Result* result) {
79 *result = kInternalError;
80 // Either wrap or copy stream data.
81 sk_sp<SkData> data = nullptr;
82 if (stream->getMemoryBase()) {
83 data = SkData::MakeWithoutCopy(stream->getMemoryBase(), stream->getLength());
84 } else {
85 data = SkCopyStreamToData(stream.get());
86 // Data is copied; stream can be released now.
87 stream.reset(nullptr);
88 }
89
90 auto priv = std::make_unique<SkJpegxlCodecPriv>();
91 JxlDecoder* dec = priv->fDecoder.get();
92
93 // Only query metadata this time.
94 auto status = JxlDecoderSubscribeEvents(dec, JXL_DEC_BASIC_INFO | JXL_DEC_COLOR_ENCODING);
95 if (status != JXL_DEC_SUCCESS) {
96 // Fresh instance must accept request for subscription.
97 SkDEBUGFAIL("libjxl returned unexpected status");
98 return nullptr;
99 }
100
101 status = JxlDecoderSetInput(dec, data->bytes(), data->size());
102 if (status != JXL_DEC_SUCCESS) {
103 // Fresh instance must accept first chunk of input.
104 SkDEBUGFAIL("libjxl returned unexpected status");
105 return nullptr;
106 }
107
108 status = JxlDecoderProcessInput(dec);
109 if (status == JXL_DEC_NEED_MORE_INPUT) {
110 *result = kIncompleteInput;
111 return nullptr;
112 }
113 if (status != JXL_DEC_BASIC_INFO) {
114 *result = kInvalidInput;
115 return nullptr;
116 }
117 JxlBasicInfo& info = priv->fInfo;
118 status = JxlDecoderGetBasicInfo(dec, &info);
119 if (status != JXL_DEC_SUCCESS) {
120 // Current event is "JXL_DEC_BASIC_INFO" -> can't fail.
121 SkDEBUGFAIL("libjxl returned unexpected status");
122 return nullptr;
123 }
124
125 // Check that image dimensions are not too large.
126 if (!SkTFitsIn<int32_t>(info.xsize) || !SkTFitsIn<int32_t>(info.ysize)) {
127 *result = kInvalidInput;
128 return nullptr;
129 }
130 int32_t width = SkTo<int32_t>(info.xsize);
131 int32_t height = SkTo<int32_t>(info.ysize);
132
133 bool hasAlpha = (info.alpha_bits != 0);
134 bool isGray = (info.num_color_channels == 1);
135 SkEncodedInfo::Alpha alpha =
136 hasAlpha ? SkEncodedInfo::kUnpremul_Alpha : SkEncodedInfo::kOpaque_Alpha;
137 SkEncodedInfo::Color color;
138 if (hasAlpha) {
139 color = isGray ? SkEncodedInfo::kGrayAlpha_Color : SkEncodedInfo::kRGBA_Color;
140 } else {
141 color = isGray ? SkEncodedInfo::kGray_Color : SkEncodedInfo::kRGB_Color;
142 }
143
144 status = JxlDecoderProcessInput(dec);
145 if (status != JXL_DEC_COLOR_ENCODING) {
146 *result = kInvalidInput;
147 return nullptr;
148 }
149
150 size_t iccSize = 0;
151 // TODO(eustas): format field is currently ignored by decoder.
152 status = JxlDecoderGetICCProfileSize(
153 dec, /* format = */ nullptr, JXL_COLOR_PROFILE_TARGET_DATA, &iccSize);
154 if (status != JXL_DEC_SUCCESS) {
155 // Likely incompatible colorspace.
156 iccSize = 0;
157 }
158 std::unique_ptr<SkEncodedInfo::ICCProfile> profile = nullptr;
159 if (iccSize) {
160 auto icc = SkData::MakeUninitialized(iccSize);
161 // TODO(eustas): format field is currently ignored by decoder.
162 status = JxlDecoderGetColorAsICCProfile(dec,
163 /* format = */ nullptr,
164 JXL_COLOR_PROFILE_TARGET_DATA,
165 reinterpret_cast<uint8_t*>(icc->writable_data()),
166 iccSize);
167 if (status != JXL_DEC_SUCCESS) {
168 // Current event is JXL_DEC_COLOR_ENCODING -> can't fail.
169 SkDEBUGFAIL("libjxl returned unexpected status");
170 return nullptr;
171 }
172 profile = SkEncodedInfo::ICCProfile::Make(std::move(icc));
173 }
174
175 int bitsPerChannel = 16;
176
177 *result = kSuccess;
178 SkEncodedInfo encodedInfo =
179 SkEncodedInfo::Make(width, height, color, alpha, bitsPerChannel, std::move(profile));
180
181 return std::unique_ptr<SkCodec>(new SkJpegxlCodec(
182 std::move(priv), std::move(encodedInfo), std::move(stream), std::move(data)));
183 }
184
onGetPixels(const SkImageInfo & dstInfo,void * dst,size_t rowBytes,const Options & options,int * rowsDecodedPtr)185 SkCodec::Result SkJpegxlCodec::onGetPixels(const SkImageInfo& dstInfo, void* dst, size_t rowBytes,
186 const Options& options, int* rowsDecodedPtr) {
187 // TODO(eustas): implement
188 if (options.fSubset) {
189 return kUnimplemented;
190 }
191 auto& codec = *fCodec.get();
192 const int index = options.fFrameIndex;
193 SkASSERT(0 == index || static_cast<size_t>(index) < codec.fFrames.size());
194 auto* dec = codec.fDecoder.get();
195 JxlDecoderStatus status;
196
197 if ((codec.fLastProcessedFrame >= index) || (codec.fLastProcessedFrame = SkCodec::kNoFrame)) {
198 codec.fLastProcessedFrame = SkCodec::kNoFrame;
199 JxlDecoderRewind(dec);
200 status = JxlDecoderSubscribeEvents(dec, JXL_DEC_FRAME | JXL_DEC_FULL_IMAGE);
201 if (status != JXL_DEC_SUCCESS) {
202 // Fresh decoder instance (after rewind) must accept subscription request.
203 SkDEBUGFAIL("libjxl returned unexpected status");
204 return kInternalError;
205 }
206 status = JxlDecoderSetInput(dec, fData->bytes(), fData->size());
207 if (status != JXL_DEC_SUCCESS) {
208 // Fresh decoder instance (after rewind) must accept first data chunk.
209 SkDEBUGFAIL("libjxl returned unexpected status");
210 return kInternalError;
211 }
212 SkASSERT(codec.fLastProcessedFrame + 1 == 0);
213 }
214
215 int nextFrame = codec.fLastProcessedFrame + 1;
216 if (nextFrame < index) {
217 JxlDecoderSkipFrames(dec, index - nextFrame);
218 }
219
220 // Decode till the frame start.
221 status = JxlDecoderProcessInput(dec);
222 // TODO(eustas): actually, frame is not completely processed; for streaming / partial decoding
223 // we should also add a flag that "last processed frame" is still incomplete, and
224 // flip that flag when frame decoding is over.
225 codec.fLastProcessedFrame = index;
226 if (status != JXL_DEC_FRAME) {
227 // TODO(eustas): check status: it might be either corrupted or incomplete input.
228 return kInternalError;
229 }
230
231 codec.fDst = dst;
232 codec.fRowBytes = rowBytes;
233
234 // TODO(eustas): consider grayscale.
235 uint32_t numColorChannels = 3;
236 // TODO(eustas): consider no-alpha.
237 uint32_t numAlphaChannels = 1;
238 // NB: SKIA works with little-endian F16s.
239 auto endianness = JXL_LITTLE_ENDIAN;
240
241 // Internally JXL does most processing in floats. By "default" we request
242 // output data type to be U8; it takes less memory, but results in some precision loss.
243 // We request F16 in two cases:
244 // - destination type is F16
245 // - color transformation is required; in this case values are remapped,
246 // and with 8-bit precision it is likely that visual artefact will appear
247 // (like banding, etc.)
248 bool halfFloatOutput = false;
249 if (fCodec->fDstColorType == kRGBA_F16_SkColorType) halfFloatOutput = true;
250 if (colorXform()) halfFloatOutput = true;
251 auto dataType = halfFloatOutput ? JXL_TYPE_FLOAT16 : JXL_TYPE_UINT8;
252
253 JxlPixelFormat format =
254 {numColorChannels + numAlphaChannels, dataType, endianness, /* align = */ 0};
255 status = JxlDecoderSetImageOutCallback(dec, &format, SkJpegxlCodec::imageOutCallback, this);
256 if (status != JXL_DEC_SUCCESS) {
257 // Current event is JXL_DEC_FRAME -> decoder must accept callback.
258 SkDEBUGFAIL("libjxl returned unexpected status");
259 return kInternalError;
260 }
261
262 // Decode till the frame start.
263 status = JxlDecoderProcessInput(dec);
264 if (status != JXL_DEC_FULL_IMAGE) {
265 // TODO(eustas): check status: it might be either corrupted or incomplete input.
266 return kInternalError;
267 }
268 // TODO(eustas): currently it is supposed that complete input is accessible;
269 // when streaming support is added JXL_DEC_NEED_MORE_INPUT would also
270 // become a legal outcome; amount of decoded scanlines should be calculated
271 // based on callback invocations / render-pipeline API.
272 *rowsDecodedPtr = dstInfo.height();
273
274 return kSuccess;
275 }
276
onRewind()277 bool SkJpegxlCodec::onRewind() {
278 JxlDecoderRewind(fCodec->fDecoder.get());
279 return true;
280 }
281
conversionSupported(const SkImageInfo & dstInfo,bool srcIsOpaque,bool needsColorXform)282 bool SkJpegxlCodec::conversionSupported(const SkImageInfo& dstInfo, bool srcIsOpaque,
283 bool needsColorXform) {
284 fCodec->fDstColorType = dstInfo.colorType();
285 switch (dstInfo.colorType()) {
286 case kRGBA_8888_SkColorType:
287 return true; // memcpy
288 case kBGRA_8888_SkColorType:
289 return true; // rgba->bgra
290
291 case kRGBA_F16_SkColorType:
292 SkASSERT(needsColorXform); // TODO(eustas): not necessary for JXL.
293 return true; // memcpy
294
295 // TODO(eustas): implement
296 case kRGB_565_SkColorType:
297 return false;
298 case kGray_8_SkColorType:
299 return false;
300 case kAlpha_8_SkColorType:
301 return false;
302
303 default:
304 return false;
305 }
306 return true;
307 }
308
imageOutCallback(void * opaque,size_t x,size_t y,size_t num_pixels,const void * pixels)309 void SkJpegxlCodec::imageOutCallback(void* opaque, size_t x, size_t y,
310 size_t num_pixels, const void* pixels) {
311 SkJpegxlCodec* instance = reinterpret_cast<SkJpegxlCodec*>(opaque);
312 auto& codec = *instance->fCodec.get();
313 size_t offset = y * codec.fRowBytes + (x << codec.fPixelShift);
314 void* dst = SkTAddOffset<void>(codec.fDst, offset);
315 if (instance->colorXform()) {
316 instance->applyColorXform(dst, pixels, num_pixels);
317 return;
318 }
319 switch (codec.fDstColorType) {
320 case kRGBA_8888_SkColorType:
321 memcpy(dst, pixels, 4 * num_pixels);
322 return;
323 case kBGRA_8888_SkColorType:
324 SkOpts::RGBA_to_bgrA((uint32_t*) dst, (const uint32_t*)(pixels), num_pixels);
325 return;
326 case kRGBA_F16_SkColorType:
327 memcpy(dst, pixels, 8 * num_pixels);
328 return;
329 default:
330 SK_ABORT("Selected output format is not supported yet");
331 return;
332 }
333 }
334
scanFrames()335 bool SkJpegxlCodec::scanFrames() {
336 auto decoder = JxlDecoderMake(/* memory_manager = */ nullptr);
337 JxlDecoder* dec = decoder.get();
338 auto* frameHolder = fCodec.get();
339 auto& frames = frameHolder->fFrames;
340 const auto& info = fCodec->fInfo;
341 frames.clear();
342
343 auto alpha = (info.alpha_bits != 0) ? SkEncodedInfo::Alpha::kUnpremul_Alpha
344 : SkEncodedInfo::Alpha::kOpaque_Alpha;
345
346 auto status = JxlDecoderSubscribeEvents(dec, JXL_DEC_FRAME);
347 if (status != JXL_DEC_SUCCESS) {
348 // Fresh instance must accept request for subscription.
349 SkDEBUGFAIL("libjxl returned unexpected status");
350 return true;
351 }
352
353 status = JxlDecoderSetInput(dec, fData->bytes(), fData->size());
354 if (status != JXL_DEC_SUCCESS) {
355 // Fresh instance must accept first input chunk.
356 SkDEBUGFAIL("libjxl returned unexpected status");
357 return true;
358 }
359
360 while (true) {
361 status = JxlDecoderProcessInput(dec);
362 switch (status) {
363 case JXL_DEC_FRAME: {
364 size_t frameId = frames.size();
365 JxlFrameHeader frameHeader;
366 if (JxlDecoderGetFrameHeader(dec, &frameHeader) != JXL_DEC_SUCCESS) {
367 return true;
368 }
369 frames.emplace_back(static_cast<int>(frameId), alpha);
370 auto& frame = frames.back();
371 // TODO(eustas): for better consistency we need to track total duration and report
372 // frame duration as delta to previous frame.
373 int duration = (1000 * frameHeader.duration * info.animation.tps_denominator) /
374 info.animation.tps_numerator;
375 frame.setDuration(duration);
376 frameHolder->setAlphaAndRequiredFrame(&frame);
377 break;
378 }
379 case JXL_DEC_SUCCESS: {
380 return true;
381 }
382 default: {
383 return false;
384 }
385 }
386 }
387 }
388
onGetFrameCount()389 int SkJpegxlCodec::onGetFrameCount() {
390 if (!fCodec->fInfo.have_animation) {
391 return 1;
392 }
393
394 if (!fCodec->fSeenAllFrames) {
395 fCodec->fSeenAllFrames = scanFrames();
396 }
397
398 return fCodec->fFrames.size();
399 }
400
onGetFrameInfo(int index,FrameInfo * frameInfo) const401 bool SkJpegxlCodec::onGetFrameInfo(int index, FrameInfo* frameInfo) const {
402 if (index < 0) {
403 return false;
404 }
405 if (static_cast<size_t>(index) >= fCodec->fFrames.size()) {
406 return false;
407 }
408 fCodec->fFrames[index].fillIn(frameInfo, true);
409 return true;
410 }
411
onGetRepetitionCount()412 int SkJpegxlCodec::onGetRepetitionCount() {
413 JxlBasicInfo& info = fCodec->fInfo;
414 if (!info.have_animation) {
415 return 0;
416 }
417
418 if (info.animation.num_loops == 0) {
419 return kRepetitionCountInfinite;
420 }
421
422 if (SkTFitsIn<int>(info.animation.num_loops)) {
423 return info.animation.num_loops - 1;
424 }
425
426 // Largest "non-infinite" value.
427 return std::numeric_limits<int>::max();
428 }
429
getFrameHolder() const430 const SkFrameHolder* SkJpegxlCodec::getFrameHolder() const {
431 return fCodec.get();
432 }
433
434 // TODO(eustas): implement
435 // SkCodec::Result SkJpegxlCodec::onStartScanlineDecode(
436 // const SkImageInfo& /*dstInfo*/, const Options& /*options*/) { return kUnimplemented; }
437
438 // TODO(eustas): implement
439 // SkCodec::Result SkJpegxlCodec::onStartIncrementalDecode(
440 // const SkImageInfo& /*dstInfo*/, void*, size_t, const Options&) { return kUnimplemented; }
441
442 // TODO(eustas): implement
443 // SkCodec::Result SkJpegxlCodec::onIncrementalDecode(int*) { return kUnimplemented; }
444
445 // TODO(eustas): implement
446 // bool SkJpegxlCodec::onSkipScanlines(int /*countLines*/) { return false; }
447
448 // TODO(eustas): implement
449 // int SkJpegxlCodec::onGetScanlines(
450 // void* /*dst*/, int /*countLines*/, size_t /*rowBytes*/) { return 0; }
451
452 // TODO(eustas): implement
453 // SkSampler* SkJpegxlCodec::getSampler(bool /*createIfNecessary*/) { return nullptr; }
454