/* * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.android.libraries.mobiledatadownload.internal.util; import android.text.TextUtils; import android.util.Log; import androidx.annotation.Nullable; import com.google.protobuf.CodedOutputStream; import com.google.protobuf.ExtensionRegistryLite; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.MessageLite; import com.google.protobuf.Parser; import java.io.IOException; import java.nio.BufferOverflowException; import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.zip.CRC32; /** Utils for moving Protobuf messages in and out of ByteBuffers. */ // LINT.IfChange public class ProtoLiteUtil { public static final String TAG = "ProtoLiteUtil"; /* * File format (with tail crc): * ( * int: number of bytes in message; * byte[number of bytes in message]: bytes of message; * )...: message blocks; * long: CRC of the above data * * File format (without tail crc): * ( * int: number of bytes in message; * byte[number of bytes in message]: bytes of message; * long: the CRC of the bytes of the message above * )...: message blocks; */ private static final byte INT_BYTE_SIZE = Integer.SIZE / Byte.SIZE; private static final byte LONG_BYTE_SIZE = Long.SIZE / Byte.SIZE; private static final byte CRC_LEN = LONG_BYTE_SIZE; // Used to help guess a good initial capacity for the ArrayList private static final int EXPECTED_MESS_SIZE = 1000; /** * @param buf MUST be a 0 based array buffer (aka, buf.arrayOffset() must return 0) and be mutable * (aka, buf.isReadOnly() must return false) * @param messageType The type of the proto * @param tailCrc True if there is a single CRC at the end of the file for the whole content, * false if there is a CRC after every record. * @return A list of proto messages read from the buffer, or null on failure. */ @Nullable public static List readFromBuffer( ByteBuffer buf, Class messageType, Parser messageParser, boolean tailCrc) { // assert buf.arrayOffset() == 0; // annoyingly, ByteBuffer#array() throws an exception if the ByteBuffer was readonly // assert !buf.isReadOnly(); String typename = messageType.toString(); if (tailCrc) { // Validate the tail CRC before reading any messages. int crcPos = buf.limit() - CRC_LEN; if (crcPos < 0) { // Equivalently, buf.limit() < CRC_LEN Log.e(TAG, "Protobuf data too short to be valid"); return null; } // First off, check the crc long crc = buf.getLong(crcPos); // Position should still be at the beginning; // the read and write operations that take an index explicitly do not touch the // position if (!validateCRC(buf.array(), buf.arrayOffset(), crcPos, crc)) { Log.e(TAG, "Ignoring corrupt protobuf data"); return null; } if (crcPos == 0) { // If the only thing in there was the CRC, then there are no messages return new ArrayList(0); } } int end = tailCrc ? buf.limit() - CRC_LEN : buf.limit(); List toReturn = new ArrayList((buf.limit() / EXPECTED_MESS_SIZE) + 1); while (buf.position() < end) { T dest; int bytesInMessage; try { bytesInMessage = buf.getInt(); } catch (BufferUnderflowException ex) { handleBufferUnderflow(ex, typename); return null; } if (bytesInMessage < 0) { // This actually can happen even if the CRC check passed, // if the user gave the wrong MessageLite type. // Same goes for all of the other exceptions that can be thrown. Log.e( TAG, String.format( "Invalid message size: %d. May have given the wrong message type: %s", bytesInMessage, typename)); return null; } if (!tailCrc) { // May have read a garbage size. Read carefully. if (end < buf.position() + bytesInMessage + CRC_LEN) { Log.e( TAG, String.format("Invalid message size: %d (buffer end is %d)", bytesInMessage, end)); return toReturn; } long crc = buf.getLong(buf.position() + bytesInMessage); if (!validateCRC(buf.array(), buf.arrayOffset() + buf.position(), bytesInMessage, crc)) { // Return the valid messages we have read so far. return toReturn; } } // According to ByteBuffer#array()'s spec, this should not copy the backing array dest = tryCreate( buf.array(), buf.arrayOffset() + buf.position(), bytesInMessage, messageType, messageParser); if (dest == null) { // Something is seriously hosed at this point, return nothing. return null; } toReturn.add(dest); // Advance the buffer manually, since we read from it "raw" from the array above buf.position(buf.position() + bytesInMessage + (tailCrc ? 0 : CRC_LEN)); } return toReturn; } @Nullable private static T tryCreate( byte[] arr, int pos, int len, Class type, Parser parser) { try { // Cannot use generated registry here, because it may cause NPE to clients. // For more detail, see b/140135059. return parser.parseFrom(arr, pos, len, ExtensionRegistryLite.getEmptyRegistry()); } catch (InvalidProtocolBufferException ex) { Log.e(TAG, "Cannot deserialize message of type " + type, ex); return null; } } /** * Serializes the given MessageLite messages into a ByteBuffer, with either a CRC of the whole * content at the end of the buffer (tail CRC) or a CRC of every message at the end of the * message. * * @param coll The messages to write. * @param tailCrc true to use a tail CRC, false to put a CRC after every message. * @return A ByteBuffer containing the serialized messages. */ @Nullable public static ByteBuffer dumpIntoBuffer( Iterable coll, boolean tailCrc) { int count = 0; long toWriteOut = tailCrc ? CRC_LEN : 0; final int extraBytesPerMessage = tailCrc ? INT_BYTE_SIZE : INT_BYTE_SIZE + CRC_LEN; // First, get the size of how much will be written out // TODO find out if there is a adder util thingy I can use (could be parallel) for (MessageLite mess : coll) { toWriteOut += extraBytesPerMessage + mess.getSerializedSize(); ++count; } if (count == 0) { // If there are no counters to write, don't even bother with the checksum. return ByteBuffer.allocate(0); } // Now we got this, make a ByteBuffer to hold all that we need to ByteBuffer buff = null; try { buff = ByteBuffer.allocate((int) toWriteOut); } catch (IllegalArgumentException ex) { Log.e(TAG, String.format("Too big to serialize, %s", prettyPrintBytes(toWriteOut)), ex); return null; } // According to ByteBuffer#array()'s spec, this should not copy the backing array byte[] arr = buff.array(); // Also conveniently is where we need to write next int writtenSoFar = 0; // Now add in the serialized forms for (MessageLite mess : coll) { // As we called getSerializedSize above, this is assured to give us a non-bogus answer int bytesInMessage = mess.getSerializedSize(); try { buff.putInt(bytesInMessage); } catch (BufferOverflowException ex) { handleBufferOverflow(ex); return null; } writtenSoFar += INT_BYTE_SIZE; // We are writing past the end of where buff is currently "looking at", // So reusing the backing array here should be fine. try { mess.writeTo(CodedOutputStream.newInstance(arr, writtenSoFar, bytesInMessage)); } catch (IOException e) { Log.e(TAG, "Exception while writing to buffer.", e); } // Same as above, but reading past the end this time. try { buff.put(arr, writtenSoFar, bytesInMessage); } catch (BufferOverflowException ex) { handleBufferOverflow(ex); return null; } writtenSoFar += bytesInMessage; if (!tailCrc) { appendCRC(buff, arr, writtenSoFar - bytesInMessage, bytesInMessage); writtenSoFar += CRC_LEN; } } if (tailCrc) { try { appendCRC(buff, arr, 0, writtenSoFar); } catch (BufferOverflowException ex) { handleBufferOverflow(ex); return null; } } buff.rewind(); return buff; } /** Return string from proto bytes when we know bytes are UTF-8. */ public static String getDataString(byte[] data) { return new String(data, Charset.forName("UTF-8")); } /** Return null if input is empty (or null). */ @Nullable public static String nullIfEmpty(String input) { return TextUtils.isEmpty(input) ? null : input; } /** Return null if input array is empty (or null). */ @Nullable public static T[] nullIfEmpty(T[] input) { return input == null || input.length == 0 ? null : input; } /** Similar to Objects.equal but available pre-kitkat. */ public static boolean safeEqual(Object a, Object b) { return a == null ? b == null : a.equals(b); } /** Wraps MessageLite.toByteArray to check for null and return null if that's the case. */ @Nullable public static final byte[] safeToByteArray(MessageLite msg) { return msg == null ? null : msg.toByteArray(); } private static void handleBufferUnderflow(BufferUnderflowException ex, String typename) { Log.e( TAG, String.format("Buffer underflow. May have given the wrong message type: %s", typename), ex); } private static void handleBufferOverflow(BufferOverflowException ex) { Log.e( TAG, "Buffer underflow. A message may have an invalid serialized form" + " or has been concurrently modified.", ex); } /** * Reads the bytes given in an array and appends the CRC32 checksum to the ByteBuffer. The * location the CRC32 checksum will be written is the {@link ByteBuffer#position() current * position} in the ByteBuffer. The given ByteBuffer must have must have enough room (starting at * its position) to fit an additonal {@link #CRC_LEN} bytes. * * @param dest where to write the CRC32 checksum; must have enough room to fit an additonal {@link * #CRC_LEN} bytes * @param src the array of bytes containing the data to checksum * @param off offset of where to start reading the array * @param len number of bytes to read in the array */ private static void appendCRC(ByteBuffer dest, byte[] src, int off, int len) { CRC32 crc = new CRC32(); crc.update(src, off, len); dest.putLong(crc.getValue()); } private static boolean validateCRC(byte[] arr, int off, int len, long expectedCRC) { CRC32 crc = new CRC32(); crc.update(arr, off, len); long computedCRC = crc.getValue(); boolean matched = computedCRC == expectedCRC; if (!matched) { Log.e( TAG, String.format( "Corrupt protobuf data, expected CRC: %d computed CRC: %d", expectedCRC, computedCRC)); } return matched; } private ProtoLiteUtil() { // No instantiation. } private static String prettyPrintBytes(long bytes) { if (bytes > 1024L * 1024 * 1024) { return String.format(Locale.US, "%.2fGB", (double) bytes / (1024L * 1024 * 1024)); } else if (bytes > 1024 * 1024) { return String.format(Locale.US, "%.2fMB", (double) bytes / (1024 * 1024)); } else if (bytes > 1024) { return String.format(Locale.US, "%.2fKB", (double) bytes / 1024); } return String.format(Locale.US, "%d Bytes", bytes); } } // LINT.ThenChange()