xref: /tools/metalava/metalava-model-text/src/main/java/com/android/tools/metalava/model/text/ApiFile.kt

/*
 * Copyright (C) 2020 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.android.tools.metalava.model.text

import com.android.tools.metalava.model.ANDROIDX_NONNULL
import com.android.tools.metalava.model.ANDROIDX_NULLABLE
import com.android.tools.metalava.model.AnnotationItem
import com.android.tools.metalava.model.AnnotationItem.Companion.unshortenAnnotation
import com.android.tools.metalava.model.ArrayTypeItem
import com.android.tools.metalava.model.CallableItem
import com.android.tools.metalava.model.ClassItem
import com.android.tools.metalava.model.ClassKind
import com.android.tools.metalava.model.ClassOrigin
import com.android.tools.metalava.model.ClassResolver
import com.android.tools.metalava.model.ClassTypeItem
import com.android.tools.metalava.model.Codebase
import com.android.tools.metalava.model.ConstructorItem
import com.android.tools.metalava.model.DefaultAnnotationItem
import com.android.tools.metalava.model.DefaultTypeParameterList
import com.android.tools.metalava.model.ExceptionTypeItem
import com.android.tools.metalava.model.FixedFieldValue
import com.android.tools.metalava.model.Item
import com.android.tools.metalava.model.ItemDocumentation
import com.android.tools.metalava.model.JAVA_LANG_DEPRECATED
import com.android.tools.metalava.model.JAVA_LANG_OBJECT
import com.android.tools.metalava.model.MetalavaApi
import com.android.tools.metalava.model.MethodItem
import com.android.tools.metalava.model.MutableModifierList
import com.android.tools.metalava.model.ParameterItem
import com.android.tools.metalava.model.PrimitiveTypeItem
import com.android.tools.metalava.model.PrimitiveTypeItem.Primitive
import com.android.tools.metalava.model.SelectableItem
import com.android.tools.metalava.model.TypeItem
import com.android.tools.metalava.model.TypeNullability
import com.android.tools.metalava.model.TypeParameterItem
import com.android.tools.metalava.model.TypeParameterList
import com.android.tools.metalava.model.TypeParameterListAndFactory
import com.android.tools.metalava.model.VisibilityLevel
import com.android.tools.metalava.model.api.surface.ApiSurfaces
import com.android.tools.metalava.model.api.surface.ApiVariant
import com.android.tools.metalava.model.api.surface.ApiVariantType
import com.android.tools.metalava.model.createImmutableModifiers
import com.android.tools.metalava.model.createMutableModifiers
import com.android.tools.metalava.model.item.DefaultClassItem
import com.android.tools.metalava.model.item.DefaultCodebase
import com.android.tools.metalava.model.item.DefaultPackageItem
import com.android.tools.metalava.model.item.DefaultTypeParameterItem
import com.android.tools.metalava.model.item.MutablePackageDoc
import com.android.tools.metalava.model.item.PackageDocs
import com.android.tools.metalava.model.item.ParameterDefaultValue
import com.android.tools.metalava.model.javaUnescapeString
import com.android.tools.metalava.model.type.MethodFingerprint
import com.android.tools.metalava.reporter.FileLocation
import com.android.tools.metalava.reporter.Issues
import java.io.File
import java.io.IOException
import java.io.InputStream
import java.io.StringReader
import java.nio.file.Path
import java.util.IdentityHashMap
import kotlin.text.Charsets.UTF_8

/** Encapsulates information needed to process a signature file. */
sealed class SignatureFile {
    /** The underlying signature [File]. */
    abstract val file: File

    /**
     * Indicates whether [file] is for the main API surface, i.e. the one that is being created.
     *
     * This will be stored in [Item.emit].
     */
    protected open val forMainApiSurface: Boolean
        get() = true

    /** The [ApiVariantType] of the signature files. */
    protected open val apiVariantType: ApiVariantType
        get() = ApiVariantType.CORE

    /**
     * Get the [ApiVariant] that this signature file represents.
     *
     * If [forMainApiSurface] is `false` then [apiSurfaces] must provide a non-null value for
     * [ApiSurfaces.base]. An exception will be thrown if it is not.
     *
     * @param apiSurfaces the [ApiSurfaces] the returned [Codebase] is required to support.
     */
    fun apiVariantFor(apiSurfaces: ApiSurfaces): ApiVariant {
        val apiSurface =
            if (forMainApiSurface) apiSurfaces.main
            else
                apiSurfaces.base
                    ?: error("$file expects a base API surface to be available but it is not")
        return apiSurface.variantFor(apiVariantType)
    }

    /** Read the contents of this signature file. */
    abstract fun readContents(): String

    companion object {
        /** Create a list of [SignatureFile]s from a varargs array of [File]s. */
        fun fromFiles(vararg files: File): List<SignatureFile> =
            files.map {
                SignatureFileFromFile(
                    it,
                )
            }

        /**
         * Create a list of [SignatureFile]s from a list of [File]s.
         *
         * @param files the list of [File]s.
         * @param apiVariantTypeChooser A lambda that will be called with the [File] of each item in
         *   [files] and whose return value will be stored in [SignatureFile.apiVariantType].
         * @param forMainApiSurfacePredicate A predicate that will be called with the index and
         *   [File] of each item in [files] and whose return value will be stored in
         *   [SignatureFile.forMainApiSurface].
         */
        fun fromFiles(
            files: List<File>,
            apiVariantTypeChooser: (File) -> ApiVariantType = { ApiVariantType.CORE },
            forMainApiSurfacePredicate: (Int, File) -> Boolean = { _, _ -> true },
        ): List<SignatureFile> =
            files.mapIndexed { index, file ->
                SignatureFileFromFile(
                    file,
                    forMainApiSurface = forMainApiSurfacePredicate(index, file),
                    apiVariantType = apiVariantTypeChooser(file),
                )
            }

        /** Create a [SignatureFile] that wraps an [InputStream]. */
        fun fromStream(filename: String, inputStream: InputStream): SignatureFile {
            return SignatureFileFromStream(File(filename), inputStream)
        }

        /**
         * Create a [SignatureFile] that wraps a [String].
         *
         * @param filename the name of the file, used for error reporting.
         * @param contents the contents of the file, will be trimmed using [String.trimIndent].
         */
        fun fromText(filename: String, contents: String): SignatureFile {
            return SignatureFileFromText(File(filename), contents.trimIndent())
        }
    }

    /** A [SignatureFile] that will read the text from the [file]. */
    private data class SignatureFileFromFile(
        override val file: File,
        override val forMainApiSurface: Boolean = true,
        override val apiVariantType: ApiVariantType = ApiVariantType.CORE,
    ) : SignatureFile() {
        override fun readContents() =
            try {
                file.readText(UTF_8)
            } catch (ex: IOException) {
                throw ApiParseException(
                    "Error reading API file",
                    location = FileLocation.createLocation(file.toPath()),
                    cause = ex
                )
            }
    }

    /** A [SignatureFile] that wraps an [InputStream]. */
    private data class SignatureFileFromStream(
        override val file: File,
        val inputStream: InputStream,
    ) : SignatureFile() {
        override fun readContents() = inputStream.bufferedReader().readText()
    }

    /** A [SignatureFile] that wraps a [String]. */
    private data class SignatureFileFromText(
        override val file: File,
        val contents: String,
    ) : SignatureFile() {
        override fun readContents() = contents
    }
}

@MetalavaApi
class ApiFile
private constructor(
    private val assembler: TextCodebaseAssembler,
    private val formatForLegacyFiles: FileFormat?,
) {

    private val codebase = assembler.codebase

    /**
     * The [FileLocationTracker] for the current file being parsed.
     *
     * Set by [parseApiSingleFile].
     */
    private lateinit var fileLocationTracker: FileLocationTracker

    /**
     * Report recoverable errors encountered while parsing.
     *
     * Retrieves the location of the error from [fileLocationTracker].
     */
    private val errorReporter =
        object : SignatureErrorReporter {
            override fun report(issue: Issues.Issue, message: String) {
                val location = fileLocationTracker.fileLocation()
                codebase.reporter.report(issue, null, message, location)
            }
        }

    /**
     * Provides support for parsing and caching [TypeItem]s.
     *
     * Defer creation until after the first file has been read and [kotlinStyleNulls] has been set
     * to a non-null value to ensure that it picks up the correct setting of [kotlinStyleNulls].
     */
    private val typeParser by
        lazy(LazyThreadSafetyMode.NONE) {
            TextTypeParser(codebase, kotlinStyleNulls!!, errorReporter)
        }

    /**
     * Provides support for creating [TypeItem]s for specific uses.
     *
     * Defer creation as it depends on [typeParser].
     */
    private val globalTypeItemFactory by
        lazy(LazyThreadSafetyMode.NONE) { TextTypeItemFactory(assembler, typeParser) }

    /** Creates [Item] instances for [codebase]. */
    private val itemFactory = assembler.itemFactory

    /**
     * Whether types should be interpreted to be in Kotlin format (e.g. ? suffix means nullable, !
     * suffix means unknown, and absence of a suffix means not nullable.
     *
     * Updated based on the header of the signature file being parsed.
     */
    private var kotlinStyleNulls: Boolean? = null

    /** The file format of the file being parsed. */
    lateinit var format: FileFormat

    /**
     * The [ApiVariant] which is defined within the current signature file being parsed.
     *
     * Set in [parseApiSingleFile].
     */
    private lateinit var apiVariant: ApiVariant

    /**
     * True if this is appending information from one signature file to a [Codebase] created from
     * another signature file.
     */
    private var appending: Boolean = false

    /** Map from [ClassItem] to [TextTypeItemFactory]. */
    private val classToTypeItemFactory = IdentityHashMap<ClassItem, TextTypeItemFactory>()

    companion object {
        /**
         * Parse API signature files.
         *
         * Used by non-Metalava Kotlin code.
         */
        @MetalavaApi
        fun parseApi(
            files: List<File>,
        ) = parseApi(SignatureFile.fromFiles(files))

        /**
         * Read API signature files into a [DefaultCodebase].
         *
         * Note: when reading from them multiple files, [DefaultCodebase.location] would refer to
         * the first file specified. each [Item.fileLocation] would correctly point out the source
         * file of each item.
         *
         * @param signatureFiles input signature files
         */
        fun parseApi(
            signatureFiles: List<SignatureFile>,
            codebaseConfig: Codebase.Config = Codebase.Config.NOOP,
            description: String? = null,
            classResolver: ClassResolver? = null,
            formatForLegacyFiles: FileFormat? = null,
            // Provides the called with access to the ApiFile.
            apiStatsConsumer: (Stats) -> Unit = {},
        ): Codebase {
            require(signatureFiles.isNotEmpty()) { "files must not be empty" }
            val actualDescription =
                description
                    ?: buildString {
                        append("Codebase loaded from ")
                        signatureFiles.joinTo(this)
                    }
            val assembler =
                TextCodebaseAssembler.createAssembler(
                    location = signatureFiles[0].file,
                    description = actualDescription,
                    codebaseConfig = codebaseConfig,
                    classResolver = classResolver,
                )
            val parser = ApiFile(assembler, formatForLegacyFiles)
            val apiSurfaces = codebaseConfig.apiSurfaces
            var first = true
            for (signatureFile in signatureFiles) {
                val file = signatureFile.file
                val apiText = signatureFile.readContents()
                val apiVariant = signatureFile.apiVariantFor(apiSurfaces)
                parser.parseApiSingleFile(
                    appending = !first,
                    path = file.toPath(),
                    apiText = apiText,
                    apiVariant = apiVariant,
                )
                first = false
            }

            apiStatsConsumer(parser.stats)

            return assembler.codebase
        }

        /**
         * Parse the API signature file from the [inputStream].
         *
         * This will consume the whole contents of the [inputStream] but it is the caller's
         * responsibility to close it.
         */
        @JvmStatic
        @MetalavaApi
        @Throws(ApiParseException::class)
        fun parseApi(filename: String, inputStream: InputStream): Codebase {
            val signatureFile = SignatureFile.fromStream(filename, inputStream)
            return parseApi(listOf(signatureFile))
        }

        /**
         * Extracts the bounds string list from the [typeParameterString].
         *
         * Given `T extends a.B & b.C<? super T>` this will return a list of `a.B` and `b.C<? super
         * T>`.
         */
        fun extractTypeParameterBoundsStringList(typeParameterString: String?): List<String> {
            val s = typeParameterString ?: return emptyList()
            val index = s.indexOf("extends ")
            if (index == -1) {
                return emptyList()
            }
            val list = mutableListOf<String>()
            var angleBracketBalance = 0
            var start = index + "extends ".length
            val length = s.length
            for (i in start until length) {
                val c = s[i]
                if (c == '&' && angleBracketBalance == 0) {
                    addNonBlankStringToList(list, typeParameterString, start, i)
                    start = i + 1
                } else if (c == '<') {
                    angleBracketBalance++
                } else if (c == '>') {
                    angleBracketBalance--
                    if (angleBracketBalance == 0) {
                        addNonBlankStringToList(list, typeParameterString, start, i + 1)
                        start = i + 1
                    }
                }
            }
            if (start < length) {
                addNonBlankStringToList(list, typeParameterString, start, length)
            }
            return list
        }

        private fun addNonBlankStringToList(
            list: MutableList<String>,
            s: String,
            from: Int,
            to: Int
        ) {
            val element = s.substring(from, to).trim()
            if (element.isNotEmpty()) list.add(element)
        }
    }

    /**
     * Mark this [SelectableItem] as being part of the main API surface, i.e. the one that is being
     * created.
     *
     * See [SignatureFile.forMainApiSurface].
     *
     * This will set [SelectableItem.emit] to [forMainApiSurface] and should only be called on
     * [SelectableItem]s which have been created from the main signature file.
     */
    private fun SelectableItem.markForMainApiSurface() {
        emit = apiVariant.surface.isMain
        markSelectedApiVariant()
    }

    /**
     * Record that this [SelectableItem] was loaded from a signature file that contains
     * [apiVariant].
     */
    private fun SelectableItem.markSelectedApiVariant() {
        if (apiVariant !in selectedApiVariants) {
            mutateSelectedApiVariants { add(apiVariant) }
        }
    }

    /**
     * It is only necessary to mark an existing class as being part of the main API surface, if it
     * should be but is not already.
     *
     * This will set [Item.emit] to `true` iff it was previously `false` and [forMainApiSurface] is
     * `true`. That ensures that a class that is not in the main API surface can be included in it
     * by another signature file, but once it is included it cannot be removed.
     *
     * e.g. Imagine that there are two files, `public.txt` and `system.txt` where the second extends
     * the first. When generating the system API classes in the `public.txt` will not be considered
     * part of it but any classes defined in `system.txt` will be, even if they were initially
     * created in `public.txt`. While `public.txt` should come first this ensures the correct
     * behavior irrespective of the order.
     */
    private fun ClassItem.markExistingClassForMainApiSurface() {
        if (!emit && apiVariant.surface.isMain) {
            markForMainApiSurface()
        }

        // Always record the ApiVariants to which this belongs, even if this was previously loaded.
        // This is safe because unlike `emit` which is Boolean the `selectedApiVariants` property is
        // a set of ApiVariants and this just adds an ApiVariant.
        markSelectedApiVariant()
    }

    private fun parseApiSingleFile(
        appending: Boolean,
        path: Path,
        apiText: String,
        apiVariant: ApiVariant,
    ) {
        if (appending) {
            // When we're appending, and the content is empty, nothing to do.
            if (apiText.isBlank()) {
                return
            }
        }

        // The behavior is slightly different when appending to an existing Codebase.
        this.appending = appending

        // Parse the header of the signature file to determine the format. If the signature file is
        // empty then `parseHeader` will return null, so it will default to `FileFormat.V2`.
        format =
            FileFormat.parseHeader(path, StringReader(apiText), formatForLegacyFiles)
                ?: FileFormat.V2

        // Remember the API variant of the file being parsed.
        this.apiVariant = apiVariant

        val tokenizer = Tokenizer(path, apiText.toCharArray())

        // Get the preceding tracker, if any.
        val precedingTracker =
            if (::fileLocationTracker.isInitialized) {
                fileLocationTracker
            } else {
                null
            }

        // Set the file location tracker to provide location information about the current file.
        fileLocationTracker = tokenizer

        // Disallow a mixture of kotlinStyleNulls settings.
        if (kotlinStyleNulls != null && kotlinStyleNulls != format.kotlinStyleNulls) {
            val precedingFile = precedingTracker!!.fileLocation().path
            errorReporter.report(
                "Preceding file $precedingFile has different setting of kotlin-style-nulls which may cause issues"
            )
        }
        kotlinStyleNulls = format.kotlinStyleNulls

        while (true) {
            val token = tokenizer.getToken() ?: break
            // TODO: Accept annotations on packages.
            if ("package" == token) {
                parsePackage(tokenizer)
            } else {
                throw ApiParseException("expected package got $token", tokenizer)
            }
        }
    }

    private fun parsePackage(tokenizer: Tokenizer) {
        var token: String = tokenizer.requireToken()

        // Metalava: including annotations in file now
        val annotations = getAnnotations(tokenizer, token)
        val modifiers = createModifiers(VisibilityLevel.PUBLIC, annotations)
        token = tokenizer.current
        tokenizer.assertIdent(token)
        val name: String = token

        // Wrap the modifiers and file location in a PackageDocs so that findOrCreatePackage(...)
        // will create a package with them and will check to make sure that an existing package, if
        // any, has matching modifiers.
        val packageDoc =
            MutablePackageDoc(
                name,
                fileLocation = tokenizer.fileLocation(),
                modifiers = modifiers,
            )
        val packageDocs = PackageDocs(mapOf(name to packageDoc))
        val pkg =
            try {
                codebase.findOrCreatePackage(name, packageDocs)
            } catch (e: IllegalStateException) {
                throw ApiParseException(e.message!!, tokenizer)
            }

        // Make sure that the package records the ApiVariants to which it belongs.
        pkg.markSelectedApiVariant()

        token = tokenizer.requireToken()
        if ("{" != token) {
            throw ApiParseException("expected '{' got $token", tokenizer)
        }
        while (true) {
            token = tokenizer.requireToken()
            if ("}" == token) {
                break
            } else {
                parseClass(pkg, tokenizer, token)
            }
        }
    }

    /**
     * Creates a type alias in the [pkg] with the [modifiers].
     *
     * It is expected that the starting position of the [tokenizer] is the "typealias" keyword, and
     * the next token will be the name and option type parameter list.
     *
     * When the method returns, the current [tokenizer] position will be the ";" at the end of the
     * typealias line.
     */
    private fun parseTypeAlias(
        pkg: DefaultPackageItem,
        tokenizer: Tokenizer,
        modifiers: MutableModifierList,
        location: FileLocation
    ) {
        var token = tokenizer.requireToken()
        tokenizer.assertIdent(token)

        val typeParameterListIndex = token.indexOf("<")

        val (name, typeParameterList, typeItemFactory) =
            if (typeParameterListIndex == -1) {
                Triple(token, TypeParameterList.NONE, globalTypeItemFactory)
            } else {
                val name = token.substring(0, typeParameterListIndex)
                val typeParameterListAndFactory =
                    createTypeParameterList(
                        globalTypeItemFactory,
                        "typealias $name",
                        token.substring(typeParameterListIndex)
                    )
                Triple(
                    name,
                    typeParameterListAndFactory.typeParameterList,
                    typeParameterListAndFactory.factory
                )
            }

        token = tokenizer.requireToken()
        if ("=" != token) {
            throw ApiParseException("expected = found $token", tokenizer)
        }

        val typeString = scanForTypeString(tokenizer, tokenizer.requireToken())
        token = tokenizer.current
        if (";" != token) {
            throw ApiParseException("expected ; found $token", tokenizer)
        }

        val type = typeItemFactory.getGeneralType(typeString)
        itemFactory.createTypeAliasItem(
            fileLocation = location,
            modifiers = modifiers,
            qualifiedName = pkg.qualifiedName() + "." + name,
            containingPackage = pkg,
            aliasedType = type,
            typeParameterList = typeParameterList,
        )
    }

    private fun parseClass(pkg: DefaultPackageItem, tokenizer: Tokenizer, startingToken: String) {
        var token = startingToken
        var classKind = ClassKind.CLASS
        var superClassType: ClassTypeItem? = null

        // Metalava: including annotations in file now
        val annotations = getAnnotations(tokenizer, token)
        token = tokenizer.current
        val modifiers = parseModifiers(tokenizer, token, annotations)

        // Remember this position as this seems like a good place to use to report issues with the
        // class item.
        val classPosition = tokenizer.fileLocation()

        token = tokenizer.current
        when (token) {
            "class" -> {
                token = tokenizer.requireToken()
            }
            "interface" -> {
                classKind = ClassKind.INTERFACE
                modifiers.setAbstract(true)
                token = tokenizer.requireToken()
            }
            "@interface" -> {
                classKind = ClassKind.ANNOTATION_TYPE
                modifiers.setAbstract(true)
                token = tokenizer.requireToken()
            }
            "enum" -> {
                classKind = ClassKind.ENUM
                modifiers.setFinal(true)
                modifiers.setStatic(true)
                superClassType = globalTypeItemFactory.superEnumType
                token = tokenizer.requireToken()
            }
            "typealias" -> {
                // Type aliases aren't classes, but they are defined at the same level as classes
                parseTypeAlias(pkg, tokenizer, modifiers, classPosition)
                // Don't continue creating a class item
                return
            }
            else -> {
                throw ApiParseException(
                    "expected one of class, interface, @interface, enum, or typealias; found: $token",
                    tokenizer
                )
            }
        }
        tokenizer.assertIdent(token)

        // The declaredClassType consists of the full name (i.e. preceded by the containing class's
        // full name followed by a '.' if there is one) plus the type parameter string.
        val declaredClassType: String = token

        // Extract lots of information from the declared class type.
        val (
            fullName,
            qualifiedClassName,
            outerClass,
            typeParameterList,
            typeItemFactory,
        ) = parseDeclaredClassType(pkg, declaredClassType, classPosition)

        token = tokenizer.requireToken()

        if ("extends" == token && classKind != ClassKind.INTERFACE) {
            val superClassTypeString = parseSuperTypeString(tokenizer, tokenizer.requireToken())
            superClassType =
                typeItemFactory.getSuperClassType(
                    superClassTypeString,
                )
            token = tokenizer.current
        }

        val interfaceTypes = mutableSetOf<ClassTypeItem>()
        if ("implements" == token || "extends" == token) {
            token = tokenizer.requireToken()
            while (true) {
                if ("{" == token) {
                    break
                } else if ("," != token) {
                    val interfaceTypeString = parseSuperTypeString(tokenizer, token)
                    val interfaceType = typeItemFactory.getInterfaceType(interfaceTypeString)
                    interfaceTypes.add(interfaceType)
                    token = tokenizer.current
                } else {
                    token = tokenizer.requireToken()
                }
            }
        }
        if (superClassType == globalTypeItemFactory.superEnumType) {
            // This can be taken either for an enum class, or a normal class that extends
            // java.lang.Enum (which was the old way of representing an enum in the API signature
            // files.
            classKind = ClassKind.ENUM
        } else if (classKind == ClassKind.ANNOTATION_TYPE) {
            // If the annotation was defined using @interface then add the implicit
            // "implements java.lang.annotation.Annotation".
            interfaceTypes.add(globalTypeItemFactory.superAnnotationType)
        } else if (globalTypeItemFactory.superAnnotationType in interfaceTypes) {
            // A normal class that implements java.lang.annotation.Annotation which was the old way
            // of representing an annotation in the API signature files. So, update the class kind
            // to match.
            classKind = ClassKind.ANNOTATION_TYPE
        }

        if ("{" != token) {
            throw ApiParseException("expected {, was $token", tokenizer)
        }

        // Above we marked all enums as static but for a top level class it's implicit
        if (classKind == ClassKind.ENUM && !fullName.contains(".")) {
            modifiers.setStatic(false)
        }

        // Get the characteristics of the class being added as they may be needed to compare against
        // the characteristics of the same class from a previously processed signature file.
        val newClassCharacteristics =
            ClassCharacteristics(
                fileLocation = classPosition,
                qualifiedName = qualifiedClassName,
                fullName = fullName,
                classKind = classKind,
                modifiers = modifiers.toImmutable(),
                superClassType = superClassType,
            )

        // Check to see if there is an existing class, if so merge this class definition into that
        // one and return. Otherwise, drop through and create a whole new class.
        if (tryMergingIntoExistingClass(tokenizer, newClassCharacteristics)) {
            return
        }

        // Default the superClassType() to java.lang.Object for any class that is not an interface,
        // annotation, or enum and which is not itself java.lang.Object.
        if (
            classKind == ClassKind.CLASS &&
                superClassType == null &&
                qualifiedClassName != JAVA_LANG_OBJECT
        ) {
            superClassType = globalTypeItemFactory.superObjectType
        }

        // Create the DefaultClassItem and set its package but do not add it to the package or
        // register it.
        val cl =
            itemFactory.createClassItem(
                fileLocation = classPosition,
                modifiers = modifiers,
                classKind = classKind,
                containingClass = outerClass,
                containingPackage = pkg,
                qualifiedName = qualifiedClassName,
                typeParameterList = typeParameterList,
                // All signature files have to be explicitly specified.
                origin = ClassOrigin.COMMAND_LINE,
                superClassType = superClassType,
                interfaceTypes = interfaceTypes.toList(),
            )
        cl.markForMainApiSurface()

        // Store the [TypeItemFactory] for this [ClassItem] so it can be retrieved later in
        // [typeItemFactoryForClass].
        if (!typeItemFactory.typeParameterScope.isEmpty()) {
            classToTypeItemFactory[cl] = typeItemFactory
        }

        // Parse the class body adding each member created to the class item being populated.
        parseClassBody(tokenizer, cl, typeItemFactory)
    }

    /**
     * Try merging the new class into an existing class that was previously loaded from a separate
     * signature file.
     *
     * Will throw an exception if there is an existing class but it is not compatible with the new
     * class.
     *
     * @return `false` if there is no existing class, `true` if there is and the merge succeeded.
     */
    private fun tryMergingIntoExistingClass(
        tokenizer: Tokenizer,
        newClassCharacteristics: ClassCharacteristics,
    ): Boolean {
        // Check for the existing class from a previously parsed file. If it could not be found
        // then return.
        val existingClass =
            codebase.findClassInCodebase(newClassCharacteristics.qualifiedName) ?: return false

        // Make sure the new class characteristics are compatible with the old class
        // characteristic.
        val existingCharacteristics = ClassCharacteristics.of(existingClass)
        if (!existingCharacteristics.isCompatible(newClassCharacteristics)) {
            throw ApiParseException(
                "Incompatible $existingClass definitions",
                newClassCharacteristics.fileLocation
            )
        }

        // Add new annotations to the existing class
        val newClassAnnotations = newClassCharacteristics.modifiers.annotations().toSet()
        val existingClassAnnotations = existingCharacteristics.modifiers.annotations().toSet()

        val extraAnnotations = newClassAnnotations.subtract(existingClassAnnotations)
        if (extraAnnotations.isNotEmpty()) {
            existingClass.mutateModifiers { mutateAnnotations { addAll(extraAnnotations) } }
        }

        // Use the latest super class.
        val newSuperClassType = newClassCharacteristics.superClassType
        if (
            newSuperClassType != null && existingCharacteristics.superClassType != newSuperClassType
        ) {
            // Duplicate class with conflicting superclass names are found. Since the class
            // definition found later should be prioritized, overwrite the superclass type.
            existingClass.setSuperClassType(newSuperClassType)
        }

        // Parse the class body adding each member created to the existing class.
        parseClassBody(tokenizer, existingClass, typeItemFactoryForClass(existingClass))

        // Although the class was first defined in a separate file it is being modified in the
        // current file so that may include it in the main API surface.
        existingClass.markExistingClassForMainApiSurface()

        return true
    }

    /** Get the [TextTypeItemFactory] for a previously created [ClassItem]. */
    private fun typeItemFactoryForClass(classItem: ClassItem?): TextTypeItemFactory =
        classItem?.let { classToTypeItemFactory[classItem] } ?: globalTypeItemFactory

    /** Parse the class body, adding members to [cl]. */
    private fun parseClassBody(
        tokenizer: Tokenizer,
        cl: DefaultClassItem,
        classTypeItemFactory: TextTypeItemFactory,
    ) {
        var token = tokenizer.requireToken()
        while (true) {
            if ("}" == token) {
                break
            } else if ("ctor" == token) {
                token = tokenizer.requireToken()
                parseConstructor(tokenizer, cl, classTypeItemFactory, token)
            } else if ("method" == token) {
                token = tokenizer.requireToken()
                parseMethod(tokenizer, cl, classTypeItemFactory, token)
            } else if ("field" == token) {
                token = tokenizer.requireToken()
                parseField(tokenizer, cl, classTypeItemFactory, token, false)
            } else if ("enum_constant" == token) {
                token = tokenizer.requireToken()
                parseField(tokenizer, cl, classTypeItemFactory, token, true)
            } else if ("property" == token) {
                token = tokenizer.requireToken()
                parseProperty(tokenizer, cl, classTypeItemFactory, token)
            } else {
                throw ApiParseException("expected ctor, enum_constant, field or method", tokenizer)
            }
            token = tokenizer.requireToken()
        }
    }

    /**
     * Parse a super type string, i.e. a string representing a super class type or a super interface
     * type.
     */
    private fun parseSuperTypeString(tokenizer: Tokenizer, initialToken: String): String {
        var token = getAnnotationCompleteToken(tokenizer, initialToken)

        // Use the token directly if it is complete, otherwise construct the super class type
        // string from as many tokens as necessary.
        return if (!isIncompleteTypeToken(token)) {
            token
        } else {
            buildString {
                append(token)

                // Make sure full super class name is found if there are type use
                // annotations. This can't use [parseType] because the next token might be a
                // separate type (classes only have a single `extends` type, but all
                // interface supertypes are listed as `extends` instead of `implements`).
                // However, this type cannot be an array, so unlike [parseType] this does
                // not need to check if the next token has annotations.
                do {
                    token = getAnnotationCompleteToken(tokenizer, tokenizer.current)
                    append(" ")
                    append(token)
                } while (isIncompleteTypeToken(token))
            }
        }
    }

    /** Encapsulates multiple return values from [parseDeclaredClassType]. */
    private data class DeclaredClassTypeComponents(
        /** The full name of the class, including outer class prefix. */
        val fullName: String,
        /** The fully qualified name, including package and full name. */
        val qualifiedName: String,
        /** The optional, resolved outer [ClassItem]. */
        val outerClass: DefaultClassItem?,
        /** The set of type parameters. */
        val typeParameterList: TypeParameterList,
        /**
         * The [TextTypeItemFactory] including any type parameters in the [typeParameterList] in its
         * [TextTypeItemFactory.typeParameterScope].
         */
        val typeItemFactory: TextTypeItemFactory,
    )

    /**
     * Splits the declared class type into [DeclaredClassTypeComponents].
     *
     * For example "Foo" would split into full name "Foo" and an empty type parameter list, while
     * `"Foo.Bar<A, B extends java.lang.String, C>"` would split into full name `"Foo.Bar"` and type
     * parameter list with `"A"`,`"B extends java.lang.String"`, and `"C"` as type parameters.
     *
     * If the qualified name matches an existing class then return its information.
     */
    private fun parseDeclaredClassType(
        pkg: DefaultPackageItem,
        declaredClassType: String,
        classFileLocation: FileLocation,
    ): DeclaredClassTypeComponents {
        // Split the declared class type into full name and type parameters.
        val paramIndex = declaredClassType.indexOf('<')
        val (fullName, typeParameterListString) =
            if (paramIndex == -1) {
                Pair(declaredClassType, "")
            } else {
                Pair(
                    declaredClassType.substring(0, paramIndex),
                    declaredClassType.substring(paramIndex)
                )
            }
        val pkgName = pkg.qualifiedName()
        val qualifiedName = qualifiedName(pkgName, fullName)

        // Split the full name into an optional outer class and a simple name.
        val nestedClassIndex = fullName.lastIndexOf('.')
        val outerClass =
            if (nestedClassIndex == -1) {
                null
            } else {
                val outerClassFullName = fullName.substring(0, nestedClassIndex)
                val qualifiedOuterClassName = qualifiedName(pkgName, outerClassFullName)

                // Search for the outer class in the codebase. This is safe as the outer class
                // always precedes its nested classes.
                assembler.getOrCreateClass(
                    qualifiedOuterClassName,
                    isOuterClassOfClassInThisCodebase = true
                ) as DefaultClassItem
            }

        // Get the [TextTypeItemFactory] for the outer class, if any, from a previously stored one,
        // otherwise use the [globalTypeItemFactory] as the [ClassItem] is a stub and so has no type
        // parameters.
        val outerClassTypeItemFactory = typeItemFactoryForClass(outerClass)

        // Create type parameter list and factory from the string and optional outer class factory.
        val (typeParameterList, typeItemFactory) =
            if (typeParameterListString == "")
                TypeParameterListAndFactory(TypeParameterList.NONE, outerClassTypeItemFactory)
            else
                createTypeParameterList(
                    outerClassTypeItemFactory,
                    "class $qualifiedName",
                    typeParameterListString,
                )

        // Decide which type parameter list and factory to actually use.
        //
        // If the class already exists then reuse its type parameter list and factory, otherwise use
        // the newly created one.
        //
        // The reason for this is that otherwise any types parsed with the newly created factory
        // would reference type parameters in the newly created list which are different to the ones
        // belonging to the existing class.
        val (actualTypeParameterList, actualTypeItemFactory) =
            codebase.findClassInCodebase(qualifiedName)?.let { existingClass ->
                // Check to make sure that the type parameter lists are the same.
                val existingTypeParameterList = existingClass.typeParameterList
                val existingTypeParameterListString = existingTypeParameterList.toString()
                val normalizedTypeParameterListString = typeParameterList.toString()
                if (normalizedTypeParameterListString != existingTypeParameterListString) {
                    val location = existingClass.fileLocation
                    throw ApiParseException(
                        "Inconsistent type parameter list for $qualifiedName, this has $normalizedTypeParameterListString but it was previously defined as $existingTypeParameterListString at $location",
                        classFileLocation
                    )
                }

                Pair(existingTypeParameterList, typeItemFactoryForClass(existingClass))
            }
                ?: Pair(typeParameterList, typeItemFactory)

        return DeclaredClassTypeComponents(
            fullName = fullName,
            qualifiedName = qualifiedName,
            outerClass = outerClass,
            typeParameterList = actualTypeParameterList,
            typeItemFactory = actualTypeItemFactory,
        )
    }

    /**
     * If the [startingToken] contains the beginning of an annotation, pulls additional tokens from
     * [tokenizer] to complete the annotation, returning the full token. If there isn't an
     * annotation, returns the original [startingToken].
     *
     * When the method returns, the [tokenizer] will point to the token after the end of the
     * returned string.
     */
    private fun getAnnotationCompleteToken(tokenizer: Tokenizer, startingToken: String): String {
        return if (startingToken.contains('@')) {
            val prefix = startingToken.substringBefore('@')
            val annotationStart = startingToken.substring(startingToken.indexOf('@'))
            val annotation = getAnnotationSource(tokenizer, annotationStart)
            "$prefix$annotation"
        } else {
            tokenizer.requireToken()
            startingToken
        }
    }

    /**
     * If the [startingToken] is the beginning of an annotation, returns the annotation parsed from
     * the [tokenizer]. Returns null otherwise.
     *
     * When the method returns, the [tokenizer] will point to the token after the annotation.
     */
    private fun getAnnotationSource(tokenizer: Tokenizer, startingToken: String): String? {
        var token = startingToken
        if (token.startsWith('@')) {
            // Annotation
            var annotation = token

            // Restore annotations that were shortened on export
            annotation = unshortenAnnotation(annotation)
            token = tokenizer.requireToken()
            if (token == "(") {
                // Annotation arguments; potentially nested
                var balance = 0
                val start = tokenizer.offset() - 1
                while (true) {
                    if (token == "(") {
                        balance++
                    } else if (token == ")") {
                        balance--
                        if (balance == 0) {
                            break
                        }
                    }
                    token = tokenizer.requireToken()
                }
                annotation += tokenizer.getStringFromOffset(start)
                // Move the tokenizer so that when the method returns it points to the token after
                // the end of the annotation.
                tokenizer.requireToken()
            }
            return annotation
        } else {
            return null
        }
    }

    /**
     * Collects all the sequential annotations from the [tokenizer] beginning with [startingToken],
     * returning them as a (possibly empty) list.
     *
     * When the method returns, the [tokenizer] will point to the token after the annotation list.
     */
    private fun getAnnotations(tokenizer: Tokenizer, startingToken: String) = buildList {
        var token = startingToken
        while (true) {
            val annotationSource = getAnnotationSource(tokenizer, token) ?: break
            token = tokenizer.current
            DefaultAnnotationItem.create(codebase, annotationSource)?.let { annotationItem ->
                add(annotationItem)
            }
        }
    }

    /**
     * Create [ParameterItem]s for the [containingCallable] from the [parameters] using the
     * [typeItemFactory] to create types.
     *
     * This is called from within the constructor of the [containingCallable] so must only access
     * its `name` and its reference. In particularly it must not access its
     * [CallableItem.parameters] property as this is called during its initialization.
     */
    private fun createParameterItems(
        containingCallable: CallableItem,
        parameters: List<ParameterInfo>,
        typeItemFactory: TextTypeItemFactory
    ): List<ParameterItem> {
        val methodFingerprint = MethodFingerprint(containingCallable.name(), parameters.size)
        return parameters.map { it.create(containingCallable, typeItemFactory, methodFingerprint) }
    }

    private fun parseConstructor(
        tokenizer: Tokenizer,
        containingClass: DefaultClassItem,
        classTypeItemFactory: TextTypeItemFactory,
        startingToken: String
    ) {
        var token = startingToken
        val method: ConstructorItem

        // Metalava: including annotations in file now
        val annotations = getAnnotations(tokenizer, token)
        token = tokenizer.current
        val modifiers = parseModifiers(tokenizer, token, annotations)

        // Get a TypeParameterList and accompanying TypeItemFactory
        val (typeParameterList, typeItemFactory) =
            parseTypeParameterList(tokenizer, classTypeItemFactory)
        token = tokenizer.current

        tokenizer.assertIdent(token)
        val name: String =
            token.substring(
                token.lastIndexOf('.') + 1
            ) // For nested classes, strip outer classes from name
        val parameters = parseParameterList(tokenizer)
        token = tokenizer.requireToken()
        var throwsList = emptyList<ExceptionTypeItem>()
        if ("throws" == token) {
            throwsList = parseThrows(tokenizer, typeItemFactory)
            token = tokenizer.current
        }
        if (";" != token) {
            throw ApiParseException("expected ; found $token", tokenizer)
        }

        method =
            itemFactory.createConstructorItem(
                fileLocation = tokenizer.fileLocation(),
                modifiers = modifiers,
                documentationFactory = ItemDocumentation.NONE_FACTORY,
                name = name,
                containingClass = containingClass,
                typeParameterList = typeParameterList,
                returnType = containingClass.type(),
                parameterItemsFactory = { methodItem ->
                    createParameterItems(methodItem, parameters, typeItemFactory)
                },
                throwsTypes = throwsList,
                // Signature files do not track implicit constructors, all constructors are treated
                // the same as whether it was created by the compiler or in the source has no effect
                // on the API surface.
                implicitConstructor = false,
            )
        method.markForMainApiSurface()

        if (!containingClass.constructors().contains(method)) {
            containingClass.addConstructor(method)
        }
    }

    private fun parseMethod(
        tokenizer: Tokenizer,
        cl: DefaultClassItem,
        classTypeItemFactory: TextTypeItemFactory,
        startingToken: String
    ) {
        var token = startingToken
        val method: MethodItem

        // Metalava: including annotations in file now
        val annotations = getAnnotations(tokenizer, token)
        token = tokenizer.current
        val modifiers = parseModifiers(tokenizer, token, annotations)

        // Get a TypeParameterList and accompanying TypeParameterScope
        val (typeParameterList, typeItemFactory) =
            parseTypeParameterList(tokenizer, classTypeItemFactory)
        token = tokenizer.current
        tokenizer.assertIdent(token)

        val returnTypeString: String
        val parameters: List<ParameterInfo>
        val name: String
        if (format.kotlinNameTypeOrder) {
            // Kotlin style: parse the name, the parameter list, then the return type.
            name = token
            parameters = parseParameterList(tokenizer)
            token = tokenizer.requireToken()
            if (token != ":") {
                throw ApiParseException(
                    "Expecting \":\" after parameter list, found $token.",
                    tokenizer
                )
            }
            token = tokenizer.requireToken()
            tokenizer.assertIdent(token)
            returnTypeString = scanForTypeString(tokenizer, token)
            token = tokenizer.current
        } else {
            // Java style: parse the return type, the name, and then the parameter list.
            returnTypeString = scanForTypeString(tokenizer, token)
            token = tokenizer.current
            tokenizer.assertIdent(token)
            name = token
            parameters = parseParameterList(tokenizer)
            token = tokenizer.requireToken()
        }

        val returnType =
            typeItemFactory.getMethodReturnType(
                returnTypeString,
                annotations,
                MethodFingerprint(name, parameters.size),
                cl.isAnnotationType()
            )
        synchronizeNullability(returnType, modifiers)

        if (cl.isInterface() && !modifiers.isDefault() && !modifiers.isStatic()) {
            modifiers.setAbstract(true)
        }

        var throwsList = emptyList<ExceptionTypeItem>()
        var defaultAnnotationMethodValue = ""

        when (token) {
            "throws" -> {
                throwsList = parseThrows(tokenizer, typeItemFactory)
                token = tokenizer.current
            }
            "default" -> {
                defaultAnnotationMethodValue = parseDefault(tokenizer)
                token = tokenizer.current
            }
        }
        if (";" != token) {
            throw ApiParseException("expected ; found $token", tokenizer)
        }

        method =
            itemFactory.createMethodItem(
                fileLocation = tokenizer.fileLocation(),
                modifiers = modifiers,
                documentationFactory = ItemDocumentation.NONE_FACTORY,
                name = name,
                containingClass = cl,
                typeParameterList = typeParameterList,
                returnType = returnType,
                parameterItemsFactory = { containingCallable ->
                    createParameterItems(containingCallable, parameters, typeItemFactory)
                },
                throwsTypes = throwsList,
                annotationDefault = defaultAnnotationMethodValue,
            )

        // Ignore enum synthetic methods. They are no longer included in signature files as they add
        // no information. However, they did use to be included and so this filters them out to
        // ensure that the resulting Codebase is consistent with the original source Codebase.
        if (method.isEnumSyntheticMethod()) return

        method.markForMainApiSurface()

        if (appending) {
            // If the method already exists in the class item because it was defined in a previous
            // signature file then replace it with this one, otherwise just add this method.
            cl.replaceOrAddMethod(method)
        } else {
            // Just add the method to the class.
            cl.addMethod(method)
        }
    }

    private fun parseField(
        tokenizer: Tokenizer,
        cl: DefaultClassItem,
        classTypeItemFactory: TextTypeItemFactory,
        startingToken: String,
        isEnumConstant: Boolean,
    ) {
        var token = startingToken
        val annotations = getAnnotations(tokenizer, token)
        token = tokenizer.current
        val modifiers = parseModifiers(tokenizer, token, annotations)
        token = tokenizer.current
        tokenizer.assertIdent(token)

        val typeString: String
        val name: String
        if (format.kotlinNameTypeOrder) {
            // Kotlin style: parse the name, then the type.
            name = parseNameWithColon(token, tokenizer)
            token = tokenizer.requireToken()
            tokenizer.assertIdent(token)
            typeString = scanForTypeString(tokenizer, token)
            token = tokenizer.current
        } else {
            // Java style: parse the name, then the type.
            typeString = scanForTypeString(tokenizer, token)
            token = tokenizer.current
            tokenizer.assertIdent(token)
            name = token
            token = tokenizer.requireToken()
        }

        // Get the optional value.
        val valueString =
            if ("=" == token) {
                token = tokenizer.requireToken(false)
                token.also { token = tokenizer.requireToken() }
            } else null

        // Parse the type string and then synchronize the field's nullability with the type.
        val type =
            classTypeItemFactory.getFieldType(
                underlyingType = typeString,
                isEnumConstant = isEnumConstant,
                isFinal = modifiers.isFinal(),
                isInitialValueNonNull = { valueString != null && valueString != "null" },
                itemAnnotations = annotations,
            )
        synchronizeNullability(type, modifiers)

        // Parse the value string.
        val fieldValue =
            valueString?.let { FixedFieldValue(parseValue(type, valueString, tokenizer)) }

        if (";" != token) {
            throw ApiParseException("expected ; found $token", tokenizer)
        }
        val field =
            itemFactory.createFieldItem(
                fileLocation = tokenizer.fileLocation(),
                modifiers = modifiers,
                documentationFactory = ItemDocumentation.NONE_FACTORY,
                name = name,
                containingClass = cl,
                type = type,
                isEnumConstant = isEnumConstant,
                fieldValue = fieldValue,
            )
        field.markForMainApiSurface()
        cl.addField(field)
    }

    private fun parseModifiers(
        tokenizer: Tokenizer,
        startingToken: String?,
        annotations: List<AnnotationItem>
    ): MutableModifierList {
        var token = startingToken
        val modifiers = createModifiers(VisibilityLevel.PACKAGE_PRIVATE, annotations)

        processModifiers@ while (true) {
            token =
                when (token) {
                    "public" -> {
                        modifiers.setVisibilityLevel(VisibilityLevel.PUBLIC)
                        tokenizer.requireToken()
                    }
                    "protected" -> {
                        modifiers.setVisibilityLevel(VisibilityLevel.PROTECTED)
                        tokenizer.requireToken()
                    }
                    "private" -> {
                        modifiers.setVisibilityLevel(VisibilityLevel.PRIVATE)
                        tokenizer.requireToken()
                    }
                    "internal" -> {
                        modifiers.setVisibilityLevel(VisibilityLevel.INTERNAL)
                        tokenizer.requireToken()
                    }
                    "static" -> {
                        modifiers.setStatic(true)
                        tokenizer.requireToken()
                    }
                    "final" -> {
                        modifiers.setFinal(true)
                        tokenizer.requireToken()
                    }
                    "deprecated" -> {
                        modifiers.setDeprecated(true)
                        tokenizer.requireToken()
                    }
                    "abstract" -> {
                        modifiers.setAbstract(true)
                        tokenizer.requireToken()
                    }
                    "transient" -> {
                        modifiers.setTransient(true)
                        tokenizer.requireToken()
                    }
                    "volatile" -> {
                        modifiers.setVolatile(true)
                        tokenizer.requireToken()
                    }
                    "sealed" -> {
                        modifiers.setSealed(true)
                        tokenizer.requireToken()
                    }
                    "default" -> {
                        modifiers.setDefault(true)
                        tokenizer.requireToken()
                    }
                    "synchronized" -> {
                        modifiers.setSynchronized(true)
                        tokenizer.requireToken()
                    }
                    "native" -> {
                        modifiers.setNative(true)
                        tokenizer.requireToken()
                    }
                    "strictfp" -> {
                        modifiers.setStrictFp(true)
                        tokenizer.requireToken()
                    }
                    "infix" -> {
                        modifiers.setInfix(true)
                        tokenizer.requireToken()
                    }
                    "operator" -> {
                        modifiers.setOperator(true)
                        tokenizer.requireToken()
                    }
                    "inline" -> {
                        modifiers.setInline(true)
                        tokenizer.requireToken()
                    }
                    "value" -> {
                        modifiers.setValue(true)
                        tokenizer.requireToken()
                    }
                    "suspend" -> {
                        modifiers.setSuspend(true)
                        tokenizer.requireToken()
                    }
                    "vararg" -> {
                        modifiers.setVarArg(true)
                        tokenizer.requireToken()
                    }
                    "fun" -> {
                        modifiers.setFunctional(true)
                        tokenizer.requireToken()
                    }
                    "data" -> {
                        modifiers.setData(true)
                        tokenizer.requireToken()
                    }
                    else -> break@processModifiers
                }
        }
        return modifiers
    }

    /** Creates a [MutableModifierList], setting the deprecation based on the [annotations]. */
    private fun createModifiers(
        visibility: VisibilityLevel,
        annotations: List<AnnotationItem>
    ): MutableModifierList {
        val modifiers = createMutableModifiers(visibility, annotations)
        // @Deprecated is also treated as a "modifier"
        if (annotations.any { it.qualifiedName == JAVA_LANG_DEPRECATED }) {
            modifiers.setDeprecated(true)
        }
        return modifiers
    }

    private fun parseValue(
        type: TypeItem,
        value: String?,
        fileLocationTracker: FileLocationTracker,
    ): Any? {
        return if (value != null) {
            if (type is PrimitiveTypeItem) {
                parsePrimitiveValue(type, value, fileLocationTracker)
            } else if (type.isString()) {
                if ("null" == value) {
                    null
                } else {
                    javaUnescapeString(value.substring(1, value.length - 1))
                }
            } else {
                value
            }
        } else null
    }

    private fun parsePrimitiveValue(
        type: PrimitiveTypeItem,
        value: String,
        fileLocationTracker: FileLocationTracker,
    ): Any {
        return when (type.kind) {
            Primitive.BOOLEAN ->
                if ("true" == value) java.lang.Boolean.TRUE else java.lang.Boolean.FALSE
            Primitive.BYTE,
            Primitive.SHORT,
            Primitive.INT -> Integer.valueOf(value)
            Primitive.LONG -> java.lang.Long.valueOf(value.substring(0, value.length - 1))
            Primitive.FLOAT ->
                when (value) {
                    "(1.0f/0.0f)",
                    "(1.0f / 0.0f)" -> Float.POSITIVE_INFINITY
                    "(-1.0f/0.0f)",
                    "(-1.0f / 0.0f)" -> Float.NEGATIVE_INFINITY
                    "(0.0f/0.0f)",
                    "(0.0f / 0.0f)" -> Float.NaN
                    else -> java.lang.Float.valueOf(value)
                }
            Primitive.DOUBLE ->
                when (value) {
                    "(1.0/0.0)",
                    "(1.0 / 0.0)" -> Double.POSITIVE_INFINITY
                    "(-1.0/0.0)",
                    "(-1.0 / 0.0)" -> Double.NEGATIVE_INFINITY
                    "(0.0/0.0)",
                    "(0.0 / 0.0)" -> Double.NaN
                    else -> java.lang.Double.valueOf(value)
                }
            Primitive.CHAR -> value.toInt().toChar()
            Primitive.VOID ->
                throw ApiParseException(
                    "Found value $value assigned to void type",
                    fileLocationTracker
                )
        }
    }

    private fun parseProperty(
        tokenizer: Tokenizer,
        cl: DefaultClassItem,
        classTypeItemFactory: TextTypeItemFactory,
        startingToken: String
    ) {
        var token = startingToken

        // Metalava: including annotations in file now
        val annotations = getAnnotations(tokenizer, token)
        token = tokenizer.current
        val modifiers = parseModifiers(tokenizer, token, annotations)

        // Get a TypeParameterList and accompanying TypeParameterScope
        val (typeParameterList, typeItemFactory) =
            parseTypeParameterList(tokenizer, classTypeItemFactory)
        token = tokenizer.current

        val typeString: String
        val receiverNamePair: Pair<TypeItem?, String>
        if (format.kotlinNameTypeOrder) {
            // Kotlin style: parse the name, then the type.
            receiverNamePair = parsePropertyReceiverAndName(tokenizer, typeItemFactory)
            token = tokenizer.current
            typeString = scanForTypeString(tokenizer, token)
            token = tokenizer.current
        } else {
            // Java style: parse the type, then the name.
            typeString = scanForTypeString(tokenizer, token)
            receiverNamePair = parsePropertyReceiverAndName(tokenizer, typeItemFactory)
            token = tokenizer.current
        }
        val type = typeItemFactory.getGeneralType(typeString)
        synchronizeNullability(type, modifiers)

        if (";" != token) {
            throw ApiParseException("expected ; found $token", tokenizer)
        }
        val property =
            itemFactory.createPropertyItem(
                fileLocation = tokenizer.fileLocation(),
                modifiers = modifiers,
                name = receiverNamePair.second,
                containingClass = cl,
                type = type,
                receiver = receiverNamePair.first,
                typeParameterList = typeParameterList,
            )
        property.markForMainApiSurface()
        cl.addProperty(property)
    }

    /**
     * Starting from the current token of [tokenizer], parses the optional receiver type and then
     * the name of a property.
     *
     * After the method returns, the caller should continue processing at the new current token of
     * [tokenizer], which will be the token after
     */
    private fun parsePropertyReceiverAndName(
        tokenizer: Tokenizer,
        typeItemFactory: TextTypeItemFactory
    ): Pair<TypeItem?, String> {
        // If there's no receiver, scanning for the type string should just return the name.
        // If there is a receiver, because of how the tokens are broken up, it should return
        // "receiver.name", which can then be split on the last "." to the receiver and name.
        val receiverAndName = scanForTypeString(tokenizer, tokenizer.current)
        val namePossiblyWithColon: String
        val receiverTypeString: String?
        if (receiverAndName.contains(".")) {
            namePossiblyWithColon = receiverAndName.substringAfterLast(".")
            receiverTypeString = receiverAndName.substringBeforeLast(".")
        } else {
            namePossiblyWithColon = receiverAndName
            receiverTypeString = null
        }

        val name =
            if (format.kotlinNameTypeOrder) {
                parseNameWithColon(namePossiblyWithColon, tokenizer)
            } else {
                tokenizer.assertIdent(namePossiblyWithColon)
                namePossiblyWithColon
            }
        val receiverType = receiverTypeString?.let { typeItemFactory.getGeneralType(it) }

        return receiverType to name
    }

    /**
     * Parses a type parameter list enclosed in "<>", if one exists.
     *
     * Starts processing from the current token of [tokenizer]. If that token is not "<", returns an
     * empty type parameter list.
     *
     * After the method returns, the caller should continue processing at the new current token of
     * [tokenizer], which will be the token after the type parameter list, if it exists, or the same
     * as the original current token, if there was no type parameter list.
     */
    private fun parseTypeParameterList(
        tokenizer: Tokenizer,
        enclosingTypeItemFactory: TextTypeItemFactory,
    ): TypeParameterListAndFactory<TextTypeItemFactory> {
        var token: String = tokenizer.current
        // No type parameters to parse. The current token is unchanged
        if ("<" != token) {
            return TypeParameterListAndFactory(TypeParameterList.NONE, enclosingTypeItemFactory)
        }

        val start = tokenizer.offset() - 1
        var balance = 1
        while (balance > 0) {
            token = tokenizer.requireToken()
            if (token == "<") {
                balance++
            } else if (token == ">") {
                balance--
            }
        }
        val typeParameterListString = tokenizer.getStringFromOffset(start)
        // Set the tokenizer to the next token, so that the caller should continue processing at
        // tokenizer.current (in alignment with the no type parameter case).
        tokenizer.requireToken()
        return if (typeParameterListString.isEmpty()) {
            TypeParameterListAndFactory(TypeParameterList.NONE, enclosingTypeItemFactory)
        } else {
            // Use the file location as a part of the description of the scope as at this point
            // there is no other information available.
            val scopeDescription = "${tokenizer.fileLocation()}"
            createTypeParameterList(
                enclosingTypeItemFactory,
                scopeDescription,
                typeParameterListString
            )
        }
    }

    /**
     * Creates a [TypeParameterList] and accompanying [TypeParameterScope].
     *
     * The [typeParameterListString] should be the string representation of a list of type
     * parameters, like "<A>" or "<A, B extends java.lang.String, C>".
     *
     * @return a [Pair] of [TypeParameterList] and [TextTypeItemFactory] that contains those type
     *   parameters.
     */
    private fun createTypeParameterList(
        enclosingTypeItemFactory: TextTypeItemFactory,
        scopeDescription: String,
        typeParameterListString: String
    ): TypeParameterListAndFactory<TextTypeItemFactory> {
        // Split the type parameter list string into a list of strings, one for each type
        // parameter.
        val typeParameterStrings = TextTypeParser.typeParameterStrings(typeParameterListString)

        // Create the List<TypeParameterItem> and the corresponding TypeItemFactory that can be
        // used to resolve TypeParameterItems from the list. This performs the construction in two
        // stages to handle cycles between the parameters.
        return DefaultTypeParameterList.createTypeParameterItemsAndFactory(
            enclosingTypeItemFactory,
            scopeDescription,
            typeParameterStrings,
            // Create a `TextTypeParameterItem` from the type parameter string.
            { createTypeParameterItem(it) },
            // Create, set and return the [BoundsTypeItem] list.
            { typeItemFactory, typeParameterString ->
                val boundsStringList = extractTypeParameterBoundsStringList(typeParameterString)
                boundsStringList.map { typeItemFactory.getBoundsType(it) }
            },
        )
    }

    /**
     * Create a partially initialized [DefaultTypeParameterItem].
     *
     * This extracts the [TypeParameterItem.isReified] and [TypeParameterItem.name] from the
     * [typeParameterString] and creates a [DefaultTypeParameterItem] with those properties
     * initialized but the [DefaultTypeParameterItem.bounds] is not.
     */
    private fun createTypeParameterItem(typeParameterString: String): DefaultTypeParameterItem {
        val length = typeParameterString.length
        var nameEnd = length

        val isReified = typeParameterString.startsWith("reified ")
        val nameStart =
            if (isReified) {
                8 // "reified ".length
            } else {
                0
            }

        for (i in nameStart until length) {
            val c = typeParameterString[i]
            if (!Character.isJavaIdentifierPart(c)) {
                nameEnd = i
                break
            }
        }
        val name = typeParameterString.substring(nameStart, nameEnd)

        // TODO: Type use annotations support will need to handle annotations on the parameter.
        val modifiers = createImmutableModifiers(VisibilityLevel.PUBLIC)

        return itemFactory.createTypeParameterItem(
            modifiers = modifiers,
            name = name,
            isReified = isReified,
        )
    }

    /**
     * Parses a list of parameters. Before calling, [tokenizer] should point to the token *before*
     * the opening `(` of the parameter list (the method starts by calling
     * [Tokenizer.requireToken]).
     *
     * When the method returns, [tokenizer] will point to the closing `)` of the parameter list.
     */
    private fun parseParameterList(
        tokenizer: Tokenizer,
    ): List<ParameterInfo> {
        val parameters = mutableListOf<ParameterInfo>()
        var token: String = tokenizer.requireToken()
        if ("(" != token) {
            throw ApiParseException("expected (, was $token", tokenizer)
        }
        token = tokenizer.requireToken()
        var index = 0
        while (true) {
            if (")" == token) {
                // All parameters are parsed, return them.
                return parameters
            }

            // Each item can be:
            //   optional-"optional" annotations optional-modifiers
            //   type-with-use-annotations-and-generics optional-name

            // Used to represent the presence of a default value, instead of showing the entire
            // default value
            val hasOptionalKeyword = token == "optional"
            if (hasOptionalKeyword) {
                token = tokenizer.requireToken()
            }

            // Metalava: including annotations in file now
            val annotations = getAnnotations(tokenizer, token)
            token = tokenizer.current
            val modifiers = parseModifiers(tokenizer, token, annotations)
            token = tokenizer.current

            val typeString: String
            val name: String
            val publicName: String?
            if (format.kotlinNameTypeOrder) {
                // Kotlin style: parse the name (only considered a public name if it is not `_`,
                // which is used as a placeholder for params without public names), then the type.
                name = parseNameWithColon(token, tokenizer)
                publicName =
                    if (name == "_") {
                        null
                    } else {
                        name
                    }
                token = tokenizer.requireToken()
                // Token should now represent the type
                typeString = scanForTypeString(tokenizer, token)
                token = tokenizer.current
            } else {
                // Java style: parse the type, then the public name if it has one.
                typeString = scanForTypeString(tokenizer, token)
                token = tokenizer.current
                if (Tokenizer.isIdent(token) && token != "=") {
                    name = token
                    publicName = name
                    token = tokenizer.requireToken()
                } else {
                    name = "arg" + (index + 1)
                    publicName = null
                }
            }

            when (token) {
                "," -> {
                    token = tokenizer.requireToken()
                }
                ")" -> {
                    // closing parenthesis
                }
                else -> {
                    throw ApiParseException("expected , or ), found $token", tokenizer)
                }
            }

            // Select the DefaultValue for the parameter.
            val defaultValue =
                if (hasOptionalKeyword) {
                    // It has an optional keyword, so it has a default value but the actual value is
                    // not known.
                    ParameterDefaultValue.UNKNOWN
                } else {
                    // It does not have an optional keyword so it has no default value.
                    ParameterDefaultValue.NONE
                }
            parameters.add(
                ParameterInfo(
                    name,
                    publicName,
                    defaultValue,
                    typeString,
                    modifiers,
                    tokenizer.fileLocation(),
                    index
                )
            )
            index++
        }
    }

    /**
     * Container for parsed information on a parameter. This is an intermediate step before a
     * [ParameterItem] is created, which is needed because
     * [TextTypeItemFactory.getMethodParameterType] requires a [MethodFingerprint] with the total
     * number of method parameters.
     */
    private inner class ParameterInfo(
        val name: String,
        val publicName: String?,
        val defaultValue: ParameterDefaultValue,
        val typeString: String,
        val modifiers: MutableModifierList,
        val location: FileLocation,
        val index: Int
    ) {
        /** Turn this [ParameterInfo] into a [ParameterItem] by parsing the [typeString]. */
        fun create(
            containingCallable: CallableItem,
            typeItemFactory: TextTypeItemFactory,
            methodFingerprint: MethodFingerprint
        ): ParameterItem {
            val type =
                typeItemFactory.getMethodParameterType(
                    typeString,
                    modifiers.annotations(),
                    methodFingerprint,
                    index,
                    modifiers.isVarArg()
                )
            synchronizeNullability(type, modifiers)

            val parameter =
                itemFactory.createParameterItem(
                    fileLocation = location,
                    modifiers = modifiers,
                    name = name,
                    publicNameProvider = { publicName },
                    containingCallable = containingCallable,
                    parameterIndex = index,
                    type = type,
                    defaultValueFactory = { defaultValue },
                )

            return parameter
        }
    }

    private fun parseDefault(tokenizer: Tokenizer): String {
        return buildString {
            while (true) {
                val token = tokenizer.requireToken()
                if (";" == token) {
                    break
                } else {
                    append(token)
                }
            }
        }
    }

    private fun parseThrows(
        tokenizer: Tokenizer,
        typeItemFactory: TextTypeItemFactory,
    ): List<ExceptionTypeItem> {
        var token = tokenizer.requireToken()
        val throwsList = buildList {
            var comma = true
            while (true) {
                when (token) {
                    ";" -> {
                        break
                    }
                    "," -> {
                        if (comma) {
                            throw ApiParseException("Expected exception, got ','", tokenizer)
                        }
                        comma = true
                    }
                    else -> {
                        if (!comma) {
                            throw ApiParseException("Expected ',' or ';' got $token", tokenizer)
                        }
                        comma = false
                        val exceptionType = typeItemFactory.getExceptionType(token)
                        add(exceptionType)
                    }
                }
                token = tokenizer.requireToken()
            }
        }

        return throwsList
    }

    /**
     * Scans the token stream from [tokenizer] for a type string, starting with the [startingToken]
     * and ensuring that the full type string is gathered, even when there are type-use annotations.
     *
     * After this method is called, `tokenizer.current` will point to the token after the type.
     *
     * Note: this **should not** be used when the token after the type could contain annotations,
     * such as when multiple types appear as consecutive tokens. (This happens in the `implements`
     * list of a class definition, e.g. `class Foo implements test.pkg.Bar test.pkg.@A Baz`.)
     *
     * To handle arrays with type-use annotations, this looks forward at the next token and includes
     * it if it contains an annotation. This is necessary to handle type strings like "Foo @A []".
     */
    private fun scanForTypeString(tokenizer: Tokenizer, startingToken: String): String {
        var prev = getAnnotationCompleteToken(tokenizer, startingToken)
        var type = prev
        var token = tokenizer.current
        // Look both at the last used token and the next one:
        // If the last token has annotations, the type string was broken up by annotations, and the
        // next token is also part of the type.
        // If the next token has annotations, this is an array type like "Foo @A []", so the next
        // token is part of the type.
        while (isIncompleteTypeToken(prev) || isIncompleteTypeToken(token)) {
            token = getAnnotationCompleteToken(tokenizer, token)
            type += " $token"
            prev = token
            token = tokenizer.current
        }
        return type
    }

    /**
     * Synchronize nullability annotations on the API item and [TypeNullability].
     *
     * If the type string uses a Kotlin nullability suffix, this adds an annotation representing
     * that nullability to [modifiers].
     *
     * @param typeItem the type of the API item.
     * @param modifiers the API item's modifiers.
     */
    private fun synchronizeNullability(typeItem: TypeItem, modifiers: MutableModifierList) {
        if (typeParser.kotlinStyleNulls) {
            // Add an annotation to the context item for the type's nullability if applicable.
            val annotationToAdd =
                // Treat varargs as non-null for consistency with the psi model.
                if (typeItem is ArrayTypeItem && typeItem.isVarargs) {
                    ANDROIDX_NONNULL
                } else {
                    val nullability = typeItem.modifiers.nullability
                    if (typeItem !is PrimitiveTypeItem && nullability == TypeNullability.NONNULL) {
                        ANDROIDX_NONNULL
                    } else if (nullability == TypeNullability.NULLABLE) {
                        ANDROIDX_NULLABLE
                    } else {
                        // No annotation to add, return.
                        return
                    }
                }
            modifiers.addAnnotation(codebase.createAnnotation("@$annotationToAdd"))
        }
    }

    /**
     * Determines whether the [type] is an incomplete type string broken up by annotations. This is
     * the case when there's an annotation that isn't contained within a parameter list (because
     * [Tokenizer.requireToken] handles not breaking in the middle of a parameter list).
     */
    private fun isIncompleteTypeToken(type: String): Boolean {
        val firstAnnotationIndex = type.indexOf('@')
        val paramStartIndex = type.indexOf('<')
        val lastAnnotationIndex = type.lastIndexOf('@')
        val paramEndIndex = type.lastIndexOf('>')
        return firstAnnotationIndex != -1 &&
            (paramStartIndex == -1 ||
                firstAnnotationIndex < paramStartIndex ||
                paramEndIndex == -1 ||
                paramEndIndex < lastAnnotationIndex)
    }

    /**
     * For Kotlin-style name/type ordering in signature files, the name is generally followed by a
     * colon (besides methods, where the colon comes after the parameter list). This method takes
     * the name [token] and removes the trailing colon, throwing an [ApiParseException] if one isn't
     * present (the [tokenizer] is only used for context for the error, if needed).
     */
    private fun parseNameWithColon(token: String, tokenizer: Tokenizer): String {
        if (!token.endsWith(':')) {
            throw ApiParseException("Expecting name ending with \":\" but found $token.", tokenizer)
        }
        return token.removeSuffix(":")
    }

    private fun qualifiedName(pkg: String, className: String): String {
        return "$pkg.$className"
    }

    private val stats
        get() =
            Stats(
                codebase.getPackages().allClasses().count(),
                typeParser.requests,
                typeParser.cacheSkip,
                typeParser.cacheHit,
                typeParser.cacheSize,
            )

    data class Stats(
        val totalClasses: Int,
        val typeCacheRequests: Int,
        val typeCacheSkip: Int,
        val typeCacheHit: Int,
        val typeCacheSize: Int,
    )
}