OpenHarmony-v6.0-Release/s

/*
 *
 * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
 *
 * This program is free software and is provided to you under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation, and any use by you of this program is subject to the terms
 * of such GNU licence.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, you can access it online at
 * http://www.gnu.org/licenses/gpl-2.0.html.
 *
 * SPDX-License-Identifier: GPL-2.0
 *
 */

/*
 * mali_kbase_kinstr_jm.c
 * Kernel driver public interface to job manager atom tracing
 */

#include "mali_kbase_kinstr_jm.h"
#include "mali_kbase_kinstr_jm_reader.h"

#include "mali_kbase.h"
#include "mali_kbase_linux.h"

#include <mali_kbase_jm_rb.h>

#include <asm/barrier.h>
#include <linux/anon_inodes.h>
#include <linux/circ_buf.h>
#include <linux/fs.h>
#include <linux/kref.h>
#include <linux/log2.h>
#include <linux/mutex.h>
#include <linux/rculist_bl.h>
#include <linux/poll.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/version.h>
#include <linux/wait.h>

#if KERNEL_VERSION(5, 1, 0) <= LINUX_VERSION_CODE
#include <linux/build_bug.h>
#else
// Stringify the expression if no message is given.
#define static_assert(e, ...) __static_assert(e, #__VA_ARGS__, #e)
#define __static_assert(e, msg, ...) _Static_assert(e, msg)
#endif

#if KERNEL_VERSION(4, 16, 0) >= LINUX_VERSION_CODE
typedef unsigned int (__poll_t);
#endif

#ifndef ENOTSUP
#define ENOTSUP EOPNOTSUPP
#endif

/* The module printing prefix */
#define PR_ "mali_kbase_kinstr_jm: "

/* Allows us to perform ASM goto for the tracing
 * https://www.kernel.org/doc/Documentation/static-keys.txt
 */
#if KERNEL_VERSION(4, 3, 0) <= LINUX_VERSION_CODE
DEFINE_STATIC_KEY_FALSE(basep_kinstr_jm_reader_static_key);
#else
struct static_key basep_kinstr_jm_reader_static_key = STATIC_KEY_INIT_FALSE;
#define static_branch_inc(key) static_key_slow_inc(key)
#define static_branch_dec(key) static_key_slow_dec(key)
#endif /* KERNEL_VERSION(4 ,3, 0) <= LINUX_VERSION_CODE */

#define KBASE_KINSTR_JM_VERSION 1

/**
 * struct kbase_kinstr_jm - The context for the kernel job manager atom tracing
 * @readers: a bitlocked list of opened readers. Readers are attached to the
 *           private data of a file descriptor that the user opens with the
 *           KBASE_IOCTL_KINSTR_JM_FD IO control call.
 * @refcount: reference count for the context. Any reader will have a link
 *            back to the context so that they can remove themselves from the
 *            list.
 *
 * This is opaque outside this compilation unit
 */
struct kbase_kinstr_jm {
    struct hlist_bl_head readers;
    struct kref refcount;
};

/**
 * struct kbase_kinstr_jm_atom_state_change - Represents an atom changing to a
 *                                            new state
 * @timestamp: Raw monotonic nanoseconds of the state change
 * @state:     The state that the atom has moved to
 * @atom:      The atom number that has changed state
 * @flags:     Flags associated with the state change. See
 *             KBASE_KINSTR_JM_ATOM_STATE_FLAG_* defines.
 * @reserved:  Reserved for future use.
 * @data:      Extra data for the state change. Active member depends on state.
 *
 * We can add new fields to the structure and old user code will gracefully
 * ignore the new fields.
 *
 * We can change the size of the structure and old user code will gracefully
 * skip over the new size via `struct kbase_kinstr_jm_fd_out->size`.
 *
 * If we remove fields, the version field in `struct
 * kbase_kinstr_jm_fd_out->version` will be incremented and old user code will
 * gracefully fail and tell the user that the kernel API is too new and has
 * backwards-incompatible changes. Note that one userspace can opt to handle
 * multiple kernel major versions of the structure.
 *
 * If we need to change the _meaning_ of one of the fields, i.e. the state
 * machine has had a incompatible change, we can keep the same members in the
 * structure and update the version as above. User code will no longer
 * recognise that it has the supported field and can gracefully explain to the
 * user that the kernel API is no longer supported.
 *
 * When making changes to this structure, make sure they are either:
 *  - additions to the end (for minor version bumps (i.e. only a size increase))
 *  such that the layout of existing fields doesn't change, or;
 *  - update the version reported to userspace so that it can fail explicitly.
 */
struct kbase_kinstr_jm_atom_state_change {
    u64 timestamp;
    s8 state; /* enum kbase_kinstr_jm_reader_atom_state */
    u8 atom;
    u8 flags;
    u8 reserved[1];
    /* Tagged union based on state. Ensure members are aligned correctly! */
    union {
        struct {
            u8 slot;
        } start;
        u8 padding[4];
    } data;
};
static_assert(((1 << 8 * sizeof(((struct kbase_kinstr_jm_atom_state_change *)0)->state)) - 1) >=
              KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT);

#define KBASE_KINSTR_JM_ATOM_STATE_FLAG_OVERFLOW BIT(0)

/**
 * struct reader_changes - The circular buffer of kernel atom state changes
 * @data:      The allocated buffer. This is allocated when the user requests
 *             the reader file descriptor. It is released when the user calls
 *             close() on the fd. When accessing this, lock the producer spin
 *             lock to prevent races on the allocated memory. The consume lock
 *             does not need to be held because newly-inserted data will always
 *             be outside the currenly-read range.
 * @producer:  The producing spinlock which allows us to push changes into the
 *             buffer at the same time as a user read occurring. This needs to
 *             be locked when saving/restoring the IRQ because we can receive an
 *             interrupt from the GPU when an atom completes. The CPU could have
 *             a task preempted that is holding this lock.
 * @consumer:  The consuming mutex which locks around the user read().
 *             Must be held when updating the tail of the circular buffer.
 * @head:      The head of the circular buffer. Can be used with Linux @c CIRC_
 *             helpers. The producer should lock and update this with an SMP
 *             store when a new change lands. The consumer can read with an
 *             SMP load. This allows the producer to safely insert new changes
 *             into the circular buffer.
 * @tail:      The tail of the circular buffer. Can be used with Linux @c CIRC_
 *             helpers. The producer should do a READ_ONCE load and the consumer
 *             should SMP store.
 * @size:      The number of changes that are allowed in @c data. Can be used
 *             with Linux @c CIRC_ helpers. Will always be a power of two. The
 *             producer lock should be held when updating this and stored with
 *             an SMP release memory barrier. This means that the consumer can
 *             do an SMP load.
 * @threshold: The number of changes above which threads polling on the reader
 *             file descriptor will be woken up.
 */
struct reader_changes {
    struct kbase_kinstr_jm_atom_state_change *data;
    spinlock_t producer;
    struct mutex consumer;
    u32 head;
    u32 tail;
    u32 size;
    u32 threshold;
};

/**
 * reader_changes_is_valid_size() - Determines if requested changes buffer size
 *                                  is valid.
 * @size: The requested memory size
 *
 * We have a constraint that the underlying physical buffer must be a
 * power of two so that we can use the efficient circular buffer helpers that
 * the kernel provides. It also needs to be representable within a u32.
 *
 * Return:
 * * true  - the size is valid
 * * false - the size is invalid
 */
static inline bool reader_changes_is_valid_size(const size_t size)
{
    typedef struct reader_changes changes_t;
    const size_t elem_size = sizeof(*((changes_t *)0)->data);
    const size_t size_size = sizeof(((changes_t *)0)->size);
    const size_t size_max = (1ull << (size_size * 8)) - 1;

    return is_power_of_2(size) &&            /* Is a power of two */
           ((size / elem_size) <= size_max); /* Small enough */
}

/**
 * reader_changes_init() - Initializes the reader changes and allocates the
 *                         changes buffer
 * @changes: The context pointer, must point to a zero-inited allocated reader
 *           changes structure. We may support allocating the structure in the
 *           future.
 * @size: The requested changes buffer size
 *
 * Return:
 * (0, U16_MAX] - the number of data elements allocated
 * -EINVAL - a pointer was invalid
 * -ENOTSUP - we do not support allocation of the context
 * -ERANGE - the requested memory size was invalid
 * -ENOMEM - could not allocate the memory
 * -EADDRINUSE - the buffer memory was already allocated
 */
static int reader_changes_init(struct reader_changes *const changes, const size_t size)
{
    BUILD_BUG_ON((PAGE_SIZE % sizeof(*changes->data)) != 0);

    if (!reader_changes_is_valid_size(size)) {
        pr_warn(PR_ "invalid size %zu\n", size);
        return -ERANGE;
    }

    changes->data = vmalloc(size);
    if (!changes->data) {
        return -ENOMEM;
    }

    spin_lock_init(&changes->producer);
    mutex_init(&changes->consumer);

    changes->size = size / sizeof(*changes->data);
    changes->threshold = min(((size_t)(changes->size)) / 0x4, ((size_t)(PAGE_SIZE)) / sizeof(*changes->data));

    return changes->size;
}

/**
 * reader_changes_term() - Cleans up a reader changes structure
 * @changes: The context to clean up
 *
 * Releases the allocated state changes memory
 */
static void reader_changes_term(struct reader_changes *const changes)
{
    struct kbase_kinstr_jm_atom_state_change *data = NULL;
    unsigned long irq;

    /*
     * Although changes->data is used on the consumer side, too, no active
     * consumer is possible by the time we clean up the reader changes, so
     * no need to take the consumer lock. However, we do need the producer
     * lock because the list removal can race with list traversal.
     */
    spin_lock_irqsave(&changes->producer, irq);
    swap(changes->data, data);
    spin_unlock_irqrestore(&changes->producer, irq);

    mutex_destroy(&changes->consumer);
    vfree(data);
}

/**
 * reader_changes_count_locked() - Retrieves the count of state changes from the
 * tail to the physical end of the buffer
 * @changes: The state changes context
 *
 * The consumer mutex must be held. Uses the CIRC_CNT_TO_END macro to
 * determine the count, so there may be more items. However, that's the maximum
 * number that can be read in one contiguous read.
 *
 * Return: the number of changes in the circular buffer until the end of the
 * allocation
 */
static u32 reader_changes_count_locked(struct reader_changes *const changes)
{
    u32 head;

    lockdep_assert_held_once(&changes->consumer);

    head = smp_load_acquire(&changes->head);

    return CIRC_CNT_TO_END(head, changes->tail, changes->size);
}

/**
 * reader_changes_count() - Retrieves the count of state changes from the
 * tail to the physical end of the buffer
 * @changes: The state changes context
 *
 * Return: the number of changes in the circular buffer until the end of the
 * allocation
 */
static u32 reader_changes_count(struct reader_changes *const changes)
{
    u32 ret;

    mutex_lock(&changes->consumer);
    ret = reader_changes_count_locked(changes);
    mutex_unlock(&changes->consumer);
    return ret;
}

/**
 * reader_changes_push() - Pushes a change into the reader circular buffer.
 * @changes:    The buffer to insert the change into
 * @change:     Kernel atom change to insert
 * @wait_queue: The queue to be kicked when changes should be read from
 *              userspace. Kicked when a threshold is reached or there is
 *              overflow.
 */
static void reader_changes_push(struct reader_changes *const changes,
                                const struct kbase_kinstr_jm_atom_state_change *const change,
                                wait_queue_head_t *const wait_queue)
{
    u32 head, tail, size, space;
    unsigned long irq;
    struct kbase_kinstr_jm_atom_state_change *data;

    spin_lock_irqsave(&changes->producer, irq);

    /* We may be called for a reader_changes that's awaiting cleanup. */
    data = changes->data;
    if (!data) {
        goto unlock;
    }

    size = changes->size;
    head = changes->head;
    tail = smp_load_acquire(&changes->tail);

    space = CIRC_SPACE(head, tail, size);
    if (space >= 1) {
        data[head] = *change;
        if (space == 1) {
            data[head].flags |= KBASE_KINSTR_JM_ATOM_STATE_FLAG_OVERFLOW;
            pr_warn(PR_ "overflow of circular buffer\n");
        }
        smp_store_release(&changes->head, (head + 1) & (size - 1));
    }

    /* Wake for either overflow or over-threshold cases. */
    if (CIRC_CNT(head + 1, tail, size) >= changes->threshold) {
        wake_up_interruptible(wait_queue);
    }

unlock:
    spin_unlock_irqrestore(&changes->producer, irq);
}

/**
 * struct reader - Allows the kernel state changes to be read by user space.
 * @node: The node in the @c readers locked list
 * @rcu_head: storage for the RCU callback to free this reader (see kfree_rcu)
 * @changes: The circular buffer of user changes
 * @wait_queue: A wait queue for poll
 * @context: a pointer to the parent context that created this reader. Can be
 *           used to remove the reader from the list of readers. Reference
 *           counted.
 *
 * The reader is a circular buffer in kernel space. State changes are pushed
 * into the buffer. The flow from user space is:
 *
 *   * Request file descriptor with KBASE_IOCTL_KINSTR_JM_FD. This will
 *     allocate the kernel side circular buffer with a size specified in the
 *     ioctl argument.
 *   * The user will then poll the file descriptor for data
 *   * Upon receiving POLLIN, perform a read() on the file descriptor to get
 *     the data out.
 *   * The buffer memory will be freed when the file descriptor is closed
 */
struct reader {
    struct hlist_bl_node node;
    struct rcu_head rcu_head;
    struct reader_changes changes;
    wait_queue_head_t wait_queue;
    struct kbase_kinstr_jm *context;
};

static struct kbase_kinstr_jm *kbase_kinstr_jm_ref_get(struct kbase_kinstr_jm *const ctx);
static void kbase_kinstr_jm_ref_put(struct kbase_kinstr_jm *const ctx);
static int kbase_kinstr_jm_readers_add(struct kbase_kinstr_jm *const ctx, struct reader *const reader);
static void kbase_kinstr_jm_readers_del(struct kbase_kinstr_jm *const ctx, struct reader *const reader);

/**
 * reader_term() - Terminate a instrumentation job manager reader context.
 * @reader: Pointer to context to be terminated.
 */
static void reader_term(struct reader *const reader)
{
    if (!reader) {
        return;
    }

    kbase_kinstr_jm_readers_del(reader->context, reader);
    reader_changes_term(&reader->changes);
    kbase_kinstr_jm_ref_put(reader->context);

    kfree_rcu(reader, rcu_head);
}

/**
 * reader_init() - Initialise a instrumentation job manager reader context.
 * @out_reader:  Non-NULL pointer to where the pointer to the created context
 *               will be stored on success.
 * @ctx:         the pointer to the parent context. Reference count will be
 *               increased if initialization is successful
 * @num_changes: The number of changes to allocate a buffer for
 *
 * Return: 0 on success, else error code.
 */
static int reader_init(struct reader **const out_reader, struct kbase_kinstr_jm *const ctx, size_t const num_changes)
{
    struct reader *reader = NULL;
    const size_t change_size = sizeof(struct kbase_kinstr_jm_atom_state_change);
    int status;

    if (!out_reader || !ctx || !num_changes) {
        return -EINVAL;
    }

    reader = kzalloc(sizeof(*reader), GFP_KERNEL);
    if (!reader) {
        return -ENOMEM;
    }

    INIT_HLIST_BL_NODE(&reader->node);
    init_waitqueue_head(&reader->wait_queue);

    reader->context = kbase_kinstr_jm_ref_get(ctx);

    status = reader_changes_init(&reader->changes, num_changes * change_size);
    if (status < 0) {
        goto fail;
    }

    status = kbase_kinstr_jm_readers_add(ctx, reader);
    if (status < 0) {
        goto fail;
    }

    *out_reader = reader;

    return 0;

fail:
    kbase_kinstr_jm_ref_put(reader->context);
    kfree(reader);
    return status;
}

/**
 * reader_release() - Invoked when the reader file descriptor is released
 * @node: The inode that the file descriptor that the file corresponds to. In
 *        our case our reader file descriptor is backed by an anonymous node so
 *        not much is in this.
 * @file: the file data. Our reader context is held in the private data
 * Return: zero on success
 */
static int reader_release(struct inode *const node, struct file *const file)
{
    struct reader *const reader = file->private_data;

    reader_term(reader);
    file->private_data = NULL;

    return 0;
}

/**
 * reader_changes_copy_to_user() - Copy any changes from a changes structure to
 * the user-provided buffer.
 * @changes: The changes structure from which to copy.
 * @buffer: The user buffer to copy the data to.
 * @buffer_size: The number of bytes in the buffer.
 * Return: The number of bytes copied or negative errno on failure.
 */
static ssize_t reader_changes_copy_to_user(struct reader_changes *const changes, char __user *buffer,
                                           size_t buffer_size)
{
    ssize_t ret = 0;
    struct kbase_kinstr_jm_atom_state_change const *src_buf = READ_ONCE(changes->data);
    size_t const entry_size = sizeof(*src_buf);
    size_t changes_tail, changes_count, read_size;

    /* Needed for the quick buffer capacity calculation below.
     * Note that we can't use is_power_of_2() since old compilers don't
     * understand it's a constant expression.
     */
#define is_power_of_two(x) ((x) && !((x) & ((x)-1)))
    static_assert(is_power_of_two(sizeof(struct kbase_kinstr_jm_atom_state_change)));
#undef is_power_of_two

    lockdep_assert_held_once(&changes->consumer);

    /* Read continuously until either:
     * - we've filled the output buffer, or
     * - there are no changes when we check.
     *
     * If more changes arrive while we're copying to the user, we can copy
     * those as well, space permitting.
     */
    do {
        changes_tail = changes->tail;
        changes_count = reader_changes_count_locked(changes);
        read_size = min(changes_count * entry_size, buffer_size & ~(entry_size - 1));

        if (!read_size) {
            break;
        }

        if (copy_to_user(buffer, &(src_buf[changes_tail]), read_size)) {
            return -EFAULT;
        }

        buffer += read_size;
        buffer_size -= read_size;
        ret += read_size;
        changes_tail = (changes_tail + read_size / entry_size) & (changes->size - 1);
        smp_store_release(&changes->tail, changes_tail);
    } while (read_size);

    return ret;
}

/**
 * reader_read() - Handles a read call on the reader file descriptor
 *
 * @filp: The file that the read was performed on
 * @buffer: The destination buffer
 * @buffer_size: The maximum number of bytes to read
 * @offset: The offset into the 'file' to read from.
 *
 * Note the destination buffer needs to be fully mapped in userspace or the read
 * will fault.
 *
 * Return:
 * * The number of bytes read or:
 * * -EBADF - the file descriptor did not have an attached reader
 * * -EFAULT - memory access fault
 * * -EAGAIN - if the file is set to nonblocking reads with O_NONBLOCK and there
 *             is no data available
 *
 * Note: The number of bytes read will always be a multiple of the size of an
 * entry.
 */
static ssize_t reader_read(struct file *const filp, char __user *const buffer, size_t const buffer_size,
                           loff_t *const offset)
{
    struct reader *const reader = filp->private_data;
    struct reader_changes *changes;
    ssize_t ret;

    if (!reader) {
        return -EBADF;
    }

    if (buffer_size < sizeof(struct kbase_kinstr_jm_atom_state_change)) {
        return -ENOBUFS;
    }

#if KERNEL_VERSION(5, 0, 0) <= LINUX_VERSION_CODE
    if (!access_ok(buffer, buffer_size)) {
        return -EIO;
    }
#else
    if (!access_ok(VERIFY_WRITE, buffer, buffer_size)) {
        return -EIO;
    }
#endif

    changes = &reader->changes;

    mutex_lock(&changes->consumer);
    if (!reader_changes_count_locked(changes)) {
        if (filp->f_flags & O_NONBLOCK) {
            ret = -EAGAIN;
            goto exit;
        }

        if (wait_event_interruptible(reader->wait_queue, !!reader_changes_count_locked(changes))) {
            ret = -EINTR;
            goto exit;
        }
    }

    ret = reader_changes_copy_to_user(changes, buffer, buffer_size);

exit:
    mutex_unlock(&changes->consumer);
    return ret;
}

/**
 * reader_poll() - Handles a poll call on the reader file descriptor
 * @file: The file that the poll was performed on
 * @wait: The poll table
 *
 * The results of the poll will be unreliable if there is no mapped memory as
 * there is no circular buffer to push atom state changes into.
 *
 * Return:
 * * 0 - no data ready
 * * POLLIN - state changes have been buffered
 * * -EBADF - the file descriptor did not have an attached reader
 * * -EINVAL - the IO control arguments were invalid
 */
static __poll_t reader_poll(struct file *const file, struct poll_table_struct *const wait)
{
    struct reader *reader;
    struct reader_changes *changes;

    if (unlikely(!file || !wait)) {
        return -EINVAL;
    }

    reader = file->private_data;
    if (unlikely(!reader)) {
        return -EBADF;
    }

    changes = &reader->changes;

    if (reader_changes_count(changes) >= changes->threshold) {
        return POLLIN;
    }

    poll_wait(file, &reader->wait_queue, wait);

    return (reader_changes_count(changes) > 0) ? POLLIN : 0;
}

/* The file operations virtual function table */
static const struct file_operations file_operations = {
    .owner = THIS_MODULE, .llseek = no_llseek, .read = reader_read, .poll = reader_poll, .release = reader_release};

/* The maximum amount of readers that can be created on a context. */
static const size_t kbase_kinstr_jm_readers_max = 16;

/**
 * kbasep_kinstr_jm_release() - Invoked when the reference count is dropped
 * @ref: the context reference count
 */
static void kbase_kinstr_jm_release(struct kref *const ref)
{
    struct kbase_kinstr_jm *const ctx = container_of(ref, struct kbase_kinstr_jm, refcount);

    kfree(ctx);
}

/**
 * kbase_kinstr_jm_ref_get() - Reference counts the instrumentation context
 * @ctx: the context to reference count
 * Return: the reference counted context
 */
static struct kbase_kinstr_jm *kbase_kinstr_jm_ref_get(struct kbase_kinstr_jm *const ctx)
{
    if (likely(ctx)) {
        kref_get(&ctx->refcount);
    }
    return ctx;
}

/**
 * kbase_kinstr_jm_ref_put() - Dereferences the instrumentation context
 * @ctx: the context to lower the reference count on
 */
static void kbase_kinstr_jm_ref_put(struct kbase_kinstr_jm *const ctx)
{
    if (likely(ctx)) {
        kref_put(&ctx->refcount, kbase_kinstr_jm_release);
    }
}

/**
 * kbase_kinstr_jm_readers_add() - Adds a reader to the list of readers
 * @ctx: the instrumentation context
 * @reader: the reader to add
 *
 * Return:
 * 0 - success
 * -ENOMEM - too many readers already added.
 */
static int kbase_kinstr_jm_readers_add(struct kbase_kinstr_jm *const ctx, struct reader *const reader)
{
    struct hlist_bl_head *const readers = &ctx->readers;
    struct hlist_bl_node *node;
    struct reader *temp;
    size_t count = 0;

    hlist_bl_lock(readers);

    hlist_bl_for_each_entry_rcu(temp, node, readers, node)++ count;

    if (kbase_kinstr_jm_readers_max < count) {
        hlist_bl_unlock(readers);
        return -ENOMEM;
    }

    hlist_bl_add_head_rcu(&reader->node, readers);

    hlist_bl_unlock(readers);

    static_branch_inc(&basep_kinstr_jm_reader_static_key);

    return 0;
}

/**
 * readers_del() - Deletes a reader from the list of readers
 * @ctx: the instrumentation context
 * @reader: the reader to delete
 */
static void kbase_kinstr_jm_readers_del(struct kbase_kinstr_jm *const ctx, struct reader *const reader)
{
    struct hlist_bl_head *const readers = &ctx->readers;

    hlist_bl_lock(readers);
    hlist_bl_del_rcu(&reader->node);
    hlist_bl_unlock(readers);

    static_branch_dec(&basep_kinstr_jm_reader_static_key);
}

int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, union kbase_kinstr_jm_fd *jm_fd_arg)
{
    struct kbase_kinstr_jm_fd_in const *in;
    struct reader *reader;
    size_t const change_size = sizeof(struct kbase_kinstr_jm_atom_state_change);
    int status;
    int fd;
    int i;

    if (!ctx || !jm_fd_arg) {
        return -EINVAL;
    }

    in = &jm_fd_arg->in;

    if (!is_power_of_2(in->count)) {
        return -EINVAL;
    }

    for (i = 0; i < sizeof(in->padding); ++i) {
        if (in->padding[i]) {
            return -EINVAL;
        }
    }

    status = reader_init(&reader, ctx, in->count);
    if (status < 0) {
        return status;
    }

    jm_fd_arg->out.version = KBASE_KINSTR_JM_VERSION;
    jm_fd_arg->out.size = change_size;
    memset(&jm_fd_arg->out.padding, 0, sizeof(jm_fd_arg->out.padding));

    fd = anon_inode_getfd("[mali_kinstr_jm]", &file_operations, reader, O_CLOEXEC);
    if (fd < 0) {
        reader_term(reader);
    }

    return fd;
}

int kbase_kinstr_jm_init(struct kbase_kinstr_jm **const out_ctx)
{
    struct kbase_kinstr_jm *ctx = NULL;

    if (!out_ctx) {
        return -EINVAL;
    }

    ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
    if (!ctx) {
        return -ENOMEM;
    }

    INIT_HLIST_BL_HEAD(&ctx->readers);
    kref_init(&ctx->refcount);

    *out_ctx = ctx;

    return 0;
}

void kbase_kinstr_jm_term(struct kbase_kinstr_jm *const ctx)
{
    kbase_kinstr_jm_ref_put(ctx);
}

void kbasep_kinstr_jm_atom_state(struct kbase_jd_atom *const katom, const enum kbase_kinstr_jm_reader_atom_state state)
{
    struct kbase_context *const kctx = katom->kctx;
    struct kbase_kinstr_jm *const ctx = kctx->kinstr_jm;
    const u8 id = kbase_jd_atom_id(kctx, katom);
    struct kbase_kinstr_jm_atom_state_change change = {.timestamp = ktime_get_raw_ns(), .atom = id, .state = state};
    struct reader *reader;
    struct hlist_bl_node *node;

    WARN(KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT < state || 0 > state, PR_ "unsupported katom (%u) state (%i)", id,
         state);

    switch (state) {
        case KBASE_KINSTR_JM_READER_ATOM_STATE_START:
            change.data.start.slot = katom->jobslot;
            break;
        default:
            break;
    }

    rcu_read_lock();
    hlist_bl_for_each_entry_rcu(reader, node, &ctx->readers, node)
        reader_changes_push(&reader->changes, &change, &reader->wait_queue);
    rcu_read_unlock();
}

KBASE_EXPORT_TEST_API(kbasep_kinstr_jm_atom_state);

void kbasep_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const katom)
{
    struct kbase_context *const kctx = katom->kctx;
    struct kbase_device *const kbdev = kctx->kbdev;
    const int slot = katom->slot_nr;
    struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0);

    BUILD_BUG_ON(SLOT_RB_SIZE != 0x2);

    lockdep_assert_held(&kbdev->hwaccess_lock);

    if (WARN_ON(slot < 0 || slot >= GPU_MAX_JOB_SLOTS)) {
        return;
    }
    if (WARN_ON(!submitted)) {
        return;
    }

    if (submitted == katom) {
        kbase_kinstr_jm_atom_state_start(katom);
    }
}

void kbasep_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const katom)
{
    struct kbase_context *const kctx = katom->kctx;
    struct kbase_device *const kbdev = kctx->kbdev;
    const int slot = katom->slot_nr;
    struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0);
    struct kbase_jd_atom *const queued = kbase_gpu_inspect(kbdev, slot, 1);

    BUILD_BUG_ON(SLOT_RB_SIZE != 0x2);

    lockdep_assert_held(&kbdev->hwaccess_lock);

    if (WARN_ON(slot < 0 || slot >= GPU_MAX_JOB_SLOTS)) {
        return;
    }
    if (WARN_ON(!submitted)) {
        return;
    }
    if (WARN_ON((submitted != katom) && (queued != katom))) {
        return;
    }

    if (queued == katom) {
        return;
    }

    if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) {
        kbase_kinstr_jm_atom_state_stop(katom);
    }
    if (queued && queued->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) {
        kbase_kinstr_jm_atom_state_start(queued);
    }
}