1 /* Copyright (c) 2015-2017 The Linux Foundation. All rights reserved.
171  * The cache is made up of one or more clusters, each cluster has its own PMU.
172  * Each cluster is associated with one or more CPUs.
175  * Events can be envisioned as a 2-dimensional array. Each column represents
193 	/* The CPU that is used for collecting events on this cluster */
195 	/* All the CPUs associated with this cluster */
216 	return *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu);  in get_cluster_pmu()
293 static void cluster_pmu_set_resr(struct cluster_pmu *cluster,  in cluster_pmu_set_resr()  argument
304 	spin_lock_irqsave(&cluster->pmu_lock, flags);  in cluster_pmu_set_resr()
312 	spin_unlock_irqrestore(&cluster->pmu_lock, flags);  in cluster_pmu_set_resr()
318  * all CPUS, subunits and ID independent events in this cluster.
349 	struct hw_perf_event *hwc = &event->hw;  in l2_cache_event_update()
351 	u32 idx = hwc->idx;  in l2_cache_event_update()
354 		prev = local64_read(&hwc->prev_count);  in l2_cache_event_update()
356 	} while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);  in l2_cache_event_update()
359 	 * The cycle counter is 64-bit, but all other counters are  in l2_cache_event_update()
360 	 * 32-bit, and we must handle 32-bit overflow explicitly.  in l2_cache_event_update()
362 	delta = now - prev;  in l2_cache_event_update()
366 	local64_add(delta, &event->count);  in l2_cache_event_update()
369 static void l2_cache_cluster_set_period(struct cluster_pmu *cluster,  in l2_cache_cluster_set_period()  argument
372 	u32 idx = hwc->idx;  in l2_cache_cluster_set_period()
385 	local64_set(&hwc->prev_count, new);  in l2_cache_cluster_set_period()
389 static int l2_cache_get_event_idx(struct cluster_pmu *cluster,  in l2_cache_get_event_idx()  argument
392 	struct hw_perf_event *hwc = &event->hw;  in l2_cache_get_event_idx()
394 	int num_ctrs = cluster->l2cache_pmu->num_counters - 1;  in l2_cache_get_event_idx()
397 	if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) {  in l2_cache_get_event_idx()
398 		if (test_and_set_bit(l2_cycle_ctr_idx, cluster->used_counters))  in l2_cache_get_event_idx()
399 			return -EAGAIN;  in l2_cache_get_event_idx()
404 	idx = find_first_zero_bit(cluster->used_counters, num_ctrs);  in l2_cache_get_event_idx()
407 		return -EAGAIN;  in l2_cache_get_event_idx()
414 	group = L2_EVT_GROUP(hwc->config_base);  in l2_cache_get_event_idx()
415 	if (test_bit(group, cluster->used_groups))  in l2_cache_get_event_idx()
416 		return -EAGAIN;  in l2_cache_get_event_idx()
418 	set_bit(idx, cluster->used_counters);  in l2_cache_get_event_idx()
419 	set_bit(group, cluster->used_groups);  in l2_cache_get_event_idx()
424 static void l2_cache_clear_event_idx(struct cluster_pmu *cluster,  in l2_cache_clear_event_idx()  argument
427 	struct hw_perf_event *hwc = &event->hw;  in l2_cache_clear_event_idx()
428 	int idx = hwc->idx;  in l2_cache_clear_event_idx()
430 	clear_bit(idx, cluster->used_counters);  in l2_cache_clear_event_idx()
431 	if (hwc->config_base != L2CYCLE_CTR_RAW_CODE)  in l2_cache_clear_event_idx()
432 		clear_bit(L2_EVT_GROUP(hwc->config_base), cluster->used_groups);  in l2_cache_clear_event_idx()
437 	struct cluster_pmu *cluster = data;  in l2_cache_handle_irq()  local
438 	int num_counters = cluster->l2cache_pmu->num_counters;  in l2_cache_handle_irq()
446 	for_each_set_bit(idx, cluster->used_counters, num_counters) {  in l2_cache_handle_irq()
447 		struct perf_event *event = cluster->events[idx];  in l2_cache_handle_irq()
457 		hwc = &event->hw;  in l2_cache_handle_irq()
459 		l2_cache_cluster_set_period(cluster, hwc);  in l2_cache_handle_irq()
474 	 * physical PMUs (per cluster), because we do not support per-task mode  in l2_cache_pmu_enable()
490 	struct hw_perf_event *hwc = &event->hw;  in l2_cache_event_init()
491 	struct cluster_pmu *cluster;  in l2_cache_event_init()  local
495 	if (event->attr.type != event->pmu->type)  in l2_cache_event_init()
496 		return -ENOENT;  in l2_cache_event_init()
498 	l2cache_pmu = to_l2cache_pmu(event->pmu);  in l2_cache_event_init()
500 	if (hwc->sample_period) {  in l2_cache_event_init()
501 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,  in l2_cache_event_init()
503 		return -EOPNOTSUPP;  in l2_cache_event_init()
506 	if (event->cpu < 0) {  in l2_cache_event_init()
507 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,  in l2_cache_event_init()
508 				    "Per-task mode not supported\n");  in l2_cache_event_init()
509 		return -EOPNOTSUPP;  in l2_cache_event_init()
513 	if (event->attr.exclude_user || event->attr.exclude_kernel ||  in l2_cache_event_init()
514 	    event->attr.exclude_hv || event->attr.exclude_idle) {  in l2_cache_event_init()
515 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,  in l2_cache_event_init()
517 		return -EOPNOTSUPP;  in l2_cache_event_init()
520 	if (((L2_EVT_GROUP(event->attr.config) > L2_EVT_GROUP_MAX) ||  in l2_cache_event_init()
521 	     ((event->attr.config & ~L2_EVT_MASK) != 0)) &&  in l2_cache_event_init()
522 	    (event->attr.config != L2CYCLE_CTR_RAW_CODE)) {  in l2_cache_event_init()
523 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,  in l2_cache_event_init()
525 				    event->attr.config);  in l2_cache_event_init()
526 		return -EINVAL;  in l2_cache_event_init()
530 	if (event->group_leader->pmu != event->pmu &&  in l2_cache_event_init()
531 	    !is_software_event(event->group_leader)) {  in l2_cache_event_init()
532 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,  in l2_cache_event_init()
534 		return -EINVAL;  in l2_cache_event_init()
537 	for_each_sibling_event(sibling, event->group_leader) {  in l2_cache_event_init()
538 		if (sibling->pmu != event->pmu &&  in l2_cache_event_init()
540 			dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,  in l2_cache_event_init()
542 			return -EINVAL;  in l2_cache_event_init()
546 	cluster = get_cluster_pmu(l2cache_pmu, event->cpu);  in l2_cache_event_init()
547 	if (!cluster) {  in l2_cache_event_init()
549 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,  in l2_cache_event_init()
550 			"CPU%d not associated with L2 cluster\n", event->cpu);  in l2_cache_event_init()
551 		return -EINVAL;  in l2_cache_event_init()
555 	if ((event->group_leader != event) &&  in l2_cache_event_init()
556 	    (cluster->on_cpu != event->group_leader->cpu)) {  in l2_cache_event_init()
557 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,  in l2_cache_event_init()
559 			 event->cpu, event->group_leader->cpu);  in l2_cache_event_init()
560 		return -EINVAL;  in l2_cache_event_init()
563 	if ((event != event->group_leader) &&  in l2_cache_event_init()
564 	    !is_software_event(event->group_leader) &&  in l2_cache_event_init()
565 	    (L2_EVT_GROUP(event->group_leader->attr.config) ==  in l2_cache_event_init()
566 	     L2_EVT_GROUP(event->attr.config))) {  in l2_cache_event_init()
567 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,  in l2_cache_event_init()
569 		       event->group_leader->attr.config,  in l2_cache_event_init()
570 		       event->attr.config);  in l2_cache_event_init()
571 		return -EINVAL;  in l2_cache_event_init()
574 	for_each_sibling_event(sibling, event->group_leader) {  in l2_cache_event_init()
577 		    (L2_EVT_GROUP(sibling->attr.config) ==  in l2_cache_event_init()
578 		     L2_EVT_GROUP(event->attr.config))) {  in l2_cache_event_init()
579 			dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,  in l2_cache_event_init()
581 					    sibling->attr.config,  in l2_cache_event_init()
582 					    event->attr.config);  in l2_cache_event_init()
583 			return -EINVAL;  in l2_cache_event_init()
587 	hwc->idx = -1;  in l2_cache_event_init()
588 	hwc->config_base = event->attr.config;  in l2_cache_event_init()
594 	event->cpu = cluster->on_cpu;  in l2_cache_event_init()
601 	struct cluster_pmu *cluster;  in l2_cache_event_start()  local
602 	struct hw_perf_event *hwc = &event->hw;  in l2_cache_event_start()
603 	int idx = hwc->idx;  in l2_cache_event_start()
607 	hwc->state = 0;  in l2_cache_event_start()
609 	cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);  in l2_cache_event_start()
611 	l2_cache_cluster_set_period(cluster, hwc);  in l2_cache_event_start()
613 	if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) {  in l2_cache_event_start()
616 		config = hwc->config_base;  in l2_cache_event_start()
622 		cluster_pmu_set_resr(cluster, event_group, event_cc);  in l2_cache_event_start()
632 	struct hw_perf_event *hwc = &event->hw;  in l2_cache_event_stop()
633 	int idx = hwc->idx;  in l2_cache_event_stop()
635 	if (hwc->state & PERF_HES_STOPPED)  in l2_cache_event_stop()
643 	hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;  in l2_cache_event_stop()
648 	struct hw_perf_event *hwc = &event->hw;  in l2_cache_event_add()
651 	struct cluster_pmu *cluster;  in l2_cache_event_add()  local
653 	cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);  in l2_cache_event_add()
655 	idx = l2_cache_get_event_idx(cluster, event);  in l2_cache_event_add()
659 	hwc->idx = idx;  in l2_cache_event_add()
660 	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;  in l2_cache_event_add()
661 	cluster->events[idx] = event;  in l2_cache_event_add()
662 	local64_set(&hwc->prev_count, 0);  in l2_cache_event_add()
675 	struct hw_perf_event *hwc = &event->hw;  in l2_cache_event_del()
676 	struct cluster_pmu *cluster;  in l2_cache_event_del()  local
677 	int idx = hwc->idx;  in l2_cache_event_del()
679 	cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);  in l2_cache_event_del()
682 	cluster->events[idx] = NULL;  in l2_cache_event_del()
683 	l2_cache_clear_event_idx(cluster, event);  in l2_cache_event_del()
699 	return cpumap_print_to_pagebuf(true, buf, &l2cache_pmu->cpumask);  in l2_cache_pmu_cpumask_show()
715 PMU_FORMAT_ATTR(l2_code,   "config:4-11");
716 PMU_FORMAT_ATTR(l2_group,  "config:0-3");
717 PMU_FORMAT_ATTR(event,     "config:0-11");
737 	return sprintf(page, "event=0x%02llx\n", pmu_attr->id);  in l2cache_pmu_event_show()
748 	L2CACHE_EVENT_ATTR(dcache-ops, L2_EVENT_DCACHE_OPS),
749 	L2CACHE_EVENT_ATTR(icache-ops, L2_EVENT_ICACHE_OPS),
752 	L2CACHE_EVENT_ATTR(total-reads, L2_EVENT_TOTAL_READS),
753 	L2CACHE_EVENT_ATTR(total-writes, L2_EVENT_TOTAL_WRITES),
754 	L2CACHE_EVENT_ATTR(total-requests, L2_EVENT_TOTAL_REQUESTS),
800 	struct cluster_pmu *cluster = NULL;  in l2_cache_associate_cpu_with_cluster()  local
804 	 * single-threaded cores, and MPIDR[aff2] for multi-threaded  in l2_cache_associate_cpu_with_cluster()
813 	list_for_each_entry(cluster, &l2cache_pmu->clusters, next) {  in l2_cache_associate_cpu_with_cluster()
814 		if (cluster->cluster_id != cpu_cluster_id)  in l2_cache_associate_cpu_with_cluster()
817 		dev_info(&l2cache_pmu->pdev->dev,  in l2_cache_associate_cpu_with_cluster()
818 			 "CPU%d associated with cluster %d\n", cpu,  in l2_cache_associate_cpu_with_cluster()
819 			 cluster->cluster_id);  in l2_cache_associate_cpu_with_cluster()
820 		cpumask_set_cpu(cpu, &cluster->cluster_cpus);  in l2_cache_associate_cpu_with_cluster()
821 		*per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu) = cluster;  in l2_cache_associate_cpu_with_cluster()
825 	return cluster;  in l2_cache_associate_cpu_with_cluster()
830 	struct cluster_pmu *cluster;  in l2cache_pmu_online_cpu()  local
834 	cluster = get_cluster_pmu(l2cache_pmu, cpu);  in l2cache_pmu_online_cpu()
835 	if (!cluster) {  in l2cache_pmu_online_cpu()
837 		cluster = l2_cache_associate_cpu_with_cluster(l2cache_pmu, cpu);  in l2cache_pmu_online_cpu()
838 		if (!cluster) {  in l2cache_pmu_online_cpu()
839 			/* Only if broken firmware doesn't list every cluster */  in l2cache_pmu_online_cpu()
840 			WARN_ONCE(1, "No L2 cache cluster for CPU%d\n", cpu);  in l2cache_pmu_online_cpu()
845 	/* If another CPU is managing this cluster, we're done */  in l2cache_pmu_online_cpu()
846 	if (cluster->on_cpu != -1)  in l2cache_pmu_online_cpu()
850 	 * All CPUs on this cluster were down, use this one.  in l2cache_pmu_online_cpu()
853 	cluster->on_cpu = cpu;  in l2cache_pmu_online_cpu()
854 	cpumask_set_cpu(cpu, &l2cache_pmu->cpumask);  in l2cache_pmu_online_cpu()
857 	WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(cpu)));  in l2cache_pmu_online_cpu()
858 	enable_irq(cluster->irq);  in l2cache_pmu_online_cpu()
865 	struct cluster_pmu *cluster;  in l2cache_pmu_offline_cpu()  local
871 	cluster = get_cluster_pmu(l2cache_pmu, cpu);  in l2cache_pmu_offline_cpu()
872 	if (!cluster)  in l2cache_pmu_offline_cpu()
875 	/* If this CPU is not managing the cluster, we're done */  in l2cache_pmu_offline_cpu()
876 	if (cluster->on_cpu != cpu)  in l2cache_pmu_offline_cpu()
879 	/* Give up ownership of cluster */  in l2cache_pmu_offline_cpu()
880 	cpumask_clear_cpu(cpu, &l2cache_pmu->cpumask);  in l2cache_pmu_offline_cpu()
881 	cluster->on_cpu = -1;  in l2cache_pmu_offline_cpu()
883 	/* Any other CPU for this cluster which is still online */  in l2cache_pmu_offline_cpu()
884 	cpumask_and(&cluster_online_cpus, &cluster->cluster_cpus,  in l2cache_pmu_offline_cpu()
888 		disable_irq(cluster->irq);  in l2cache_pmu_offline_cpu()
892 	perf_pmu_migrate_context(&l2cache_pmu->pmu, cpu, target);  in l2cache_pmu_offline_cpu()
893 	cluster->on_cpu = target;  in l2cache_pmu_offline_cpu()
894 	cpumask_set_cpu(target, &l2cache_pmu->cpumask);  in l2cache_pmu_offline_cpu()
895 	WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(target)));  in l2cache_pmu_offline_cpu()
902 	struct platform_device *pdev = to_platform_device(dev->parent);  in l2_cache_pmu_probe_cluster()
905 	struct cluster_pmu *cluster;  in l2_cache_pmu_probe_cluster()  local
912 		return -ENODEV;  in l2_cache_pmu_probe_cluster()
914 	if (kstrtoul(device->pnp.unique_id, 10, &fw_cluster_id) < 0) {  in l2_cache_pmu_probe_cluster()
915 		dev_err(&pdev->dev, "unable to read ACPI uid\n");  in l2_cache_pmu_probe_cluster()
916 		return -ENODEV;  in l2_cache_pmu_probe_cluster()
919 	cluster = devm_kzalloc(&pdev->dev, sizeof(*cluster), GFP_KERNEL);  in l2_cache_pmu_probe_cluster()
920 	if (!cluster)  in l2_cache_pmu_probe_cluster()
921 		return -ENOMEM;  in l2_cache_pmu_probe_cluster()
923 	INIT_LIST_HEAD(&cluster->next);  in l2_cache_pmu_probe_cluster()
924 	list_add(&cluster->next, &l2cache_pmu->clusters);  in l2_cache_pmu_probe_cluster()
925 	cluster->cluster_id = fw_cluster_id;  in l2_cache_pmu_probe_cluster()
929 		dev_err(&pdev->dev,  in l2_cache_pmu_probe_cluster()
930 			"Failed to get valid irq for cluster %ld\n",  in l2_cache_pmu_probe_cluster()
935 	cluster->irq = irq;  in l2_cache_pmu_probe_cluster()
937 	cluster->l2cache_pmu = l2cache_pmu;  in l2_cache_pmu_probe_cluster()
938 	cluster->on_cpu = -1;  in l2_cache_pmu_probe_cluster()
940 	err = devm_request_irq(&pdev->dev, irq, l2_cache_handle_irq,  in l2_cache_pmu_probe_cluster()
942 			       "l2-cache-pmu", cluster);  in l2_cache_pmu_probe_cluster()
944 		dev_err(&pdev->dev,  in l2_cache_pmu_probe_cluster()
949 	dev_info(&pdev->dev,  in l2_cache_pmu_probe_cluster()
950 		"Registered L2 cache PMU cluster %ld\n", fw_cluster_id);  in l2_cache_pmu_probe_cluster()
952 	spin_lock_init(&cluster->pmu_lock);  in l2_cache_pmu_probe_cluster()
954 	l2cache_pmu->num_pmus++;  in l2_cache_pmu_probe_cluster()
965 		devm_kzalloc(&pdev->dev, sizeof(*l2cache_pmu), GFP_KERNEL);  in l2_cache_pmu_probe()
967 		return -ENOMEM;  in l2_cache_pmu_probe()
969 	INIT_LIST_HEAD(&l2cache_pmu->clusters);  in l2_cache_pmu_probe()
972 	l2cache_pmu->pmu = (struct pmu) {  in l2_cache_pmu_probe()
987 	l2cache_pmu->num_counters = get_num_counters();  in l2_cache_pmu_probe()
988 	l2cache_pmu->pdev = pdev;  in l2_cache_pmu_probe()
989 	l2cache_pmu->pmu_cluster = devm_alloc_percpu(&pdev->dev,  in l2_cache_pmu_probe()
991 	if (!l2cache_pmu->pmu_cluster)  in l2_cache_pmu_probe()
992 		return -ENOMEM;  in l2_cache_pmu_probe()
994 	l2_cycle_ctr_idx = l2cache_pmu->num_counters - 1;  in l2_cache_pmu_probe()
995 	l2_counter_present_mask = GENMASK(l2cache_pmu->num_counters - 2, 0) |  in l2_cache_pmu_probe()
998 	cpumask_clear(&l2cache_pmu->cpumask);  in l2_cache_pmu_probe()
1000 	/* Read cluster info and initialize each cluster */  in l2_cache_pmu_probe()
1001 	err = device_for_each_child(&pdev->dev, l2cache_pmu,  in l2_cache_pmu_probe()
1006 	if (l2cache_pmu->num_pmus == 0) {  in l2_cache_pmu_probe()
1007 		dev_err(&pdev->dev, "No hardware L2 cache PMUs found\n");  in l2_cache_pmu_probe()
1008 		return -ENODEV;  in l2_cache_pmu_probe()
1012 				       &l2cache_pmu->node);  in l2_cache_pmu_probe()
1014 		dev_err(&pdev->dev, "Error %d registering hotplug", err);  in l2_cache_pmu_probe()
1018 	err = perf_pmu_register(&l2cache_pmu->pmu, l2cache_pmu->pmu.name, -1);  in l2_cache_pmu_probe()
1020 		dev_err(&pdev->dev, "Error %d registering L2 cache PMU\n", err);  in l2_cache_pmu_probe()
1024 	dev_info(&pdev->dev, "Registered L2 cache PMU using %d HW PMUs\n",  in l2_cache_pmu_probe()
1025 		 l2cache_pmu->num_pmus);  in l2_cache_pmu_probe()
1031 				    &l2cache_pmu->node);  in l2_cache_pmu_probe()
1040 	perf_pmu_unregister(&l2cache_pmu->pmu);  in l2_cache_pmu_remove()
1042 				    &l2cache_pmu->node);  in l2_cache_pmu_remove()
1048 		.name = "qcom-l2cache-pmu",