1 #include <linux/module.h>
2 #include <linux/slab.h>
3 #include <linux/pci.h>
4 #include <linux/perf_event.h>
5 #include "perf_event.h"
6
7 #define UNCORE_PMU_NAME_LEN 32
8 #define UNCORE_PMU_HRTIMER_INTERVAL (60LL * NSEC_PER_SEC)
9 #define UNCORE_SNB_IMC_HRTIMER_INTERVAL (5ULL * NSEC_PER_SEC)
10
11 #define UNCORE_FIXED_EVENT 0xff
12 #define UNCORE_PMC_IDX_MAX_GENERIC 8
13 #define UNCORE_PMC_IDX_FIXED UNCORE_PMC_IDX_MAX_GENERIC
14 #define UNCORE_PMC_IDX_MAX (UNCORE_PMC_IDX_FIXED + 1)
15
16 #define UNCORE_PCI_DEV_DATA(type, idx) ((type << 8) | idx)
17 #define UNCORE_PCI_DEV_TYPE(data) ((data >> 8) & 0xff)
18 #define UNCORE_PCI_DEV_IDX(data) (data & 0xff)
19 #define UNCORE_EXTRA_PCI_DEV 0xff
20 #define UNCORE_EXTRA_PCI_DEV_MAX 3
21
22 /* support up to 8 sockets */
23 #define UNCORE_SOCKET_MAX 8
24
25 #define UNCORE_EVENT_CONSTRAINT(c, n) EVENT_CONSTRAINT(c, n, 0xff)
26
27 struct intel_uncore_ops;
28 struct intel_uncore_pmu;
29 struct intel_uncore_box;
30 struct uncore_event_desc;
31
32 struct intel_uncore_type {
33 const char *name;
34 int num_counters;
35 int num_boxes;
36 int perf_ctr_bits;
37 int fixed_ctr_bits;
38 unsigned perf_ctr;
39 unsigned event_ctl;
40 unsigned event_mask;
41 unsigned fixed_ctr;
42 unsigned fixed_ctl;
43 unsigned box_ctl;
44 unsigned msr_offset;
45 unsigned num_shared_regs:8;
46 unsigned single_fixed:1;
47 unsigned pair_ctr_ctl:1;
48 unsigned *msr_offsets;
49 struct event_constraint unconstrainted;
50 struct event_constraint *constraints;
51 struct intel_uncore_pmu *pmus;
52 struct intel_uncore_ops *ops;
53 struct uncore_event_desc *event_descs;
54 const struct attribute_group *attr_groups[4];
55 struct pmu *pmu; /* for custom pmu ops */
56 };
57
58 #define pmu_group attr_groups[0]
59 #define format_group attr_groups[1]
60 #define events_group attr_groups[2]
61
62 struct intel_uncore_ops {
63 void (*init_box)(struct intel_uncore_box *);
64 void (*disable_box)(struct intel_uncore_box *);
65 void (*enable_box)(struct intel_uncore_box *);
66 void (*disable_event)(struct intel_uncore_box *, struct perf_event *);
67 void (*enable_event)(struct intel_uncore_box *, struct perf_event *);
68 u64 (*read_counter)(struct intel_uncore_box *, struct perf_event *);
69 int (*hw_config)(struct intel_uncore_box *, struct perf_event *);
70 struct event_constraint *(*get_constraint)(struct intel_uncore_box *,
71 struct perf_event *);
72 void (*put_constraint)(struct intel_uncore_box *, struct perf_event *);
73 };
74
75 struct intel_uncore_pmu {
76 struct pmu pmu;
77 char name[UNCORE_PMU_NAME_LEN];
78 int pmu_idx;
79 int func_id;
80 struct intel_uncore_type *type;
81 struct intel_uncore_box ** __percpu box;
82 struct list_head box_list;
83 };
84
85 struct intel_uncore_extra_reg {
86 raw_spinlock_t lock;
87 u64 config, config1, config2;
88 atomic_t ref;
89 };
90
91 struct intel_uncore_box {
92 int phys_id;
93 int n_active; /* number of active events */
94 int n_events;
95 int cpu; /* cpu to collect events */
96 unsigned long flags;
97 atomic_t refcnt;
98 struct perf_event *events[UNCORE_PMC_IDX_MAX];
99 struct perf_event *event_list[UNCORE_PMC_IDX_MAX];
100 unsigned long active_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
101 u64 tags[UNCORE_PMC_IDX_MAX];
102 struct pci_dev *pci_dev;
103 struct intel_uncore_pmu *pmu;
104 u64 hrtimer_duration; /* hrtimer timeout for this box */
105 struct hrtimer hrtimer;
106 struct list_head list;
107 struct list_head active_list;
108 void *io_addr;
109 struct intel_uncore_extra_reg shared_regs[0];
110 };
111
112 #define UNCORE_BOX_FLAG_INITIATED 0
113
114 struct uncore_event_desc {
115 struct kobj_attribute attr;
116 const char *config;
117 };
118
119 ssize_t uncore_event_show(struct kobject *kobj,
120 struct kobj_attribute *attr, char *buf);
121
122 #define INTEL_UNCORE_EVENT_DESC(_name, _config) \
123 { \
124 .attr = __ATTR(_name, 0444, uncore_event_show, NULL), \
125 .config = _config, \
126 }
127
128 #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format) \
129 static ssize_t __uncore_##_var##_show(struct kobject *kobj, \
130 struct kobj_attribute *attr, \
131 char *page) \
132 { \
133 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
134 return sprintf(page, _format "\n"); \
135 } \
136 static struct kobj_attribute format_attr_##_var = \
137 __ATTR(_name, 0444, __uncore_##_var##_show, NULL)
138
uncore_pci_box_ctl(struct intel_uncore_box * box)139 static inline unsigned uncore_pci_box_ctl(struct intel_uncore_box *box)
140 {
141 return box->pmu->type->box_ctl;
142 }
143
uncore_pci_fixed_ctl(struct intel_uncore_box * box)144 static inline unsigned uncore_pci_fixed_ctl(struct intel_uncore_box *box)
145 {
146 return box->pmu->type->fixed_ctl;
147 }
148
uncore_pci_fixed_ctr(struct intel_uncore_box * box)149 static inline unsigned uncore_pci_fixed_ctr(struct intel_uncore_box *box)
150 {
151 return box->pmu->type->fixed_ctr;
152 }
153
154 static inline
uncore_pci_event_ctl(struct intel_uncore_box * box,int idx)155 unsigned uncore_pci_event_ctl(struct intel_uncore_box *box, int idx)
156 {
157 return idx * 4 + box->pmu->type->event_ctl;
158 }
159
160 static inline
uncore_pci_perf_ctr(struct intel_uncore_box * box,int idx)161 unsigned uncore_pci_perf_ctr(struct intel_uncore_box *box, int idx)
162 {
163 return idx * 8 + box->pmu->type->perf_ctr;
164 }
165
uncore_msr_box_offset(struct intel_uncore_box * box)166 static inline unsigned uncore_msr_box_offset(struct intel_uncore_box *box)
167 {
168 struct intel_uncore_pmu *pmu = box->pmu;
169 return pmu->type->msr_offsets ?
170 pmu->type->msr_offsets[pmu->pmu_idx] :
171 pmu->type->msr_offset * pmu->pmu_idx;
172 }
173
uncore_msr_box_ctl(struct intel_uncore_box * box)174 static inline unsigned uncore_msr_box_ctl(struct intel_uncore_box *box)
175 {
176 if (!box->pmu->type->box_ctl)
177 return 0;
178 return box->pmu->type->box_ctl + uncore_msr_box_offset(box);
179 }
180
uncore_msr_fixed_ctl(struct intel_uncore_box * box)181 static inline unsigned uncore_msr_fixed_ctl(struct intel_uncore_box *box)
182 {
183 if (!box->pmu->type->fixed_ctl)
184 return 0;
185 return box->pmu->type->fixed_ctl + uncore_msr_box_offset(box);
186 }
187
uncore_msr_fixed_ctr(struct intel_uncore_box * box)188 static inline unsigned uncore_msr_fixed_ctr(struct intel_uncore_box *box)
189 {
190 return box->pmu->type->fixed_ctr + uncore_msr_box_offset(box);
191 }
192
193 static inline
uncore_msr_event_ctl(struct intel_uncore_box * box,int idx)194 unsigned uncore_msr_event_ctl(struct intel_uncore_box *box, int idx)
195 {
196 return box->pmu->type->event_ctl +
197 (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) +
198 uncore_msr_box_offset(box);
199 }
200
201 static inline
uncore_msr_perf_ctr(struct intel_uncore_box * box,int idx)202 unsigned uncore_msr_perf_ctr(struct intel_uncore_box *box, int idx)
203 {
204 return box->pmu->type->perf_ctr +
205 (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) +
206 uncore_msr_box_offset(box);
207 }
208
209 static inline
uncore_fixed_ctl(struct intel_uncore_box * box)210 unsigned uncore_fixed_ctl(struct intel_uncore_box *box)
211 {
212 if (box->pci_dev)
213 return uncore_pci_fixed_ctl(box);
214 else
215 return uncore_msr_fixed_ctl(box);
216 }
217
218 static inline
uncore_fixed_ctr(struct intel_uncore_box * box)219 unsigned uncore_fixed_ctr(struct intel_uncore_box *box)
220 {
221 if (box->pci_dev)
222 return uncore_pci_fixed_ctr(box);
223 else
224 return uncore_msr_fixed_ctr(box);
225 }
226
227 static inline
uncore_event_ctl(struct intel_uncore_box * box,int idx)228 unsigned uncore_event_ctl(struct intel_uncore_box *box, int idx)
229 {
230 if (box->pci_dev)
231 return uncore_pci_event_ctl(box, idx);
232 else
233 return uncore_msr_event_ctl(box, idx);
234 }
235
236 static inline
uncore_perf_ctr(struct intel_uncore_box * box,int idx)237 unsigned uncore_perf_ctr(struct intel_uncore_box *box, int idx)
238 {
239 if (box->pci_dev)
240 return uncore_pci_perf_ctr(box, idx);
241 else
242 return uncore_msr_perf_ctr(box, idx);
243 }
244
uncore_perf_ctr_bits(struct intel_uncore_box * box)245 static inline int uncore_perf_ctr_bits(struct intel_uncore_box *box)
246 {
247 return box->pmu->type->perf_ctr_bits;
248 }
249
uncore_fixed_ctr_bits(struct intel_uncore_box * box)250 static inline int uncore_fixed_ctr_bits(struct intel_uncore_box *box)
251 {
252 return box->pmu->type->fixed_ctr_bits;
253 }
254
uncore_num_counters(struct intel_uncore_box * box)255 static inline int uncore_num_counters(struct intel_uncore_box *box)
256 {
257 return box->pmu->type->num_counters;
258 }
259
uncore_disable_box(struct intel_uncore_box * box)260 static inline void uncore_disable_box(struct intel_uncore_box *box)
261 {
262 if (box->pmu->type->ops->disable_box)
263 box->pmu->type->ops->disable_box(box);
264 }
265
uncore_enable_box(struct intel_uncore_box * box)266 static inline void uncore_enable_box(struct intel_uncore_box *box)
267 {
268 if (box->pmu->type->ops->enable_box)
269 box->pmu->type->ops->enable_box(box);
270 }
271
uncore_disable_event(struct intel_uncore_box * box,struct perf_event * event)272 static inline void uncore_disable_event(struct intel_uncore_box *box,
273 struct perf_event *event)
274 {
275 box->pmu->type->ops->disable_event(box, event);
276 }
277
uncore_enable_event(struct intel_uncore_box * box,struct perf_event * event)278 static inline void uncore_enable_event(struct intel_uncore_box *box,
279 struct perf_event *event)
280 {
281 box->pmu->type->ops->enable_event(box, event);
282 }
283
uncore_read_counter(struct intel_uncore_box * box,struct perf_event * event)284 static inline u64 uncore_read_counter(struct intel_uncore_box *box,
285 struct perf_event *event)
286 {
287 return box->pmu->type->ops->read_counter(box, event);
288 }
289
uncore_box_init(struct intel_uncore_box * box)290 static inline void uncore_box_init(struct intel_uncore_box *box)
291 {
292 if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) {
293 if (box->pmu->type->ops->init_box)
294 box->pmu->type->ops->init_box(box);
295 }
296 }
297
uncore_box_is_fake(struct intel_uncore_box * box)298 static inline bool uncore_box_is_fake(struct intel_uncore_box *box)
299 {
300 return (box->phys_id < 0);
301 }
302
303 struct intel_uncore_pmu *uncore_event_to_pmu(struct perf_event *event);
304 struct intel_uncore_box *uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu);
305 struct intel_uncore_box *uncore_event_to_box(struct perf_event *event);
306 u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event);
307 void uncore_pmu_start_hrtimer(struct intel_uncore_box *box);
308 void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box);
309 void uncore_pmu_event_read(struct perf_event *event);
310 void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event);
311 struct event_constraint *
312 uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event);
313 void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event);
314 u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx);
315
316 extern struct intel_uncore_type **uncore_msr_uncores;
317 extern struct intel_uncore_type **uncore_pci_uncores;
318 extern struct pci_driver *uncore_pci_driver;
319 extern int uncore_pcibus_to_physid[256];
320 extern struct pci_dev *uncore_extra_pci_dev[UNCORE_SOCKET_MAX][UNCORE_EXTRA_PCI_DEV_MAX];
321 extern struct event_constraint uncore_constraint_empty;
322
323 /* perf_event_intel_uncore_snb.c */
324 int snb_uncore_pci_init(void);
325 int ivb_uncore_pci_init(void);
326 int hsw_uncore_pci_init(void);
327 void snb_uncore_cpu_init(void);
328 void nhm_uncore_cpu_init(void);
329
330 /* perf_event_intel_uncore_snbep.c */
331 int snbep_uncore_pci_init(void);
332 void snbep_uncore_cpu_init(void);
333 int ivbep_uncore_pci_init(void);
334 void ivbep_uncore_cpu_init(void);
335 int hswep_uncore_pci_init(void);
336 void hswep_uncore_cpu_init(void);
337
338 /* perf_event_intel_uncore_nhmex.c */
339 void nhmex_uncore_cpu_init(void);
340