1 /*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10 #include <linux/spinlock.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/device.h>
14 #include <linux/mutex.h>
15 #include <linux/init.h>
16 #include <linux/list.h>
17 #include <linux/mm.h>
18 #include <linux/of.h>
19 #include <linux/slab.h>
20 #include <linux/idr.h>
21 #include <linux/pci.h>
22 #include <asm/cputable.h>
23 #include <misc/cxl-base.h>
24
25 #include "cxl.h"
26 #include "trace.h"
27
28 static DEFINE_SPINLOCK(adapter_idr_lock);
29 static DEFINE_IDR(cxl_adapter_idr);
30
31 uint cxl_verbose;
32 module_param_named(verbose, cxl_verbose, uint, 0600);
33 MODULE_PARM_DESC(verbose, "Enable verbose dmesg output");
34
_cxl_slbia(struct cxl_context * ctx,struct mm_struct * mm)35 static inline void _cxl_slbia(struct cxl_context *ctx, struct mm_struct *mm)
36 {
37 struct task_struct *task;
38 unsigned long flags;
39 if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {
40 pr_devel("%s unable to get task %i\n",
41 __func__, pid_nr(ctx->pid));
42 return;
43 }
44
45 if (task->mm != mm)
46 goto out_put;
47
48 pr_devel("%s matched mm - card: %i afu: %i pe: %i\n", __func__,
49 ctx->afu->adapter->adapter_num, ctx->afu->slice, ctx->pe);
50
51 spin_lock_irqsave(&ctx->sste_lock, flags);
52 trace_cxl_slbia(ctx);
53 memset(ctx->sstp, 0, ctx->sst_size);
54 spin_unlock_irqrestore(&ctx->sste_lock, flags);
55 mb();
56 cxl_afu_slbia(ctx->afu);
57 out_put:
58 put_task_struct(task);
59 }
60
cxl_slbia_core(struct mm_struct * mm)61 static inline void cxl_slbia_core(struct mm_struct *mm)
62 {
63 struct cxl *adapter;
64 struct cxl_afu *afu;
65 struct cxl_context *ctx;
66 int card, slice, id;
67
68 pr_devel("%s called\n", __func__);
69
70 spin_lock(&adapter_idr_lock);
71 idr_for_each_entry(&cxl_adapter_idr, adapter, card) {
72 /* XXX: Make this lookup faster with link from mm to ctx */
73 spin_lock(&adapter->afu_list_lock);
74 for (slice = 0; slice < adapter->slices; slice++) {
75 afu = adapter->afu[slice];
76 if (!afu || !afu->enabled)
77 continue;
78 rcu_read_lock();
79 idr_for_each_entry(&afu->contexts_idr, ctx, id)
80 _cxl_slbia(ctx, mm);
81 rcu_read_unlock();
82 }
83 spin_unlock(&adapter->afu_list_lock);
84 }
85 spin_unlock(&adapter_idr_lock);
86 }
87
88 static struct cxl_calls cxl_calls = {
89 .cxl_slbia = cxl_slbia_core,
90 .owner = THIS_MODULE,
91 };
92
cxl_alloc_sst(struct cxl_context * ctx)93 int cxl_alloc_sst(struct cxl_context *ctx)
94 {
95 unsigned long vsid;
96 u64 ea_mask, size, sstp0, sstp1;
97
98 sstp0 = 0;
99 sstp1 = 0;
100
101 ctx->sst_size = PAGE_SIZE;
102 ctx->sst_lru = 0;
103 ctx->sstp = (struct cxl_sste *)get_zeroed_page(GFP_KERNEL);
104 if (!ctx->sstp) {
105 pr_err("cxl_alloc_sst: Unable to allocate segment table\n");
106 return -ENOMEM;
107 }
108 pr_devel("SSTP allocated at 0x%p\n", ctx->sstp);
109
110 vsid = get_kernel_vsid((u64)ctx->sstp, mmu_kernel_ssize) << 12;
111
112 sstp0 |= (u64)mmu_kernel_ssize << CXL_SSTP0_An_B_SHIFT;
113 sstp0 |= (SLB_VSID_KERNEL | mmu_psize_defs[mmu_linear_psize].sllp) << 50;
114
115 size = (((u64)ctx->sst_size >> 8) - 1) << CXL_SSTP0_An_SegTableSize_SHIFT;
116 if (unlikely(size & ~CXL_SSTP0_An_SegTableSize_MASK)) {
117 WARN(1, "Impossible segment table size\n");
118 return -EINVAL;
119 }
120 sstp0 |= size;
121
122 if (mmu_kernel_ssize == MMU_SEGSIZE_256M)
123 ea_mask = 0xfffff00ULL;
124 else
125 ea_mask = 0xffffffff00ULL;
126
127 sstp0 |= vsid >> (50-14); /* Top 14 bits of VSID */
128 sstp1 |= (vsid << (64-(50-14))) & ~ea_mask;
129 sstp1 |= (u64)ctx->sstp & ea_mask;
130 sstp1 |= CXL_SSTP1_An_V;
131
132 pr_devel("Looked up %#llx: slbfee. %#llx (ssize: %x, vsid: %#lx), copied to SSTP0: %#llx, SSTP1: %#llx\n",
133 (u64)ctx->sstp, (u64)ctx->sstp & ESID_MASK, mmu_kernel_ssize, vsid, sstp0, sstp1);
134
135 /* Store calculated sstp hardware points for use later */
136 ctx->sstp0 = sstp0;
137 ctx->sstp1 = sstp1;
138
139 return 0;
140 }
141
142 /* Find a CXL adapter by it's number and increase it's refcount */
get_cxl_adapter(int num)143 struct cxl *get_cxl_adapter(int num)
144 {
145 struct cxl *adapter;
146
147 spin_lock(&adapter_idr_lock);
148 if ((adapter = idr_find(&cxl_adapter_idr, num)))
149 get_device(&adapter->dev);
150 spin_unlock(&adapter_idr_lock);
151
152 return adapter;
153 }
154
cxl_alloc_adapter_nr(struct cxl * adapter)155 int cxl_alloc_adapter_nr(struct cxl *adapter)
156 {
157 int i;
158
159 idr_preload(GFP_KERNEL);
160 spin_lock(&adapter_idr_lock);
161 i = idr_alloc(&cxl_adapter_idr, adapter, 0, 0, GFP_NOWAIT);
162 spin_unlock(&adapter_idr_lock);
163 idr_preload_end();
164 if (i < 0)
165 return i;
166
167 adapter->adapter_num = i;
168
169 return 0;
170 }
171
cxl_remove_adapter_nr(struct cxl * adapter)172 void cxl_remove_adapter_nr(struct cxl *adapter)
173 {
174 idr_remove(&cxl_adapter_idr, adapter->adapter_num);
175 }
176
cxl_afu_select_best_mode(struct cxl_afu * afu)177 int cxl_afu_select_best_mode(struct cxl_afu *afu)
178 {
179 if (afu->modes_supported & CXL_MODE_DIRECTED)
180 return cxl_afu_activate_mode(afu, CXL_MODE_DIRECTED);
181
182 if (afu->modes_supported & CXL_MODE_DEDICATED)
183 return cxl_afu_activate_mode(afu, CXL_MODE_DEDICATED);
184
185 dev_warn(&afu->dev, "No supported programming modes available\n");
186 /* We don't fail this so the user can inspect sysfs */
187 return 0;
188 }
189
init_cxl(void)190 static int __init init_cxl(void)
191 {
192 int rc = 0;
193
194 if (!cpu_has_feature(CPU_FTR_HVMODE))
195 return -EPERM;
196
197 if ((rc = cxl_file_init()))
198 return rc;
199
200 cxl_debugfs_init();
201
202 if ((rc = register_cxl_calls(&cxl_calls)))
203 goto err;
204
205 if ((rc = pci_register_driver(&cxl_pci_driver)))
206 goto err1;
207
208 return 0;
209 err1:
210 unregister_cxl_calls(&cxl_calls);
211 err:
212 cxl_debugfs_exit();
213 cxl_file_exit();
214
215 return rc;
216 }
217
exit_cxl(void)218 static void exit_cxl(void)
219 {
220 pci_unregister_driver(&cxl_pci_driver);
221
222 cxl_debugfs_exit();
223 cxl_file_exit();
224 unregister_cxl_calls(&cxl_calls);
225 idr_destroy(&cxl_adapter_idr);
226 }
227
228 module_init(init_cxl);
229 module_exit(exit_cxl);
230
231 MODULE_DESCRIPTION("IBM Coherent Accelerator");
232 MODULE_AUTHOR("Ian Munsie <imunsie@au1.ibm.com>");
233 MODULE_LICENSE("GPL");
234