• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2012 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "rsCpuCore.h"
18 #include "rsCpuScript.h"
19 #include "rsCpuScriptGroup.h"
20 
21 #include <malloc.h>
22 #include "rsContext.h"
23 
24 #include <sys/types.h>
25 #include <sys/resource.h>
26 #include <sched.h>
27 #include <sys/syscall.h>
28 #include <string.h>
29 
30 #ifndef RS_SERVER
31 #include <cutils/properties.h>
32 #include "utils/StopWatch.h"
33 #endif
34 
35 #ifdef RS_SERVER
36 // Android exposes gettid(), standard Linux does not
gettid()37 static pid_t gettid() {
38     return syscall(SYS_gettid);
39 }
40 #endif
41 
42 using namespace android;
43 using namespace android::renderscript;
44 
45 typedef void (*outer_foreach_t)(
46     const android::renderscript::RsForEachStubParamStruct *,
47     uint32_t x1, uint32_t x2,
48     uint32_t instep, uint32_t outstep);
49 
50 
51 static pthread_key_t gThreadTLSKey = 0;
52 static uint32_t gThreadTLSKeyCount = 0;
53 static pthread_mutex_t gInitMutex = PTHREAD_MUTEX_INITIALIZER;
54 
~RsdCpuReference()55 RsdCpuReference::~RsdCpuReference() {
56 }
57 
create(Context * rsc,uint32_t version_major,uint32_t version_minor,sym_lookup_t lfn,script_lookup_t slfn,bcc::RSLinkRuntimeCallback pLinkRuntimeCallback,RSSelectRTCallback pSelectRTCallback)58 RsdCpuReference * RsdCpuReference::create(Context *rsc, uint32_t version_major,
59         uint32_t version_minor, sym_lookup_t lfn, script_lookup_t slfn
60 #ifndef RS_COMPATIBILITY_LIB
61         , bcc::RSLinkRuntimeCallback pLinkRuntimeCallback,
62         RSSelectRTCallback pSelectRTCallback
63 #endif
64         ) {
65 
66     RsdCpuReferenceImpl *cpu = new RsdCpuReferenceImpl(rsc);
67     if (!cpu) {
68         return NULL;
69     }
70     if (!cpu->init(version_major, version_minor, lfn, slfn)) {
71         delete cpu;
72         return NULL;
73     }
74 
75 #ifndef RS_COMPATIBILITY_LIB
76     cpu->setLinkRuntimeCallback(pLinkRuntimeCallback);
77     cpu->setSelectRTCallback(pSelectRTCallback);
78 #endif
79 
80     return cpu;
81 }
82 
83 
getTlsContext()84 Context * RsdCpuReference::getTlsContext() {
85     ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey);
86     return tls->mContext;
87 }
88 
getTlsScript()89 const Script * RsdCpuReference::getTlsScript() {
90     ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey);
91     return tls->mScript;
92 }
93 
getThreadTLSKey()94 pthread_key_t RsdCpuReference::getThreadTLSKey(){ return gThreadTLSKey; }
95 
96 ////////////////////////////////////////////////////////////
97 ///
98 
RsdCpuReferenceImpl(Context * rsc)99 RsdCpuReferenceImpl::RsdCpuReferenceImpl(Context *rsc) {
100     mRSC = rsc;
101 
102     version_major = 0;
103     version_minor = 0;
104     mInForEach = false;
105     memset(&mWorkers, 0, sizeof(mWorkers));
106     memset(&mTlsStruct, 0, sizeof(mTlsStruct));
107     mExit = false;
108 #ifndef RS_COMPATIBILITY_LIB
109     mLinkRuntimeCallback = NULL;
110     mSelectRTCallback = NULL;
111 #endif
112 }
113 
114 
helperThreadProc(void * vrsc)115 void * RsdCpuReferenceImpl::helperThreadProc(void *vrsc) {
116     RsdCpuReferenceImpl *dc = (RsdCpuReferenceImpl *)vrsc;
117 
118     uint32_t idx = __sync_fetch_and_add(&dc->mWorkers.mLaunchCount, 1);
119 
120     //ALOGV("RS helperThread starting %p idx=%i", dc, idx);
121 
122     dc->mWorkers.mLaunchSignals[idx].init();
123     dc->mWorkers.mNativeThreadId[idx] = gettid();
124 
125     memset(&dc->mTlsStruct, 0, sizeof(dc->mTlsStruct));
126     int status = pthread_setspecific(gThreadTLSKey, &dc->mTlsStruct);
127     if (status) {
128         ALOGE("pthread_setspecific %i", status);
129     }
130 
131 #if 0
132     typedef struct {uint64_t bits[1024 / 64]; } cpu_set_t;
133     cpu_set_t cpuset;
134     memset(&cpuset, 0, sizeof(cpuset));
135     cpuset.bits[idx / 64] |= 1ULL << (idx % 64);
136     int ret = syscall(241, rsc->mWorkers.mNativeThreadId[idx],
137               sizeof(cpuset), &cpuset);
138     ALOGE("SETAFFINITY ret = %i %s", ret, EGLUtils::strerror(ret));
139 #endif
140 
141     while (!dc->mExit) {
142         dc->mWorkers.mLaunchSignals[idx].wait();
143         if (dc->mWorkers.mLaunchCallback) {
144            // idx +1 is used because the calling thread is always worker 0.
145            dc->mWorkers.mLaunchCallback(dc->mWorkers.mLaunchData, idx+1);
146         }
147         __sync_fetch_and_sub(&dc->mWorkers.mRunningCount, 1);
148         dc->mWorkers.mCompleteSignal.set();
149     }
150 
151     //ALOGV("RS helperThread exited %p idx=%i", dc, idx);
152     return NULL;
153 }
154 
launchThreads(WorkerCallback_t cbk,void * data)155 void RsdCpuReferenceImpl::launchThreads(WorkerCallback_t cbk, void *data) {
156     mWorkers.mLaunchData = data;
157     mWorkers.mLaunchCallback = cbk;
158 
159     // fast path for very small launches
160     MTLaunchStruct *mtls = (MTLaunchStruct *)data;
161     if (mtls && mtls->fep.dimY <= 1 && mtls->xEnd <= mtls->xStart + mtls->mSliceSize) {
162         if (mWorkers.mLaunchCallback) {
163             mWorkers.mLaunchCallback(mWorkers.mLaunchData, 0);
164         }
165         return;
166     }
167 
168     mWorkers.mRunningCount = mWorkers.mCount;
169     __sync_synchronize();
170 
171     for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) {
172         mWorkers.mLaunchSignals[ct].set();
173     }
174 
175     // We use the calling thread as one of the workers so we can start without
176     // the delay of the thread wakeup.
177     if (mWorkers.mLaunchCallback) {
178         mWorkers.mLaunchCallback(mWorkers.mLaunchData, 0);
179     }
180 
181     while (__sync_fetch_and_or(&mWorkers.mRunningCount, 0) != 0) {
182         mWorkers.mCompleteSignal.wait();
183     }
184 }
185 
186 
lockMutex()187 void RsdCpuReferenceImpl::lockMutex() {
188     pthread_mutex_lock(&gInitMutex);
189 }
190 
unlockMutex()191 void RsdCpuReferenceImpl::unlockMutex() {
192     pthread_mutex_unlock(&gInitMutex);
193 }
194 
init(uint32_t version_major,uint32_t version_minor,sym_lookup_t lfn,script_lookup_t slfn)195 bool RsdCpuReferenceImpl::init(uint32_t version_major, uint32_t version_minor,
196                                sym_lookup_t lfn, script_lookup_t slfn) {
197 
198     mSymLookupFn = lfn;
199     mScriptLookupFn = slfn;
200 
201     lockMutex();
202     if (!gThreadTLSKeyCount) {
203         int status = pthread_key_create(&gThreadTLSKey, NULL);
204         if (status) {
205             ALOGE("Failed to init thread tls key.");
206             unlockMutex();
207             return false;
208         }
209     }
210     gThreadTLSKeyCount++;
211     unlockMutex();
212 
213     mTlsStruct.mContext = mRSC;
214     mTlsStruct.mScript = NULL;
215     int status = pthread_setspecific(gThreadTLSKey, &mTlsStruct);
216     if (status) {
217         ALOGE("pthread_setspecific %i", status);
218     }
219 
220     int cpu = sysconf(_SC_NPROCESSORS_ONLN);
221     if(mRSC->props.mDebugMaxThreads) {
222         cpu = mRSC->props.mDebugMaxThreads;
223     }
224     if (cpu < 2) {
225         mWorkers.mCount = 0;
226         return true;
227     }
228 
229     // Subtract one from the cpu count because we also use the command thread as a worker.
230     mWorkers.mCount = (uint32_t)(cpu - 1);
231 
232     ALOGV("%p Launching thread(s), CPUs %i", mRSC, mWorkers.mCount + 1);
233 
234     mWorkers.mThreadId = (pthread_t *) calloc(mWorkers.mCount, sizeof(pthread_t));
235     mWorkers.mNativeThreadId = (pid_t *) calloc(mWorkers.mCount, sizeof(pid_t));
236     mWorkers.mLaunchSignals = new Signal[mWorkers.mCount];
237     mWorkers.mLaunchCallback = NULL;
238 
239     mWorkers.mCompleteSignal.init();
240 
241     mWorkers.mRunningCount = mWorkers.mCount;
242     mWorkers.mLaunchCount = 0;
243     __sync_synchronize();
244 
245     pthread_attr_t threadAttr;
246     status = pthread_attr_init(&threadAttr);
247     if (status) {
248         ALOGE("Failed to init thread attribute.");
249         return false;
250     }
251 
252     for (uint32_t ct=0; ct < mWorkers.mCount; ct++) {
253         status = pthread_create(&mWorkers.mThreadId[ct], &threadAttr, helperThreadProc, this);
254         if (status) {
255             mWorkers.mCount = ct;
256             ALOGE("Created fewer than expected number of RS threads.");
257             break;
258         }
259     }
260     while (__sync_fetch_and_or(&mWorkers.mRunningCount, 0) != 0) {
261         usleep(100);
262     }
263 
264     pthread_attr_destroy(&threadAttr);
265     return true;
266 }
267 
268 
setPriority(int32_t priority)269 void RsdCpuReferenceImpl::setPriority(int32_t priority) {
270     for (uint32_t ct=0; ct < mWorkers.mCount; ct++) {
271         setpriority(PRIO_PROCESS, mWorkers.mNativeThreadId[ct], priority);
272     }
273 }
274 
~RsdCpuReferenceImpl()275 RsdCpuReferenceImpl::~RsdCpuReferenceImpl() {
276     mExit = true;
277     mWorkers.mLaunchData = NULL;
278     mWorkers.mLaunchCallback = NULL;
279     mWorkers.mRunningCount = mWorkers.mCount;
280     __sync_synchronize();
281     for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) {
282         mWorkers.mLaunchSignals[ct].set();
283     }
284     void *res;
285     for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) {
286         pthread_join(mWorkers.mThreadId[ct], &res);
287     }
288     rsAssert(__sync_fetch_and_or(&mWorkers.mRunningCount, 0) == 0);
289 
290     // Global structure cleanup.
291     lockMutex();
292     --gThreadTLSKeyCount;
293     if (!gThreadTLSKeyCount) {
294         pthread_key_delete(gThreadTLSKey);
295     }
296     unlockMutex();
297 
298 }
299 
300 typedef void (*rs_t)(const void *, void *, const void *, uint32_t, uint32_t, uint32_t, uint32_t);
301 
wc_xy(void * usr,uint32_t idx)302 static void wc_xy(void *usr, uint32_t idx) {
303     MTLaunchStruct *mtls = (MTLaunchStruct *)usr;
304     RsForEachStubParamStruct p;
305     memcpy(&p, &mtls->fep, sizeof(p));
306     p.lid = idx;
307     uint32_t sig = mtls->sig;
308 
309     outer_foreach_t fn = (outer_foreach_t) mtls->kernel;
310     while (1) {
311         uint32_t slice = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1);
312         uint32_t yStart = mtls->yStart + slice * mtls->mSliceSize;
313         uint32_t yEnd = yStart + mtls->mSliceSize;
314         yEnd = rsMin(yEnd, mtls->yEnd);
315         if (yEnd <= yStart) {
316             return;
317         }
318 
319         //ALOGE("usr idx %i, x %i,%i  y %i,%i", idx, mtls->xStart, mtls->xEnd, yStart, yEnd);
320         //ALOGE("usr ptr in %p,  out %p", mtls->fep.ptrIn, mtls->fep.ptrOut);
321 
322         for (p.y = yStart; p.y < yEnd; p.y++) {
323             p.out = mtls->fep.ptrOut + (mtls->fep.yStrideOut * p.y) +
324                     (mtls->fep.eStrideOut * mtls->xStart);
325             p.in = mtls->fep.ptrIn + (mtls->fep.yStrideIn * p.y) +
326                    (mtls->fep.eStrideIn * mtls->xStart);
327             fn(&p, mtls->xStart, mtls->xEnd, mtls->fep.eStrideIn, mtls->fep.eStrideOut);
328         }
329     }
330 }
331 
wc_x(void * usr,uint32_t idx)332 static void wc_x(void *usr, uint32_t idx) {
333     MTLaunchStruct *mtls = (MTLaunchStruct *)usr;
334     RsForEachStubParamStruct p;
335     memcpy(&p, &mtls->fep, sizeof(p));
336     p.lid = idx;
337     uint32_t sig = mtls->sig;
338 
339     outer_foreach_t fn = (outer_foreach_t) mtls->kernel;
340     while (1) {
341         uint32_t slice = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1);
342         uint32_t xStart = mtls->xStart + slice * mtls->mSliceSize;
343         uint32_t xEnd = xStart + mtls->mSliceSize;
344         xEnd = rsMin(xEnd, mtls->xEnd);
345         if (xEnd <= xStart) {
346             return;
347         }
348 
349         //ALOGE("usr slice %i idx %i, x %i,%i", slice, idx, xStart, xEnd);
350         //ALOGE("usr ptr in %p,  out %p", mtls->fep.ptrIn, mtls->fep.ptrOut);
351 
352         p.out = mtls->fep.ptrOut + (mtls->fep.eStrideOut * xStart);
353         p.in = mtls->fep.ptrIn + (mtls->fep.eStrideIn * xStart);
354         fn(&p, xStart, xEnd, mtls->fep.eStrideIn, mtls->fep.eStrideOut);
355     }
356 }
357 
launchThreads(const Allocation * ain,Allocation * aout,const RsScriptCall * sc,MTLaunchStruct * mtls)358 void RsdCpuReferenceImpl::launchThreads(const Allocation * ain, Allocation * aout,
359                                      const RsScriptCall *sc, MTLaunchStruct *mtls) {
360 
361     //android::StopWatch kernel_time("kernel time");
362 
363     if ((mWorkers.mCount >= 1) && mtls->isThreadable && !mInForEach) {
364         const size_t targetByteChunk = 16 * 1024;
365         mInForEach = true;
366         if (mtls->fep.dimY > 1) {
367             uint32_t s1 = mtls->fep.dimY / ((mWorkers.mCount + 1) * 4);
368             uint32_t s2 = 0;
369 
370             // This chooses our slice size to rate limit atomic ops to
371             // one per 16k bytes of reads/writes.
372             if (mtls->fep.yStrideOut) {
373                 s2 = targetByteChunk / mtls->fep.yStrideOut;
374             } else {
375                 s2 = targetByteChunk / mtls->fep.yStrideIn;
376             }
377             mtls->mSliceSize = rsMin(s1, s2);
378 
379             if(mtls->mSliceSize < 1) {
380                 mtls->mSliceSize = 1;
381             }
382 
383          //   mtls->mSliceSize = 2;
384             launchThreads(wc_xy, mtls);
385         } else {
386             uint32_t s1 = mtls->fep.dimX / ((mWorkers.mCount + 1) * 4);
387             uint32_t s2 = 0;
388 
389             // This chooses our slice size to rate limit atomic ops to
390             // one per 16k bytes of reads/writes.
391             if (mtls->fep.eStrideOut) {
392                 s2 = targetByteChunk / mtls->fep.eStrideOut;
393             } else {
394                 s2 = targetByteChunk / mtls->fep.eStrideIn;
395             }
396             mtls->mSliceSize = rsMin(s1, s2);
397 
398             if(mtls->mSliceSize < 1) {
399                 mtls->mSliceSize = 1;
400             }
401 
402             launchThreads(wc_x, mtls);
403         }
404         mInForEach = false;
405 
406         //ALOGE("launch 1");
407     } else {
408         RsForEachStubParamStruct p;
409         memcpy(&p, &mtls->fep, sizeof(p));
410         uint32_t sig = mtls->sig;
411 
412         //ALOGE("launch 3");
413         outer_foreach_t fn = (outer_foreach_t) mtls->kernel;
414         for (p.ar[0] = mtls->arrayStart; p.ar[0] < mtls->arrayEnd; p.ar[0]++) {
415             for (p.z = mtls->zStart; p.z < mtls->zEnd; p.z++) {
416                 for (p.y = mtls->yStart; p.y < mtls->yEnd; p.y++) {
417                     uint32_t offset = mtls->fep.dimY * mtls->fep.dimZ * p.ar[0] +
418                                       mtls->fep.dimY * p.z + p.y;
419                     p.out = mtls->fep.ptrOut + (mtls->fep.yStrideOut * offset) +
420                             (mtls->fep.eStrideOut * mtls->xStart);
421                     p.in = mtls->fep.ptrIn + (mtls->fep.yStrideIn * offset) +
422                            (mtls->fep.eStrideIn * mtls->xStart);
423                     fn(&p, mtls->xStart, mtls->xEnd, mtls->fep.eStrideIn, mtls->fep.eStrideOut);
424                 }
425             }
426         }
427     }
428 }
429 
setTLS(RsdCpuScriptImpl * sc)430 RsdCpuScriptImpl * RsdCpuReferenceImpl::setTLS(RsdCpuScriptImpl *sc) {
431     //ALOGE("setTls %p", sc);
432     ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey);
433     rsAssert(tls);
434     RsdCpuScriptImpl *old = tls->mImpl;
435     tls->mImpl = sc;
436     tls->mContext = mRSC;
437     if (sc) {
438         tls->mScript = sc->getScript();
439     } else {
440         tls->mScript = NULL;
441     }
442     return old;
443 }
444 
symLookup(const char * name)445 const RsdCpuReference::CpuSymbol * RsdCpuReferenceImpl::symLookup(const char *name) {
446     return mSymLookupFn(mRSC, name);
447 }
448 
449 
createScript(const ScriptC * s,char const * resName,char const * cacheDir,uint8_t const * bitcode,size_t bitcodeSize,uint32_t flags)450 RsdCpuReference::CpuScript * RsdCpuReferenceImpl::createScript(const ScriptC *s,
451                                     char const *resName, char const *cacheDir,
452                                     uint8_t const *bitcode, size_t bitcodeSize,
453                                     uint32_t flags) {
454 
455     RsdCpuScriptImpl *i = new RsdCpuScriptImpl(this, s);
456     if (!i->init(resName, cacheDir, bitcode, bitcodeSize, flags)) {
457         delete i;
458         return NULL;
459     }
460     return i;
461 }
462 
463 extern RsdCpuScriptImpl * rsdIntrinsic_3DLUT(RsdCpuReferenceImpl *ctx,
464                                              const Script *s, const Element *e);
465 extern RsdCpuScriptImpl * rsdIntrinsic_Convolve3x3(RsdCpuReferenceImpl *ctx,
466                                                    const Script *s, const Element *e);
467 extern RsdCpuScriptImpl * rsdIntrinsic_ColorMatrix(RsdCpuReferenceImpl *ctx,
468                                                    const Script *s, const Element *e);
469 extern RsdCpuScriptImpl * rsdIntrinsic_LUT(RsdCpuReferenceImpl *ctx,
470                                            const Script *s, const Element *e);
471 extern RsdCpuScriptImpl * rsdIntrinsic_Convolve5x5(RsdCpuReferenceImpl *ctx,
472                                                    const Script *s, const Element *e);
473 extern RsdCpuScriptImpl * rsdIntrinsic_Blur(RsdCpuReferenceImpl *ctx,
474                                             const Script *s, const Element *e);
475 extern RsdCpuScriptImpl * rsdIntrinsic_YuvToRGB(RsdCpuReferenceImpl *ctx,
476                                                 const Script *s, const Element *e);
477 extern RsdCpuScriptImpl * rsdIntrinsic_Blend(RsdCpuReferenceImpl *ctx,
478                                              const Script *s, const Element *e);
479 
createIntrinsic(const Script * s,RsScriptIntrinsicID iid,Element * e)480 RsdCpuReference::CpuScript * RsdCpuReferenceImpl::createIntrinsic(const Script *s,
481                                     RsScriptIntrinsicID iid, Element *e) {
482 
483     RsdCpuScriptImpl *i = NULL;
484     switch (iid) {
485     case RS_SCRIPT_INTRINSIC_ID_3DLUT:
486         i = rsdIntrinsic_3DLUT(this, s, e);
487         break;
488     case RS_SCRIPT_INTRINSIC_ID_CONVOLVE_3x3:
489         i = rsdIntrinsic_Convolve3x3(this, s, e);
490         break;
491     case RS_SCRIPT_INTRINSIC_ID_COLOR_MATRIX:
492         i = rsdIntrinsic_ColorMatrix(this, s, e);
493         break;
494     case RS_SCRIPT_INTRINSIC_ID_LUT:
495         i = rsdIntrinsic_LUT(this, s, e);
496         break;
497     case RS_SCRIPT_INTRINSIC_ID_CONVOLVE_5x5:
498         i = rsdIntrinsic_Convolve5x5(this, s, e);
499         break;
500     case RS_SCRIPT_INTRINSIC_ID_BLUR:
501         i = rsdIntrinsic_Blur(this, s, e);
502         break;
503     case RS_SCRIPT_INTRINSIC_ID_YUV_TO_RGB:
504         i = rsdIntrinsic_YuvToRGB(this, s, e);
505         break;
506     case RS_SCRIPT_INTRINSIC_ID_BLEND:
507         i = rsdIntrinsic_Blend(this, s, e);
508         break;
509 
510     default:
511         rsAssert(0);
512     }
513 
514     return i;
515 }
516 
createScriptGroup(const ScriptGroup * sg)517 RsdCpuReference::CpuScriptGroup * RsdCpuReferenceImpl::createScriptGroup(const ScriptGroup *sg) {
518     CpuScriptGroupImpl *sgi = new CpuScriptGroupImpl(this, sg);
519     if (!sgi->init()) {
520         delete sgi;
521         return NULL;
522     }
523     return sgi;
524 }
525 
526 
527