vm/interp/Jit.c

   1 /*
   2  * Copyright (C) 2008 The Android Open Source Project
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 #ifdef WITH_JIT
  17
  18 /*
  19  * Target independent portion of Android's Jit
  20  */
  21
  22 #include "Dalvik.h"
  23 #include "Jit.h"
  24
  25 #include "libdex/DexOpcodes.h"
  26 #include <unistd.h>
  27 #include <pthread.h>
  28 #include <sys/time.h>
  29 #include <signal.h>
  30 #include "compiler/Compiler.h"
  31 #include "compiler/CompilerUtility.h"
  32 #include "compiler/CompilerIR.h"
  33 #include <errno.h>
  34
  35 #if defined(WITH_SELF_VERIFICATION)
  36 /* Allocate space for per-thread ShadowSpace data structures */
  37 void* dvmSelfVerificationShadowSpaceAlloc(Thread* self)
  38 {
  39     self->shadowSpace = (ShadowSpace*) calloc(1, sizeof(ShadowSpace));
  40     if (self->shadowSpace == NULL)
  41         return NULL;
  42
  43     self->shadowSpace->registerSpaceSize = REG_SPACE;
  44     self->shadowSpace->registerSpace =
  45         (int*) calloc(self->shadowSpace->registerSpaceSize, sizeof(int));
  46
  47     return self->shadowSpace->registerSpace;
  48 }
  49
  50 /* Free per-thread ShadowSpace data structures */
  51 void dvmSelfVerificationShadowSpaceFree(Thread* self)
  52 {
  53     free(self->shadowSpace->registerSpace);
  54     free(self->shadowSpace);
  55 }
  56
  57 /*
  58  * Save out PC, FP, thread state, and registers to shadow space.
  59  * Return a pointer to the shadow space for JIT to use.
  60  *
  61  * The set of saved state from the Thread structure is:
  62  *     pc  (Dalvik PC)
  63  *     fp  (Dalvik FP)
  64  *     retval
  65  *     method
  66  *     methodClassDex
  67  *     interpStackEnd
  68  */
  69 void* dvmSelfVerificationSaveState(const u2* pc, u4* fp,
  70                                    Thread* self, int targetTrace)
  71 {
  72     ShadowSpace *shadowSpace = self->shadowSpace;
  73     unsigned preBytes = self->interpSave.method->outsSize*4 +
  74         sizeof(StackSaveArea);
  75     unsigned postBytes = self->interpSave.method->registersSize*4;
  76
  77     //LOGD("### selfVerificationSaveState(%d) pc: 0x%x fp: 0x%x",
  78     //    self->threadId, (int)pc, (int)fp);
  79
  80     if (shadowSpace->selfVerificationState != kSVSIdle) {
  81         LOGD("~~~ Save: INCORRECT PREVIOUS STATE(%d): %d",
  82             self->threadId, shadowSpace->selfVerificationState);
  83         LOGD("********** SHADOW STATE DUMP **********");
  84         LOGD("PC: 0x%x FP: 0x%x", (int)pc, (int)fp);
  85     }
  86     shadowSpace->selfVerificationState = kSVSStart;
  87
  88     // Dynamically grow shadow register space if necessary
  89     if (preBytes + postBytes > shadowSpace->registerSpaceSize * sizeof(u4)) {
  90         free(shadowSpace->registerSpace);
  91         shadowSpace->registerSpaceSize = (preBytes + postBytes) / sizeof(u4);
  92         shadowSpace->registerSpace =
  93             (int*) calloc(shadowSpace->registerSpaceSize, sizeof(u4));
  94     }
  95
  96     // Remember original state
  97     shadowSpace->startPC = pc;
  98     shadowSpace->fp = fp;
  99     shadowSpace->retval = self->retval;
 100     shadowSpace->interpStackEnd = self->interpStackEnd;
 101
 102     /*
 103      * Store the original method here in case the trace ends with a
 104      * return/invoke, the last method.
 105      */
 106     shadowSpace->method = self->interpSave.method;
 107     shadowSpace->methodClassDex = self->interpSave.methodClassDex;
 108
 109     shadowSpace->shadowFP = shadowSpace->registerSpace +
 110                             shadowSpace->registerSpaceSize - postBytes/4;
 111
 112     self->interpSave.fp = (u4*)shadowSpace->shadowFP;
 113     self->interpStackEnd = (u1*)shadowSpace->registerSpace;
 114
 115     // Create a copy of the stack
 116     memcpy(((char*)shadowSpace->shadowFP)-preBytes, ((char*)fp)-preBytes,
 117         preBytes+postBytes);
 118
 119     // Setup the shadowed heap space
 120     shadowSpace->heapSpaceTail = shadowSpace->heapSpace;
 121
 122     // Reset trace length
 123     shadowSpace->traceLength = 0;
 124
 125     return shadowSpace;
 126 }
 127
 128 /*
 129  * Save ending PC, FP and compiled code exit point to shadow space.
 130  * Return a pointer to the shadow space for JIT to restore state.
 131  */
 132 void* dvmSelfVerificationRestoreState(const u2* pc, u4* fp,
 133                                       SelfVerificationState exitState,
 134                                       Thread* self)
 135 {
 136     ShadowSpace *shadowSpace = self->shadowSpace;
 137     shadowSpace->endPC = pc;
 138     shadowSpace->endShadowFP = fp;
 139     shadowSpace->jitExitState = exitState;
 140
 141     //LOGD("### selfVerificationRestoreState(%d) pc: 0x%x fp: 0x%x endPC: 0x%x",
 142     //    self->threadId, (int)shadowSpace->startPC, (int)shadowSpace->fp,
 143     //    (int)pc);
 144
 145     if (shadowSpace->selfVerificationState != kSVSStart) {
 146         LOGD("~~~ Restore: INCORRECT PREVIOUS STATE(%d): %d",
 147             self->threadId, shadowSpace->selfVerificationState);
 148         LOGD("********** SHADOW STATE DUMP **********");
 149         LOGD("Dalvik PC: 0x%x endPC: 0x%x", (int)shadowSpace->startPC,
 150             (int)shadowSpace->endPC);
 151         LOGD("Interp FP: 0x%x", (int)shadowSpace->fp);
 152         LOGD("Shadow FP: 0x%x endFP: 0x%x", (int)shadowSpace->shadowFP,
 153             (int)shadowSpace->endShadowFP);
 154     }
 155
 156     // Special case when punting after a single instruction
 157     if (exitState == kSVSPunt && pc == shadowSpace->startPC) {
 158         shadowSpace->selfVerificationState = kSVSIdle;
 159     } else if (exitState == kSVSBackwardBranch && pc < shadowSpace->startPC) {
 160         /*
 161          * Consider a trace with a backward branch:
 162          *   1: ..
 163          *   2: ..
 164          *   3: ..
 165          *   4: ..
 166          *   5: Goto {1 or 2 or 3 or 4}
 167          *
 168          * If there instruction 5 goes to 1 and there is no single-step
 169          * instruction in the loop, pc is equal to shadowSpace->startPC and
 170          * we will honor the backward branch condition.
 171          *
 172          * If the single-step instruction is outside the loop, then after
 173          * resuming in the trace the startPC will be less than pc so we will
 174          * also honor the backward branch condition.
 175          *
 176          * If the single-step is inside the loop, we won't hit the same endPC
 177          * twice when the interpreter is re-executing the trace so we want to
 178          * cancel the backward branch condition. In this case it can be
 179          * detected as the endPC (ie pc) will be less than startPC.
 180          */
 181         shadowSpace->selfVerificationState = kSVSNormal;
 182     } else {
 183         shadowSpace->selfVerificationState = exitState;
 184     }
 185
 186     /* Restore state before returning */
 187     self->interpSave.pc = shadowSpace->startPC;
 188     self->interpSave.fp = shadowSpace->fp;
 189     self->interpSave.method = shadowSpace->method;
 190     self->interpSave.methodClassDex = shadowSpace->methodClassDex;
 191     self->retval = shadowSpace->retval;
 192     self->interpStackEnd = shadowSpace->interpStackEnd;
 193
 194     return shadowSpace;
 195 }
 196
 197 /* Print contents of virtual registers */
 198 static void selfVerificationPrintRegisters(int* addr, int* addrRef,
 199                                            int numWords)
 200 {
 201     int i;
 202     for (i = 0; i < numWords; i++) {
 203         LOGD("(v%d) 0x%8x%s", i, addr[i], addr[i] != addrRef[i] ? " X" : "");
 204     }
 205 }
 206
 207 /* Print values maintained in shadowSpace */
 208 static void selfVerificationDumpState(const u2* pc, Thread* self)
 209 {
 210     ShadowSpace* shadowSpace = self->shadowSpace;
 211     StackSaveArea* stackSave = SAVEAREA_FROM_FP(self->curFrame);
 212     int frameBytes = (int) shadowSpace->registerSpace +
 213                      shadowSpace->registerSpaceSize*4 -
 214                      (int) shadowSpace->shadowFP;
 215     int localRegs = 0;
 216     int frameBytes2 = 0;
 217     if ((uintptr_t)self->curFrame < (uintptr_t)shadowSpace->fp) {
 218         localRegs = (stackSave->method->registersSize -
 219                      stackSave->method->insSize)*4;
 220         frameBytes2 = (int) shadowSpace->fp - (int) self->curFrame - localRegs;
 221     }
 222     LOGD("********** SHADOW STATE DUMP **********");
 223     LOGD("CurrentPC: 0x%x, Offset: 0x%04x", (int)pc,
 224         (int)(pc - stackSave->method->insns));
 225     LOGD("Class: %s", shadowSpace->method->clazz->descriptor);
 226     LOGD("Method: %s", shadowSpace->method->name);
 227     LOGD("Dalvik PC: 0x%x endPC: 0x%x", (int)shadowSpace->startPC,
 228         (int)shadowSpace->endPC);
 229     LOGD("Interp FP: 0x%x endFP: 0x%x", (int)shadowSpace->fp,
 230         (int)self->curFrame);
 231     LOGD("Shadow FP: 0x%x endFP: 0x%x", (int)shadowSpace->shadowFP,
 232         (int)shadowSpace->endShadowFP);
 233     LOGD("Frame1 Bytes: %d Frame2 Local: %d Bytes: %d", frameBytes,
 234         localRegs, frameBytes2);
 235     LOGD("Trace length: %d State: %d", shadowSpace->traceLength,
 236         shadowSpace->selfVerificationState);
 237 }
 238
 239 /* Print decoded instructions in the current trace */
 240 static void selfVerificationDumpTrace(const u2* pc, Thread* self)
 241 {
 242     ShadowSpace* shadowSpace = self->shadowSpace;
 243     StackSaveArea* stackSave = SAVEAREA_FROM_FP(self->curFrame);
 244     int i, addr, offset;
 245     DecodedInstruction *decInsn;
 246
 247     LOGD("********** SHADOW TRACE DUMP **********");
 248     for (i = 0; i < shadowSpace->traceLength; i++) {
 249         addr = shadowSpace->trace[i].addr;
 250         offset =  (int)((u2*)addr - stackSave->method->insns);
 251         decInsn = &(shadowSpace->trace[i].decInsn);
 252         /* Not properly decoding instruction, some registers may be garbage */
 253         LOGD("0x%x: (0x%04x) %s",
 254             addr, offset, dexGetOpcodeName(decInsn->opcode));
 255     }
 256 }
 257
 258 /* Code is forced into this spin loop when a divergence is detected */
 259 static void selfVerificationSpinLoop(ShadowSpace *shadowSpace)
 260 {
 261     const u2 *startPC = shadowSpace->startPC;
 262     JitTraceDescription* desc = dvmCopyTraceDescriptor(startPC, NULL);
 263     if (desc) {
 264         dvmCompilerWorkEnqueue(startPC, kWorkOrderTraceDebug, desc);
 265         /*
 266          * This function effectively terminates the VM right here, so not
 267          * freeing the desc pointer when the enqueuing fails is acceptable.
 268          */
 269     }
 270     gDvmJit.selfVerificationSpin = true;
 271     while(gDvmJit.selfVerificationSpin) sleep(10);
 272 }
 273
 274 /*
 275  * If here, we're re-interpreting an instruction that was included
 276  * in a trace that was just executed.  This routine is called for
 277  * each instruction in the original trace, and compares state
 278  * when it reaches the end point.
 279  *
 280  * TUNING: the interpretation mechanism now supports a counted
 281  * single-step mechanism.  If we were to associate an instruction
 282  * count with each trace exit, we could just single-step the right
 283  * number of cycles and then compare.  This would improve detection
 284  * of control divergences, as well as (slightly) simplify this code.
 285  */
 286 void dvmCheckSelfVerification(const u2* pc, Thread* self)
 287 {
 288     ShadowSpace *shadowSpace = self->shadowSpace;
 289     SelfVerificationState state = shadowSpace->selfVerificationState;
 290
 291     DecodedInstruction decInsn;
 292     dexDecodeInstruction(pc, &decInsn);
 293
 294     //LOGD("### DbgIntp(%d): PC: 0x%x endPC: 0x%x state: %d len: %d %s",
 295     //    self->threadId, (int)pc, (int)shadowSpace->endPC, state,
 296     //    shadowSpace->traceLength, dexGetOpcodeName(decInsn.opcode));
 297
 298     if (state == kSVSIdle || state == kSVSStart) {
 299         LOGD("~~~ DbgIntrp: INCORRECT PREVIOUS STATE(%d): %d",
 300             self->threadId, state);
 301         selfVerificationDumpState(pc, self);
 302         selfVerificationDumpTrace(pc, self);
 303     }
 304
 305     /*
 306      * Skip endPC once when trace has a backward branch. If the SV state is
 307      * single step, keep it that way.
 308      */
 309     if ((state == kSVSBackwardBranch && pc == shadowSpace->endPC) ||
 310         (state != kSVSBackwardBranch && state != kSVSSingleStep)) {
 311         shadowSpace->selfVerificationState = kSVSDebugInterp;
 312     }
 313
 314     /* Check that the current pc is the end of the trace */
 315     if ((state == kSVSDebugInterp || state == kSVSSingleStep) &&
 316         pc == shadowSpace->endPC) {
 317
 318         shadowSpace->selfVerificationState = kSVSIdle;
 319
 320         /* Check register space */
 321         int frameBytes = (int) shadowSpace->registerSpace +
 322                          shadowSpace->registerSpaceSize*4 -
 323                          (int) shadowSpace->shadowFP;
 324         if (memcmp(shadowSpace->fp, shadowSpace->shadowFP, frameBytes)) {
 325             LOGD("~~~ DbgIntp(%d): REGISTERS DIVERGENCE!", self->threadId);
 326             selfVerificationDumpState(pc, self);
 327             selfVerificationDumpTrace(pc, self);
 328             LOGD("*** Interp Registers: addr: 0x%x bytes: %d",
 329                 (int)shadowSpace->fp, frameBytes);
 330             selfVerificationPrintRegisters((int*)shadowSpace->fp,
 331                                            (int*)shadowSpace->shadowFP,
 332                                            frameBytes/4);
 333             LOGD("*** Shadow Registers: addr: 0x%x bytes: %d",
 334                 (int)shadowSpace->shadowFP, frameBytes);
 335             selfVerificationPrintRegisters((int*)shadowSpace->shadowFP,
 336                                            (int*)shadowSpace->fp,
 337                                            frameBytes/4);
 338             selfVerificationSpinLoop(shadowSpace);
 339         }
 340         /* Check new frame if it exists (invokes only) */
 341         if ((uintptr_t)self->curFrame < (uintptr_t)shadowSpace->fp) {
 342             StackSaveArea* stackSave = SAVEAREA_FROM_FP(self->curFrame);
 343             int localRegs = (stackSave->method->registersSize -
 344                              stackSave->method->insSize)*4;
 345             int frameBytes2 = (int) shadowSpace->fp -
 346                               (int) self->curFrame - localRegs;
 347             if (memcmp(((char*)self->curFrame)+localRegs,
 348                 ((char*)shadowSpace->endShadowFP)+localRegs, frameBytes2)) {
 349                 LOGD("~~~ DbgIntp(%d): REGISTERS (FRAME2) DIVERGENCE!",
 350                     self->threadId);
 351                 selfVerificationDumpState(pc, self);
 352                 selfVerificationDumpTrace(pc, self);
 353                 LOGD("*** Interp Registers: addr: 0x%x l: %d bytes: %d",
 354                     (int)self->curFrame, localRegs, frameBytes2);
 355                 selfVerificationPrintRegisters((int*)self->curFrame,
 356                                                (int*)shadowSpace->endShadowFP,
 357                                                (frameBytes2+localRegs)/4);
 358                 LOGD("*** Shadow Registers: addr: 0x%x l: %d bytes: %d",
 359                     (int)shadowSpace->endShadowFP, localRegs, frameBytes2);
 360                 selfVerificationPrintRegisters((int*)shadowSpace->endShadowFP,
 361                                                (int*)self->curFrame,
 362                                                (frameBytes2+localRegs)/4);
 363                 selfVerificationSpinLoop(shadowSpace);
 364             }
 365         }
 366
 367         /* Check memory space */
 368         bool memDiff = false;
 369         ShadowHeap* heapSpacePtr;
 370         for (heapSpacePtr = shadowSpace->heapSpace;
 371              heapSpacePtr != shadowSpace->heapSpaceTail; heapSpacePtr++) {
 372             int memData = *((unsigned int*) heapSpacePtr->addr);
 373             if (heapSpacePtr->data != memData) {
 374                 LOGD("~~~ DbgIntp(%d): MEMORY DIVERGENCE!", self->threadId);
 375                 LOGD("Addr: 0x%x Intrp Data: 0x%x Jit Data: 0x%x",
 376                     heapSpacePtr->addr, memData, heapSpacePtr->data);
 377                 selfVerificationDumpState(pc, self);
 378                 selfVerificationDumpTrace(pc, self);
 379                 memDiff = true;
 380             }
 381         }
 382         if (memDiff) selfVerificationSpinLoop(shadowSpace);
 383
 384
 385         /*
 386          * Success.  If this shadowed trace included a single-stepped
 387          * instruction, we need to stay in the interpreter for one
 388          * more interpretation before resuming.
 389          */
 390         if (state == kSVSSingleStep) {
 391             assert(self->jitResumeNPC != NULL);
 392             assert(self->singleStepCount == 0);
 393             self->singleStepCount = 1;
 394             dvmUpdateInterpBreak(self, kInterpSingleStep, kSubModeNormal,
 395                                  true /* enable */);
 396         }
 397
 398         /*
 399          * Switch off shadow replay mode.  The next shadowed trace
 400          * execution will turn it back on.
 401          */
 402         dvmUpdateInterpBreak(self, kInterpJitBreak, kSubModeJitSV,
 403                              false /* disable */);
 404         self->jitState = kJitDone;
 405         return;
 406
 407     /* If end not been reached, make sure max length not exceeded */
 408     } else if (shadowSpace->traceLength >= JIT_MAX_TRACE_LEN) {
 409         LOGD("~~~ DbgIntp(%d): CONTROL DIVERGENCE!", self->threadId);
 410         LOGD("startPC: 0x%x endPC: 0x%x currPC: 0x%x",
 411             (int)shadowSpace->startPC, (int)shadowSpace->endPC, (int)pc);
 412         selfVerificationDumpState(pc, self);
 413         selfVerificationDumpTrace(pc, self);
 414         selfVerificationSpinLoop(shadowSpace);
 415
 416         return;
 417     }
 418     /* Log the instruction address and decoded instruction for debug */
 419     shadowSpace->trace[shadowSpace->traceLength].addr = (int)pc;
 420     shadowSpace->trace[shadowSpace->traceLength].decInsn = decInsn;
 421     shadowSpace->traceLength++;
 422 }
 423 #endif
 424
 425 /*
 426  * If one of our fixed tables or the translation buffer fills up,
 427  * call this routine to avoid wasting cycles on future translation requests.
 428  */
 429 void dvmJitStopTranslationRequests()
 430 {
 431     /*
 432      * Note 1: This won't necessarily stop all translation requests, and
 433      * operates on a delayed mechanism.  Running threads look to the copy
 434      * of this value in their private thread structures and won't see
 435      * this change until it is refreshed (which happens on interpreter
 436      * entry).
 437      * Note 2: This is a one-shot memory leak on this table. Because this is a
 438      * permanent off switch for Jit profiling, it is a one-time leak of 1K
 439      * bytes, and no further attempt will be made to re-allocate it.  Can't
 440      * free it because some thread may be holding a reference.
 441      */
 442     gDvmJit.pProfTable = NULL;
 443     dvmJitUpdateThreadStateAll();
 444 }
 445
 446 #if defined(WITH_JIT_TUNING)
 447 /* Convenience function to increment counter from assembly code */
 448 void dvmBumpNoChain(int from)
 449 {
 450     gDvmJit.noChainExit[from]++;
 451 }
 452
 453 /* Convenience function to increment counter from assembly code */
 454 void dvmBumpNormal()
 455 {
 456     gDvmJit.normalExit++;
 457 }
 458
 459 /* Convenience function to increment counter from assembly code */
 460 void dvmBumpPunt(int from)
 461 {
 462     gDvmJit.puntExit++;
 463 }
 464 #endif
 465
 466 /* Dumps debugging & tuning stats to the log */
 467 void dvmJitStats()
 468 {
 469     int i;
 470     int hit;
 471     int not_hit;
 472     int chains;
 473     int stubs;
 474     if (gDvmJit.pJitEntryTable) {
 475         for (i=0, stubs=chains=hit=not_hit=0;
 476              i < (int) gDvmJit.jitTableSize;
 477              i++) {
 478             if (gDvmJit.pJitEntryTable[i].dPC != 0) {
 479                 hit++;
 480                 if (gDvmJit.pJitEntryTable[i].codeAddress ==
 481                       dvmCompilerGetInterpretTemplate())
 482                     stubs++;
 483             } else
 484                 not_hit++;
 485             if (gDvmJit.pJitEntryTable[i].u.info.chain != gDvmJit.jitTableSize)
 486                 chains++;
 487         }
 488         LOGD("JIT: table size is %d, entries used is %d",
 489              gDvmJit.jitTableSize,  gDvmJit.jitTableEntriesUsed);
 490         LOGD("JIT: %d traces, %d slots, %d chains, %d thresh, %s",
 491              hit, not_hit + hit, chains, gDvmJit.threshold,
 492              gDvmJit.blockingMode ? "Blocking" : "Non-blocking");
 493
 494 #if defined(WITH_JIT_TUNING)
 495         LOGD("JIT: Code cache patches: %d", gDvmJit.codeCachePatches);
 496
 497         LOGD("JIT: Lookups: %d hits, %d misses; %d normal, %d punt",
 498              gDvmJit.addrLookupsFound, gDvmJit.addrLookupsNotFound,
 499              gDvmJit.normalExit, gDvmJit.puntExit);
 500
 501         LOGD("JIT: ICHits: %d", gDvmICHitCount);
 502
 503         LOGD("JIT: noChainExit: %d IC miss, %d interp callsite, "
 504              "%d switch overflow",
 505              gDvmJit.noChainExit[kInlineCacheMiss],
 506              gDvmJit.noChainExit[kCallsiteInterpreted],
 507              gDvmJit.noChainExit[kSwitchOverflow]);
 508
 509         LOGD("JIT: ICPatch: %d init, %d rejected, %d lock-free, %d queued, "
 510              "%d dropped",
 511              gDvmJit.icPatchInit, gDvmJit.icPatchRejected,
 512              gDvmJit.icPatchLockFree, gDvmJit.icPatchQueued,
 513              gDvmJit.icPatchDropped);
 514
 515         LOGD("JIT: Invoke: %d mono, %d poly, %d native, %d return",
 516              gDvmJit.invokeMonomorphic, gDvmJit.invokePolymorphic,
 517              gDvmJit.invokeNative, gDvmJit.returnOp);
 518         LOGD("JIT: Inline: %d mgetter, %d msetter, %d pgetter, %d psetter",
 519              gDvmJit.invokeMonoGetterInlined, gDvmJit.invokeMonoSetterInlined,
 520              gDvmJit.invokePolyGetterInlined, gDvmJit.invokePolySetterInlined);
 521         LOGD("JIT: Total compilation time: %llu ms", gDvmJit.jitTime / 1000);
 522         LOGD("JIT: Avg unit compilation time: %llu us",
 523              gDvmJit.numCompilations == 0 ? 0 :
 524              gDvmJit.jitTime / gDvmJit.numCompilations);
 525         LOGD("JIT: Potential GC blocked by compiler: max %llu us / "
 526              "avg %llu us (%d)",
 527              gDvmJit.maxCompilerThreadBlockGCTime,
 528              gDvmJit.numCompilerThreadBlockGC == 0 ?
 529                  0 : gDvmJit.compilerThreadBlockGCTime /
 530                      gDvmJit.numCompilerThreadBlockGC,
 531              gDvmJit.numCompilerThreadBlockGC);
 532 #endif
 533
 534         LOGD("JIT: %d Translation chains, %d interp stubs",
 535              gDvmJit.translationChains, stubs);
 536         if (gDvmJit.profileMode == kTraceProfilingContinuous) {
 537             dvmCompilerSortAndPrintTraceProfiles();
 538         }
 539     }
 540 }
 541
 542
 543 /* End current trace now & don't include current instruction */
 544 void dvmJitEndTraceSelect(Thread* self, const u2* dPC)
 545 {
 546     if (self->jitState == kJitTSelect) {
 547         self->jitState = kJitTSelectEnd;
 548     }
 549     if (self->jitState == kJitTSelectEnd) {
 550         // Clean up and finish now.
 551         dvmCheckJit(dPC, self);
 552     }
 553 }
 554
 555 /*
 556  * Find an entry in the JitTable, creating if necessary.
 557  * Returns null if table is full.
 558  */
 559 static JitEntry *lookupAndAdd(const u2* dPC, bool callerLocked,
 560                               bool isMethodEntry)
 561 {
 562     u4 chainEndMarker = gDvmJit.jitTableSize;
 563     u4 idx = dvmJitHash(dPC);
 564
 565     /*
 566      * Walk the bucket chain to find an exact match for our PC and trace/method
 567      * type
 568      */
 569     while ((gDvmJit.pJitEntryTable[idx].u.info.chain != chainEndMarker) &&
 570            ((gDvmJit.pJitEntryTable[idx].dPC != dPC) ||
 571             (gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry !=
 572              isMethodEntry))) {
 573         idx = gDvmJit.pJitEntryTable[idx].u.info.chain;
 574     }
 575
 576     if (gDvmJit.pJitEntryTable[idx].dPC != dPC ||
 577         gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry != isMethodEntry) {
 578         /*
 579          * No match.  Aquire jitTableLock and find the last
 580          * slot in the chain. Possibly continue the chain walk in case
 581          * some other thread allocated the slot we were looking
 582          * at previuosly (perhaps even the dPC we're trying to enter).
 583          */
 584         if (!callerLocked)
 585             dvmLockMutex(&gDvmJit.tableLock);
 586         /*
 587          * At this point, if .dPC is NULL, then the slot we're
 588          * looking at is the target slot from the primary hash
 589          * (the simple, and common case).  Otherwise we're going
 590          * to have to find a free slot and chain it.
 591          */
 592         ANDROID_MEMBAR_FULL(); /* Make sure we reload [].dPC after lock */
 593         if (gDvmJit.pJitEntryTable[idx].dPC != NULL) {
 594             u4 prev;
 595             while (gDvmJit.pJitEntryTable[idx].u.info.chain != chainEndMarker) {
 596                 if (gDvmJit.pJitEntryTable[idx].dPC == dPC &&
 597                     gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry ==
 598                         isMethodEntry) {
 599                     /* Another thread got there first for this dPC */
 600                     if (!callerLocked)
 601                         dvmUnlockMutex(&gDvmJit.tableLock);
 602                     return &gDvmJit.pJitEntryTable[idx];
 603                 }
 604                 idx = gDvmJit.pJitEntryTable[idx].u.info.chain;
 605             }
 606             /* Here, idx should be pointing to the last cell of an
 607              * active chain whose last member contains a valid dPC */
 608             assert(gDvmJit.pJitEntryTable[idx].dPC != NULL);
 609             /* Linear walk to find a free cell and add it to the end */
 610             prev = idx;
 611             while (true) {
 612                 idx++;
 613                 if (idx == chainEndMarker)
 614                     idx = 0;  /* Wraparound */
 615                 if ((gDvmJit.pJitEntryTable[idx].dPC == NULL) ||
 616                     (idx == prev))
 617                     break;
 618             }
 619             if (idx != prev) {
 620                 JitEntryInfoUnion oldValue;
 621                 JitEntryInfoUnion newValue;
 622                 /*
 623                  * Although we hold the lock so that noone else will
 624                  * be trying to update a chain field, the other fields
 625                  * packed into the word may be in use by other threads.
 626                  */
 627                 do {
 628                     oldValue = gDvmJit.pJitEntryTable[prev].u;
 629                     newValue = oldValue;
 630                     newValue.info.chain = idx;
 631                 } while (android_atomic_release_cas(oldValue.infoWord,
 632                         newValue.infoWord,
 633                         &gDvmJit.pJitEntryTable[prev].u.infoWord) != 0);
 634             }
 635         }
 636         if (gDvmJit.pJitEntryTable[idx].dPC == NULL) {
 637             gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry = isMethodEntry;
 638             /*
 639              * Initialize codeAddress and allocate the slot.  Must
 640              * happen in this order (since dPC is set, the entry is live.
 641              */
 642             android_atomic_release_store((int32_t)dPC,
 643                  (volatile int32_t *)(void *)&gDvmJit.pJitEntryTable[idx].dPC);
 644             gDvmJit.pJitEntryTable[idx].dPC = dPC;
 645             gDvmJit.jitTableEntriesUsed++;
 646         } else {
 647             /* Table is full */
 648             idx = chainEndMarker;
 649         }
 650         if (!callerLocked)
 651             dvmUnlockMutex(&gDvmJit.tableLock);
 652     }
 653     return (idx == chainEndMarker) ? NULL : &gDvmJit.pJitEntryTable[idx];
 654 }
 655
 656 /* Dump a trace description */
 657 void dvmJitDumpTraceDesc(JitTraceDescription *trace)
 658 {
 659     int i;
 660     bool done = false;
 661     const u2* dpc;
 662     const u2* dpcBase;
 663     int curFrag = 0;
 664     LOGD("===========================================");
 665     LOGD("Trace dump 0x%x, Method %s off 0x%x",(int)trace,
 666          trace->method->name,trace->trace[curFrag].info.frag.startOffset);
 667     dpcBase = trace->method->insns;
 668     while (!done) {
 669         DecodedInstruction decInsn;
 670         if (trace->trace[curFrag].isCode) {
 671             LOGD("Frag[%d]- Insts: %d, start: 0x%x, hint: 0x%x, end: %d",
 672                  curFrag, trace->trace[curFrag].info.frag.numInsts,
 673                  trace->trace[curFrag].info.frag.startOffset,
 674                  trace->trace[curFrag].info.frag.hint,
 675                  trace->trace[curFrag].info.frag.runEnd);
 676             dpc = dpcBase + trace->trace[curFrag].info.frag.startOffset;
 677             for (i=0; i<trace->trace[curFrag].info.frag.numInsts; i++) {
 678                 dexDecodeInstruction(dpc, &decInsn);
 679                 LOGD("    0x%04x - %s 0x%x",(dpc-dpcBase),
 680                      dexGetOpcodeName(decInsn.opcode),(int)dpc);
 681                 dpc += dexGetWidthFromOpcode(decInsn.opcode);
 682             }
 683             if (trace->trace[curFrag].info.frag.runEnd) {
 684                 done = true;
 685             }
 686         } else {
 687             LOGD("Frag[%d]- META info: 0x%08x", curFrag,
 688                  (int)trace->trace[curFrag].info.meta);
 689         }
 690         curFrag++;
 691     }
 692     LOGD("-------------------------------------------");
 693 }
 694
 695 /*
 696  * Append the class ptr of "this" and the current method ptr to the current
 697  * trace. That is, the trace runs will contain the following components:
 698  *  + trace run that ends with an invoke (existing entry)
 699  *  + thisClass (new)
 700  *  + calleeMethod (new)
 701  */
 702 static void insertClassMethodInfo(Thread* self,
 703                                   const ClassObject* thisClass,
 704                                   const Method* calleeMethod,
 705                                   const DecodedInstruction* insn)
 706 {
 707     int currTraceRun = ++self->currTraceRun;
 708     self->trace[currTraceRun].info.meta = thisClass ?
 709                                     (void *) thisClass->descriptor : NULL;
 710     self->trace[currTraceRun].isCode = false;
 711
 712     currTraceRun = ++self->currTraceRun;
 713     self->trace[currTraceRun].info.meta = thisClass ?
 714                                     (void *) thisClass->classLoader : NULL;
 715     self->trace[currTraceRun].isCode = false;
 716
 717     currTraceRun = ++self->currTraceRun;
 718     self->trace[currTraceRun].info.meta = (void *) calleeMethod;
 719     self->trace[currTraceRun].isCode = false;
 720 }
 721
 722 /*
 723  * Check if the next instruction following the invoke is a move-result and if
 724  * so add it to the trace. That is, this will add the trace run that includes
 725  * the move-result to the trace list.
 726  *
 727  *  + trace run that ends with an invoke (existing entry)
 728  *  + thisClass (existing entry)
 729  *  + calleeMethod (existing entry)
 730  *  + move result (new)
 731  *
 732  * lastPC, len, offset are all from the preceding invoke instruction
 733  */
 734 static void insertMoveResult(const u2 *lastPC, int len, int offset,
 735                              Thread *self)
 736 {
 737     DecodedInstruction nextDecInsn;
 738     const u2 *moveResultPC = lastPC + len;
 739
 740     dexDecodeInstruction(moveResultPC, &nextDecInsn);
 741     if ((nextDecInsn.opcode != OP_MOVE_RESULT) &&
 742         (nextDecInsn.opcode != OP_MOVE_RESULT_WIDE) &&
 743         (nextDecInsn.opcode != OP_MOVE_RESULT_OBJECT))
 744         return;
 745
 746     /* We need to start a new trace run */
 747     int currTraceRun = ++self->currTraceRun;
 748     self->currRunHead = moveResultPC;
 749     self->trace[currTraceRun].info.frag.startOffset = offset + len;
 750     self->trace[currTraceRun].info.frag.numInsts = 1;
 751     self->trace[currTraceRun].info.frag.runEnd = false;
 752     self->trace[currTraceRun].info.frag.hint = kJitHintNone;
 753     self->trace[currTraceRun].isCode = true;
 754     self->totalTraceLen++;
 755
 756     self->currRunLen = dexGetWidthFromInstruction(moveResultPC);
 757 }
 758
 759 /*
 760  * Adds to the current trace request one instruction at a time, just
 761  * before that instruction is interpreted.  This is the primary trace
 762  * selection function.  NOTE: return instruction are handled a little
 763  * differently.  In general, instructions are "proposed" to be added
 764  * to the current trace prior to interpretation.  If the interpreter
 765  * then successfully completes the instruction, is will be considered
 766  * part of the request.  This allows us to examine machine state prior
 767  * to interpretation, and also abort the trace request if the instruction
 768  * throws or does something unexpected.  However, return instructions
 769  * will cause an immediate end to the translation request - which will
 770  * be passed to the compiler before the return completes.  This is done
 771  * in response to special handling of returns by the interpreter (and
 772  * because returns cannot throw in a way that causes problems for the
 773  * translated code.
 774  */
 775 void dvmCheckJit(const u2* pc, Thread* self)
 776 {
 777     const ClassObject *thisClass = self->callsiteClass;
 778     const Method* curMethod = self->methodToCall;
 779     int flags, len;
 780     int allDone = false;
 781     /* Stay in break/single-stop mode for the next instruction */
 782     bool stayOneMoreInst = false;
 783
 784     /* Prepare to handle last PC and stage the current PC & method*/
 785     const u2 *lastPC = self->lastPC;
 786
 787     self->lastPC = pc;
 788
 789     switch (self->jitState) {
 790         int offset;
 791         DecodedInstruction decInsn;
 792         case kJitTSelect:
 793             /* First instruction - just remember the PC and exit */
 794             if (lastPC == NULL) break;
 795             /* Grow the trace around the last PC if jitState is kJitTSelect */
 796             dexDecodeInstruction(lastPC, &decInsn);
 797
 798             /*
 799              * Treat {PACKED,SPARSE}_SWITCH as trace-ending instructions due
 800              * to the amount of space it takes to generate the chaining
 801              * cells.
 802              */
 803             if (self->totalTraceLen != 0 &&
 804                 (decInsn.opcode == OP_PACKED_SWITCH ||
 805                  decInsn.opcode == OP_SPARSE_SWITCH)) {
 806                 self->jitState = kJitTSelectEnd;
 807                 break;
 808             }
 809
 810 #if defined(SHOW_TRACE)
 811             LOGD("TraceGen: adding %s. lpc:0x%x, pc:0x%x",
 812                  dexGetOpcodeName(decInsn.opcode), (int)lastPC, (int)pc);
 813 #endif
 814             flags = dexGetFlagsFromOpcode(decInsn.opcode);
 815             len = dexGetWidthFromInstruction(lastPC);
 816             offset = lastPC - self->traceMethod->insns;
 817             assert((unsigned) offset <
 818                    dvmGetMethodInsnsSize(self->traceMethod));
 819             if (lastPC != self->currRunHead + self->currRunLen) {
 820                 int currTraceRun;
 821                 /* We need to start a new trace run */
 822                 currTraceRun = ++self->currTraceRun;
 823                 self->currRunLen = 0;
 824                 self->currRunHead = (u2*)lastPC;
 825                 self->trace[currTraceRun].info.frag.startOffset = offset;
 826                 self->trace[currTraceRun].info.frag.numInsts = 0;
 827                 self->trace[currTraceRun].info.frag.runEnd = false;
 828                 self->trace[currTraceRun].info.frag.hint = kJitHintNone;
 829                 self->trace[currTraceRun].isCode = true;
 830             }
 831             self->trace[self->currTraceRun].info.frag.numInsts++;
 832             self->totalTraceLen++;
 833             self->currRunLen += len;
 834
 835             /*
 836              * If the last instruction is an invoke, we will try to sneak in
 837              * the move-result* (if existent) into a separate trace run.
 838              */
 839             int needReservedRun = (flags & kInstrInvoke) ? 1 : 0;
 840
 841             /* Will probably never hit this with the current trace buildier */
 842             if (self->currTraceRun ==
 843                 (MAX_JIT_RUN_LEN - 1 - needReservedRun)) {
 844                 self->jitState = kJitTSelectEnd;
 845             }
 846
 847             if (!dexIsGoto(flags) &&
 848                   ((flags & (kInstrCanBranch |
 849                              kInstrCanSwitch |
 850                              kInstrCanReturn |
 851                              kInstrInvoke)) != 0)) {
 852                     self->jitState = kJitTSelectEnd;
 853 #if defined(SHOW_TRACE)
 854                 LOGD("TraceGen: ending on %s, basic block end",
 855                      dexGetOpcodeName(decInsn.opcode));
 856 #endif
 857
 858                 /*
 859                  * If the current invoke is a {virtual,interface}, get the
 860                  * current class/method pair into the trace as well.
 861                  * If the next instruction is a variant of move-result, insert
 862                  * it to the trace too.
 863                  */
 864                 if (flags & kInstrInvoke) {
 865                     insertClassMethodInfo(self, thisClass, curMethod,
 866                                           &decInsn);
 867                     insertMoveResult(lastPC, len, offset, self);
 868                 }
 869             }
 870             /* Break on throw or self-loop */
 871             if ((decInsn.opcode == OP_THROW) || (lastPC == pc)){
 872                 self->jitState = kJitTSelectEnd;
 873             }
 874             if (self->totalTraceLen >= JIT_MAX_TRACE_LEN) {
 875                 self->jitState = kJitTSelectEnd;
 876             }
 877             if ((flags & kInstrCanReturn) != kInstrCanReturn) {
 878                 break;
 879             }
 880             else {
 881                 /*
 882                  * Last instruction is a return - stay in the dbg interpreter
 883                  * for one more instruction if it is a non-void return, since
 884                  * we don't want to start a trace with move-result as the first
 885                  * instruction (which is already included in the trace
 886                  * containing the invoke.
 887                  */
 888                 if (decInsn.opcode != OP_RETURN_VOID) {
 889                     stayOneMoreInst = true;
 890                 }
 891             }
 892             /* NOTE: intentional fallthrough for returns */
 893         case kJitTSelectEnd:
 894             {
 895                 /* Empty trace - set to bail to interpreter */
 896                 if (self->totalTraceLen == 0) {
 897                     dvmJitSetCodeAddr(self->currTraceHead,
 898                                       dvmCompilerGetInterpretTemplate(),
 899                                       dvmCompilerGetInterpretTemplateSet(),
 900                                       false /* Not method entry */, 0);
 901                     self->jitState = kJitDone;
 902                     allDone = true;
 903                     break;
 904                 }
 905
 906                 int lastTraceDesc = self->currTraceRun;
 907
 908                 /* Extend a new empty desc if the last slot is meta info */
 909                 if (!self->trace[lastTraceDesc].isCode) {
 910                     lastTraceDesc = ++self->currTraceRun;
 911                     self->trace[lastTraceDesc].info.frag.startOffset = 0;
 912                     self->trace[lastTraceDesc].info.frag.numInsts = 0;
 913                     self->trace[lastTraceDesc].info.frag.hint = kJitHintNone;
 914                     self->trace[lastTraceDesc].isCode = true;
 915                 }
 916
 917                 /* Mark the end of the trace runs */
 918                 self->trace[lastTraceDesc].info.frag.runEnd = true;
 919
 920                 JitTraceDescription* desc =
 921                    (JitTraceDescription*)malloc(sizeof(JitTraceDescription) +
 922                      sizeof(JitTraceRun) * (self->currTraceRun+1));
 923
 924                 if (desc == NULL) {
 925                     LOGE("Out of memory in trace selection");
 926                     dvmJitStopTranslationRequests();
 927                     self->jitState = kJitDone;
 928                     allDone = true;
 929                     break;
 930                 }
 931
 932                 desc->method = self->traceMethod;
 933                 memcpy((char*)&(desc->trace[0]),
 934                     (char*)&(self->trace[0]),
 935                     sizeof(JitTraceRun) * (self->currTraceRun+1));
 936 #if defined(SHOW_TRACE)
 937                 LOGD("TraceGen:  trace done, adding to queue");
 938                 dvmJitDumpTraceDesc(desc);
 939 #endif
 940                 if (dvmCompilerWorkEnqueue(
 941                        self->currTraceHead,kWorkOrderTrace,desc)) {
 942                     /* Work order successfully enqueued */
 943                     if (gDvmJit.blockingMode) {
 944                         dvmCompilerDrainQueue();
 945                     }
 946                 } else {
 947                     /*
 948                      * Make sure the descriptor for the abandoned work order is
 949                      * freed.
 950                      */
 951                     free(desc);
 952                 }
 953                 self->jitState = kJitDone;
 954                 allDone = true;
 955             }
 956             break;
 957         case kJitDone:
 958             allDone = true;
 959             break;
 960         case kJitNot:
 961             allDone = true;
 962             break;
 963         default:
 964             LOGE("Unexpected JIT state: %d", self->jitState);
 965             dvmAbort();
 966             break;
 967     }
 968
 969     /*
 970      * If we're done with trace selection, switch off the control flags.
 971      */
 972      if (allDone) {
 973          dvmUpdateInterpBreak(self, kInterpJitBreak,
 974                               kSubModeJitTraceBuild, false);
 975          if (stayOneMoreInst) {
 976              // Keep going in single-step mode for at least one more inst
 977              assert(self->jitResumeNPC == NULL);
 978              self->singleStepCount = MIN(1, self->singleStepCount);
 979              dvmUpdateInterpBreak(self, kInterpSingleStep, kSubModeNormal,
 980                                   true /* enable */);
 981          }
 982      }
 983      return;
 984 }
 985
 986 JitEntry *dvmJitFindEntry(const u2* pc, bool isMethodEntry)
 987 {
 988     int idx = dvmJitHash(pc);
 989
 990     /* Expect a high hit rate on 1st shot */
 991     if ((gDvmJit.pJitEntryTable[idx].dPC == pc) &&
 992         (gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry == isMethodEntry))
 993         return &gDvmJit.pJitEntryTable[idx];
 994     else {
 995         int chainEndMarker = gDvmJit.jitTableSize;
 996         while (gDvmJit.pJitEntryTable[idx].u.info.chain != chainEndMarker) {
 997             idx = gDvmJit.pJitEntryTable[idx].u.info.chain;
 998             if ((gDvmJit.pJitEntryTable[idx].dPC == pc) &&
 999                 (gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry ==
1000                 isMethodEntry))
1001                 return &gDvmJit.pJitEntryTable[idx];
1002         }
1003     }
1004     return NULL;
1005 }
1006
1007 /*
1008  * Walk through the JIT profile table and find the corresponding JIT code, in
1009  * the specified format (ie trace vs method). This routine needs to be fast.
1010  */
1011 void* getCodeAddrCommon(const u2* dPC, bool methodEntry)
1012 {
1013     int idx = dvmJitHash(dPC);
1014     const u2* pc = gDvmJit.pJitEntryTable[idx].dPC;
1015     if (pc != NULL) {
1016         bool hideTranslation = dvmJitHideTranslation();
1017         if (pc == dPC &&
1018             gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry == methodEntry) {
1019             int offset = (gDvmJit.profileMode >= kTraceProfilingContinuous) ?
1020                  0 : gDvmJit.pJitEntryTable[idx].u.info.profileOffset;
1021             intptr_t codeAddress =
1022                 (intptr_t)gDvmJit.pJitEntryTable[idx].codeAddress;
1023 #if defined(WITH_JIT_TUNING)
1024             gDvmJit.addrLookupsFound++;
1025 #endif
1026             return hideTranslation || !codeAddress ?  NULL :
1027                   (void *)(codeAddress + offset);
1028         } else {
1029             int chainEndMarker = gDvmJit.jitTableSize;
1030             while (gDvmJit.pJitEntryTable[idx].u.info.chain != chainEndMarker) {
1031                 idx = gDvmJit.pJitEntryTable[idx].u.info.chain;
1032                 if (gDvmJit.pJitEntryTable[idx].dPC == dPC &&
1033                     gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry ==
1034                         methodEntry) {
1035                     int offset = (gDvmJit.profileMode >=
1036                         kTraceProfilingContinuous) ? 0 :
1037                         gDvmJit.pJitEntryTable[idx].u.info.profileOffset;
1038                     intptr_t codeAddress =
1039                         (intptr_t)gDvmJit.pJitEntryTable[idx].codeAddress;
1040 #if defined(WITH_JIT_TUNING)
1041                     gDvmJit.addrLookupsFound++;
1042 #endif
1043                     return hideTranslation || !codeAddress ? NULL :
1044                         (void *)(codeAddress + offset);
1045                 }
1046             }
1047         }
1048     }
1049 #if defined(WITH_JIT_TUNING)
1050     gDvmJit.addrLookupsNotFound++;
1051 #endif
1052     return NULL;
1053 }
1054
1055 /*
1056  * If a translated code address, in trace format, exists for the davik byte code
1057  * pointer return it.
1058  */
1059 void* dvmJitGetTraceAddr(const u2* dPC)
1060 {
1061     return getCodeAddrCommon(dPC, false /* method entry */);
1062 }
1063
1064 /*
1065  * If a translated code address, in whole-method format, exists for the davik
1066  * byte code pointer return it.
1067  */
1068 void* dvmJitGetMethodAddr(const u2* dPC)
1069 {
1070     return getCodeAddrCommon(dPC, true /* method entry */);
1071 }
1072
1073 /*
1074  * Similar to dvmJitGetTraceAddr, but returns null if the calling
1075  * thread is in a single-step mode.
1076  */
1077 void* dvmJitGetTraceAddrThread(const u2* dPC, Thread* self)
1078 {
1079     return (self->interpBreak.ctl.breakFlags != 0) ? NULL :
1080             getCodeAddrCommon(dPC, false /* method entry */);
1081 }
1082
1083 /*
1084  * Similar to dvmJitGetMethodAddr, but returns null if the calling
1085  * thread is in a single-step mode.
1086  */
1087 void* dvmJitGetMethodAddrThread(const u2* dPC, Thread* self)
1088 {
1089     return (self->interpBreak.ctl.breakFlags != 0) ? NULL :
1090             getCodeAddrCommon(dPC, true /* method entry */);
1091 }
1092
1093 /*
1094  * Register the translated code pointer into the JitTable.
1095  * NOTE: Once a codeAddress field transitions from initial state to
1096  * JIT'd code, it must not be altered without first halting all
1097  * threads.  We defer the setting of the profile prefix size until
1098  * after the new code address is set to ensure that the prefix offset
1099  * is never applied to the initial interpret-only translation.  All
1100  * translations with non-zero profile prefixes will still be correct
1101  * if entered as if the profile offset is 0, but the interpret-only
1102  * template cannot handle a non-zero prefix.
1103  * NOTE: JitTable must not be in danger of reset while this
1104  * code is executing. see Issue 4271784 for details.
1105  */
1106 void dvmJitSetCodeAddr(const u2* dPC, void *nPC, JitInstructionSetType set,
1107                        bool isMethodEntry, int profilePrefixSize)
1108 {
1109     JitEntryInfoUnion oldValue;
1110     JitEntryInfoUnion newValue;
1111     /*
1112      * Get the JitTable slot for this dPC (or create one if JitTable
1113      * has been reset between the time the trace was requested and
1114      * now.
1115      */
1116     JitEntry *jitEntry = isMethodEntry ?
1117         lookupAndAdd(dPC, false /* caller holds tableLock */, isMethodEntry) :
1118                      dvmJitFindEntry(dPC, isMethodEntry);
1119     assert(jitEntry);
1120     /* Note: order of update is important */
1121     do {
1122         oldValue = jitEntry->u;
1123         newValue = oldValue;
1124         newValue.info.isMethodEntry = isMethodEntry;
1125         newValue.info.instructionSet = set;
1126         newValue.info.profileOffset = profilePrefixSize;
1127     } while (android_atomic_release_cas(
1128              oldValue.infoWord, newValue.infoWord,
1129              &jitEntry->u.infoWord) != 0);
1130     jitEntry->codeAddress = nPC;
1131 }
1132
1133 /*
1134  * Determine if valid trace-bulding request is active.  If so, set
1135  * the proper flags in interpBreak and return.  Trace selection will
1136  * then begin normally via dvmCheckBefore.
1137  */
1138 void dvmJitCheckTraceRequest(Thread* self)
1139 {
1140     int i;
1141     /*
1142      * A note on trace "hotness" filtering:
1143      *
1144      * Our first level trigger is intentionally loose - we need it to
1145      * fire easily not just to identify potential traces to compile, but
1146      * also to allow re-entry into the code cache.
1147      *
1148      * The 2nd level filter (done here) exists to be selective about
1149      * what we actually compile.  It works by requiring the same
1150      * trace head "key" (defined as filterKey below) to appear twice in
1151      * a relatively short period of time.   The difficulty is defining the
1152      * shape of the filterKey.  Unfortunately, there is no "one size fits
1153      * all" approach.
1154      *
1155      * For spiky execution profiles dominated by a smallish
1156      * number of very hot loops, we would want the second-level filter
1157      * to be very selective.  A good selective filter is requiring an
1158      * exact match of the Dalvik PC.  In other words, defining filterKey as:
1159      *     intptr_t filterKey = (intptr_t)self->interpSave.pc
1160      *
1161      * However, for flat execution profiles we do best when aggressively
1162      * translating.  A heuristically decent proxy for this is to use
1163      * the value of the method pointer containing the trace as the filterKey.
1164      * Intuitively, this is saying that once any trace in a method appears hot,
1165      * immediately translate any other trace from that same method that
1166      * survives the first-level filter.  Here, filterKey would be defined as:
1167      *     intptr_t filterKey = (intptr_t)self->interpSave.method
1168      *
1169      * The problem is that we can't easily detect whether we're dealing
1170      * with a spiky or flat profile.  If we go with the "pc" match approach,
1171      * flat profiles perform poorly.  If we go with the loose "method" match,
1172      * we end up generating a lot of useless translations.  Probably the
1173      * best approach in the future will be to retain profile information
1174      * across runs of each application in order to determine it's profile,
1175      * and then choose once we have enough history.
1176      *
1177      * However, for now we've decided to chose a compromise filter scheme that
1178      * includes elements of both.  The high order bits of the filter key
1179      * are drawn from the enclosing method, and are combined with a slice
1180      * of the low-order bits of the Dalvik pc of the trace head.  The
1181      * looseness of the filter can be adjusted by changing with width of
1182      * the Dalvik pc slice (JIT_TRACE_THRESH_FILTER_PC_BITS).  The wider
1183      * the slice, the tighter the filter.
1184      *
1185      * Note: the fixed shifts in the function below reflect assumed word
1186      * alignment for method pointers, and half-word alignment of the Dalvik pc.
1187      * for method pointers and half-word alignment for dalvik pc.
1188      */
1189     u4 methodKey = (u4)self->interpSave.method <<
1190                    (JIT_TRACE_THRESH_FILTER_PC_BITS - 2);
1191     u4 pcKey = ((u4)self->interpSave.pc >> 1) &
1192                ((1 << JIT_TRACE_THRESH_FILTER_PC_BITS) - 1);
1193     intptr_t filterKey = (intptr_t)(methodKey | pcKey);
1194
1195     // Shouldn't be here if already building a trace.
1196     assert((self->interpBreak.ctl.subMode & kSubModeJitTraceBuild)==0);
1197
1198     /* Check if the JIT request can be handled now */
1199     if ((gDvmJit.pJitEntryTable != NULL) &&
1200         ((self->interpBreak.ctl.breakFlags & kInterpSingleStep) == 0)){
1201         /* Bypass the filter for hot trace requests or during stress mode */
1202         if (self->jitState == kJitTSelectRequest &&
1203             gDvmJit.threshold > 6) {
1204             /* Two-level filtering scheme */
1205             for (i=0; i< JIT_TRACE_THRESH_FILTER_SIZE; i++) {
1206                 if (filterKey == self->threshFilter[i]) {
1207                     self->threshFilter[i] = 0; // Reset filter entry
1208                     break;
1209                 }
1210             }
1211             if (i == JIT_TRACE_THRESH_FILTER_SIZE) {
1212                 /*
1213                  * Use random replacement policy - otherwise we could miss a
1214                  * large loop that contains more traces than the size of our
1215                  * filter array.
1216                  */
1217                 i = rand() % JIT_TRACE_THRESH_FILTER_SIZE;
1218                 self->threshFilter[i] = filterKey;
1219                 self->jitState = kJitDone;
1220             }
1221         }
1222
1223         /* If the compiler is backlogged, cancel any JIT actions */
1224         if (gDvmJit.compilerQueueLength >= gDvmJit.compilerHighWater) {
1225             self->jitState = kJitDone;
1226         }
1227
1228         /*
1229          * Check for additional reasons that might force the trace select
1230          * request to be dropped
1231          */
1232         if (self->jitState == kJitTSelectRequest ||
1233             self->jitState == kJitTSelectRequestHot) {
1234             if (dvmJitFindEntry(self->interpSave.pc, false)) {
1235                 /* In progress - nothing do do */
1236                self->jitState = kJitDone;
1237             } else {
1238                 JitEntry *slot = lookupAndAdd(self->interpSave.pc,
1239                                               false /* lock */,
1240                                               false /* method entry */);
1241                 if (slot == NULL) {
1242                     /*
1243                      * Table is full.  This should have been
1244                      * detected by the compiler thread and the table
1245                      * resized before we run into it here.  Assume bad things
1246                      * are afoot and disable profiling.
1247                      */
1248                     self->jitState = kJitDone;
1249                     LOGD("JIT: JitTable full, disabling profiling");
1250                     dvmJitStopTranslationRequests();
1251                 }
1252             }
1253         }
1254
1255         switch (self->jitState) {
1256             case kJitTSelectRequest:
1257             case kJitTSelectRequestHot:
1258                 self->jitState = kJitTSelect;
1259                 self->traceMethod = self->interpSave.method;
1260                 self->currTraceHead = self->interpSave.pc;
1261                 self->currTraceRun = 0;
1262                 self->totalTraceLen = 0;
1263                 self->currRunHead = self->interpSave.pc;
1264                 self->currRunLen = 0;
1265                 self->trace[0].info.frag.startOffset =
1266                      self->interpSave.pc - self->interpSave.method->insns;
1267                 self->trace[0].info.frag.numInsts = 0;
1268                 self->trace[0].info.frag.runEnd = false;
1269                 self->trace[0].info.frag.hint = kJitHintNone;
1270                 self->trace[0].isCode = true;
1271                 self->lastPC = 0;
1272                 /* Turn on trace selection mode */
1273                 dvmUpdateInterpBreak(self, kInterpJitBreak,
1274                                      kSubModeJitTraceBuild, true);
1275 #if defined(SHOW_TRACE)
1276                 LOGD("Starting trace for %s at 0x%x",
1277                      self->interpSave.method->name, (int)self->interpSave.pc);
1278 #endif
1279                 break;
1280             case kJitDone:
1281                 break;
1282             default:
1283                 LOGE("Unexpected JIT state: %d", self->jitState);
1284                 dvmAbort();
1285         }
1286     } else {
1287         /* Cannot build trace this time */
1288         self->jitState = kJitDone;
1289     }
1290 }
1291
1292 /*
1293  * Resizes the JitTable.  Must be a power of 2, and returns true on failure.
1294  * Stops all threads, and thus is a heavyweight operation. May only be called
1295  * by the compiler thread.
1296  */
1297 bool dvmJitResizeJitTable( unsigned int size )
1298 {
1299     JitEntry *pNewTable;
1300     JitEntry *pOldTable;
1301     JitEntry tempEntry;
1302     u4 newMask;
1303     unsigned int oldSize;
1304     unsigned int i;
1305
1306     assert(gDvmJit.pJitEntryTable != NULL);
1307     assert(size && !(size & (size - 1)));   /* Is power of 2? */
1308
1309     LOGI("Jit: resizing JitTable from %d to %d", gDvmJit.jitTableSize, size);
1310
1311     newMask = size - 1;
1312
1313     if (size <= gDvmJit.jitTableSize) {
1314         return true;
1315     }
1316
1317     /* Make sure requested size is compatible with chain field width */
1318     tempEntry.u.info.chain = size;
1319     if (tempEntry.u.info.chain != size) {
1320         LOGD("Jit: JitTable request of %d too big", size);
1321         return true;
1322     }
1323
1324     pNewTable = (JitEntry*)calloc(size, sizeof(*pNewTable));
1325     if (pNewTable == NULL) {
1326         return true;
1327     }
1328     for (i=0; i< size; i++) {
1329         pNewTable[i].u.info.chain = size;  /* Initialize chain termination */
1330     }
1331
1332     /* Stop all other interpreting/jit'ng threads */
1333     dvmSuspendAllThreads(SUSPEND_FOR_TBL_RESIZE);
1334
1335     pOldTable = gDvmJit.pJitEntryTable;
1336     oldSize = gDvmJit.jitTableSize;
1337
1338     dvmLockMutex(&gDvmJit.tableLock);
1339     gDvmJit.pJitEntryTable = pNewTable;
1340     gDvmJit.jitTableSize = size;
1341     gDvmJit.jitTableMask = size - 1;
1342     gDvmJit.jitTableEntriesUsed = 0;
1343
1344     for (i=0; i < oldSize; i++) {
1345         if (pOldTable[i].dPC) {
1346             JitEntry *p;
1347             u2 chain;
1348             p = lookupAndAdd(pOldTable[i].dPC, true /* holds tableLock*/,
1349                              pOldTable[i].u.info.isMethodEntry);
1350             p->codeAddress = pOldTable[i].codeAddress;
1351             /* We need to preserve the new chain field, but copy the rest */
1352             chain = p->u.info.chain;
1353             p->u = pOldTable[i].u;
1354             p->u.info.chain = chain;
1355         }
1356     }
1357
1358     dvmUnlockMutex(&gDvmJit.tableLock);
1359
1360     free(pOldTable);
1361
1362     /* Restart the world */
1363     dvmResumeAllThreads(SUSPEND_FOR_TBL_RESIZE);
1364
1365     return false;
1366 }
1367
1368 /*
1369  * Reset the JitTable to the initial clean state.
1370  */
1371 void dvmJitResetTable(void)
1372 {
1373     JitEntry *jitEntry = gDvmJit.pJitEntryTable;
1374     unsigned int size = gDvmJit.jitTableSize;
1375     unsigned int i;
1376
1377     dvmLockMutex(&gDvmJit.tableLock);
1378
1379     /* Note: If need to preserve any existing counts. Do so here. */
1380     if (gDvmJit.pJitTraceProfCounters) {
1381         for (i=0; i < JIT_PROF_BLOCK_BUCKETS; i++) {
1382             if (gDvmJit.pJitTraceProfCounters->buckets[i])
1383                 memset((void *) gDvmJit.pJitTraceProfCounters->buckets[i],
1384                        0, sizeof(JitTraceCounter_t) * JIT_PROF_BLOCK_ENTRIES);
1385         }
1386         gDvmJit.pJitTraceProfCounters->next = 0;
1387     }
1388
1389     memset((void *) jitEntry, 0, sizeof(JitEntry) * size);
1390     for (i=0; i< size; i++) {
1391         jitEntry[i].u.info.chain = size;  /* Initialize chain termination */
1392     }
1393     gDvmJit.jitTableEntriesUsed = 0;
1394     dvmUnlockMutex(&gDvmJit.tableLock);
1395 }
1396
1397 /*
1398  * Return the address of the next trace profile counter.  This address
1399  * will be embedded in the generated code for the trace, and thus cannot
1400  * change while the trace exists.
1401  */
1402 JitTraceCounter_t *dvmJitNextTraceCounter()
1403 {
1404     int idx = gDvmJit.pJitTraceProfCounters->next / JIT_PROF_BLOCK_ENTRIES;
1405     int elem = gDvmJit.pJitTraceProfCounters->next % JIT_PROF_BLOCK_ENTRIES;
1406     JitTraceCounter_t *res;
1407     /* Lazily allocate blocks of counters */
1408     if (!gDvmJit.pJitTraceProfCounters->buckets[idx]) {
1409         JitTraceCounter_t *p =
1410               (JitTraceCounter_t*) calloc(JIT_PROF_BLOCK_ENTRIES, sizeof(*p));
1411         if (!p) {
1412             LOGE("Failed to allocate block of trace profile counters");
1413             dvmAbort();
1414         }
1415         gDvmJit.pJitTraceProfCounters->buckets[idx] = p;
1416     }
1417     res = &gDvmJit.pJitTraceProfCounters->buckets[idx][elem];
1418     gDvmJit.pJitTraceProfCounters->next++;
1419     return res;
1420 }
1421
1422 /*
1423  * Float/double conversion requires clamping to min and max of integer form.  If
1424  * target doesn't support this normally, use these.
1425  */
1426 s8 dvmJitd2l(double d)
1427 {
1428     static const double kMaxLong = (double)(s8)0x7fffffffffffffffULL;
1429     static const double kMinLong = (double)(s8)0x8000000000000000ULL;
1430     if (d >= kMaxLong)
1431         return (s8)0x7fffffffffffffffULL;
1432     else if (d <= kMinLong)
1433         return (s8)0x8000000000000000ULL;
1434     else if (d != d) // NaN case
1435         return 0;
1436     else
1437         return (s8)d;
1438 }
1439
1440 s8 dvmJitf2l(float f)
1441 {
1442     static const float kMaxLong = (float)(s8)0x7fffffffffffffffULL;
1443     static const float kMinLong = (float)(s8)0x8000000000000000ULL;
1444     if (f >= kMaxLong)
1445         return (s8)0x7fffffffffffffffULL;
1446     else if (f <= kMinLong)
1447         return (s8)0x8000000000000000ULL;
1448     else if (f != f) // NaN case
1449         return 0;
1450     else
1451         return (s8)f;
1452 }
1453
1454 /* Should only be called by the compiler thread */
1455 void dvmJitChangeProfileMode(TraceProfilingModes newState)
1456 {
1457     if (gDvmJit.profileMode != newState) {
1458         gDvmJit.profileMode = newState;
1459         dvmJitUnchainAll();
1460     }
1461 }
1462
1463 void dvmJitTraceProfilingOn()
1464 {
1465     if (gDvmJit.profileMode == kTraceProfilingPeriodicOff)
1466         dvmCompilerForceWorkEnqueue(NULL, kWorkOrderProfileMode,
1467                                     (void*) kTraceProfilingPeriodicOn);
1468     else if (gDvmJit.profileMode == kTraceProfilingDisabled)
1469         dvmCompilerForceWorkEnqueue(NULL, kWorkOrderProfileMode,
1470                                     (void*) kTraceProfilingContinuous);
1471 }
1472
1473 void dvmJitTraceProfilingOff()
1474 {
1475     if (gDvmJit.profileMode == kTraceProfilingPeriodicOn)
1476         dvmCompilerForceWorkEnqueue(NULL, kWorkOrderProfileMode,
1477                                     (void*) kTraceProfilingPeriodicOff);
1478     else if (gDvmJit.profileMode == kTraceProfilingContinuous)
1479         dvmCompilerForceWorkEnqueue(NULL, kWorkOrderProfileMode,
1480                                     (void*) kTraceProfilingDisabled);
1481 }
1482
1483 /*
1484  * Update JIT-specific info in Thread structure for a single thread
1485  */
1486 void dvmJitUpdateThreadStateSingle(Thread* thread)
1487 {
1488     thread->pJitProfTable = gDvmJit.pProfTable;
1489     thread->jitThreshold = gDvmJit.threshold;
1490 }
1491
1492 /*
1493  * Walk through the thread list and refresh all local copies of
1494  * JIT global state (which was placed there for fast access).
1495  */
1496 void dvmJitUpdateThreadStateAll()
1497 {
1498     Thread* self = dvmThreadSelf();
1499     Thread* thread;
1500
1501     dvmLockThreadList(self);
1502     for (thread = gDvm.threadList; thread != NULL; thread = thread->next) {
1503         dvmJitUpdateThreadStateSingle(thread);
1504     }
1505     dvmUnlockThreadList();
1506
1507 }
1508 #endif /* WITH_JIT */