vm/interp/Jit.cpp

   1 /*
   2  * Copyright (C) 2008 The Android Open Source Project
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 #ifdef WITH_JIT
  17
  18 /*
  19  * Target independent portion of Android's Jit
  20  */
  21
  22 #include "Dalvik.h"
  23 #include "Jit.h"
  24
  25 #include "libdex/DexOpcodes.h"
  26 #include <unistd.h>
  27 #include <pthread.h>
  28 #include <sys/time.h>
  29 #include <signal.h>
  30 #include "compiler/Compiler.h"
  31 #include "compiler/CompilerUtility.h"
  32 #include "compiler/CompilerIR.h"
  33 #include <errno.h>
  34
  35 #if defined(WITH_SELF_VERIFICATION)
  36 /* Allocate space for per-thread ShadowSpace data structures */
  37 void* dvmSelfVerificationShadowSpaceAlloc(Thread* self)
  38 {
  39     self->shadowSpace = (ShadowSpace*) calloc(1, sizeof(ShadowSpace));
  40     if (self->shadowSpace == NULL)
  41         return NULL;
  42
  43     self->shadowSpace->registerSpaceSize = REG_SPACE;
  44     self->shadowSpace->registerSpace =
  45         (int*) calloc(self->shadowSpace->registerSpaceSize, sizeof(int));
  46
  47     return self->shadowSpace->registerSpace;
  48 }
  49
  50 /* Free per-thread ShadowSpace data structures */
  51 void dvmSelfVerificationShadowSpaceFree(Thread* self)
  52 {
  53     free(self->shadowSpace->registerSpace);
  54     free(self->shadowSpace);
  55 }
  56
  57 /*
  58  * Save out PC, FP, thread state, and registers to shadow space.
  59  * Return a pointer to the shadow space for JIT to use.
  60  *
  61  * The set of saved state from the Thread structure is:
  62  *     pc  (Dalvik PC)
  63  *     fp  (Dalvik FP)
  64  *     retval
  65  *     method
  66  *     methodClassDex
  67  *     interpStackEnd
  68  */
  69 void* dvmSelfVerificationSaveState(const u2* pc, u4* fp,
  70                                    Thread* self, int targetTrace)
  71 {
  72     ShadowSpace *shadowSpace = self->shadowSpace;
  73     unsigned preBytes = self->interpSave.method->outsSize*4 +
  74         sizeof(StackSaveArea);
  75     unsigned postBytes = self->interpSave.method->registersSize*4;
  76
  77     //ALOGD("### selfVerificationSaveState(%d) pc: %#x fp: %#x",
  78     //    self->threadId, (int)pc, (int)fp);
  79
  80     if (shadowSpace->selfVerificationState != kSVSIdle) {
  81         ALOGD("~~~ Save: INCORRECT PREVIOUS STATE(%d): %d",
  82             self->threadId, shadowSpace->selfVerificationState);
  83         ALOGD("********** SHADOW STATE DUMP **********");
  84         ALOGD("PC: %#x FP: %#x", (int)pc, (int)fp);
  85     }
  86     shadowSpace->selfVerificationState = kSVSStart;
  87
  88     // Dynamically grow shadow register space if necessary
  89     if (preBytes + postBytes > shadowSpace->registerSpaceSize * sizeof(u4)) {
  90         free(shadowSpace->registerSpace);
  91         shadowSpace->registerSpaceSize = (preBytes + postBytes) / sizeof(u4);
  92         shadowSpace->registerSpace =
  93             (int*) calloc(shadowSpace->registerSpaceSize, sizeof(u4));
  94     }
  95
  96     // Remember original state
  97     shadowSpace->startPC = pc;
  98     shadowSpace->fp = fp;
  99     shadowSpace->retval = self->interpSave.retval;
 100     shadowSpace->interpStackEnd = self->interpStackEnd;
 101
 102     /*
 103      * Store the original method here in case the trace ends with a
 104      * return/invoke, the last method.
 105      */
 106     shadowSpace->method = self->interpSave.method;
 107     shadowSpace->methodClassDex = self->interpSave.methodClassDex;
 108
 109     shadowSpace->shadowFP = shadowSpace->registerSpace +
 110                             shadowSpace->registerSpaceSize - postBytes/4;
 111
 112     self->interpSave.curFrame = (u4*)shadowSpace->shadowFP;
 113     self->interpStackEnd = (u1*)shadowSpace->registerSpace;
 114
 115     // Create a copy of the stack
 116     memcpy(((char*)shadowSpace->shadowFP)-preBytes, ((char*)fp)-preBytes,
 117         preBytes+postBytes);
 118
 119     // Setup the shadowed heap space
 120     shadowSpace->heapSpaceTail = shadowSpace->heapSpace;
 121
 122     // Reset trace length
 123     shadowSpace->traceLength = 0;
 124
 125     return shadowSpace;
 126 }
 127
 128 /*
 129  * Save ending PC, FP and compiled code exit point to shadow space.
 130  * Return a pointer to the shadow space for JIT to restore state.
 131  */
 132 void* dvmSelfVerificationRestoreState(const u2* pc, u4* fp,
 133                                       SelfVerificationState exitState,
 134                                       Thread* self)
 135 {
 136     ShadowSpace *shadowSpace = self->shadowSpace;
 137     shadowSpace->endPC = pc;
 138     shadowSpace->endShadowFP = fp;
 139     shadowSpace->jitExitState = exitState;
 140
 141     //ALOGD("### selfVerificationRestoreState(%d) pc: %#x fp: %#x endPC: %#x",
 142     //    self->threadId, (int)shadowSpace->startPC, (int)shadowSpace->fp,
 143     //    (int)pc);
 144
 145     if (shadowSpace->selfVerificationState != kSVSStart) {
 146         ALOGD("~~~ Restore: INCORRECT PREVIOUS STATE(%d): %d",
 147             self->threadId, shadowSpace->selfVerificationState);
 148         ALOGD("********** SHADOW STATE DUMP **********");
 149         ALOGD("Dalvik PC: %#x endPC: %#x", (int)shadowSpace->startPC,
 150             (int)shadowSpace->endPC);
 151         ALOGD("Interp FP: %#x", (int)shadowSpace->fp);
 152         ALOGD("Shadow FP: %#x endFP: %#x", (int)shadowSpace->shadowFP,
 153             (int)shadowSpace->endShadowFP);
 154     }
 155
 156     // Special case when punting after a single instruction
 157     if (exitState == kSVSPunt && pc == shadowSpace->startPC) {
 158         shadowSpace->selfVerificationState = kSVSIdle;
 159     } else {
 160         shadowSpace->selfVerificationState = exitState;
 161     }
 162
 163     /* Restore state before returning */
 164     self->interpSave.pc = shadowSpace->startPC;
 165     self->interpSave.curFrame = shadowSpace->fp;
 166     self->interpSave.method = shadowSpace->method;
 167     self->interpSave.methodClassDex = shadowSpace->methodClassDex;
 168     self->interpSave.retval = shadowSpace->retval;
 169     self->interpStackEnd = shadowSpace->interpStackEnd;
 170
 171     return shadowSpace;
 172 }
 173
 174 /* Print contents of virtual registers */
 175 static void selfVerificationPrintRegisters(int* addr, int* addrRef,
 176                                            int numWords)
 177 {
 178     int i;
 179     for (i = 0; i < numWords; i++) {
 180         ALOGD("(v%d) 0x%8x%s", i, addr[i], addr[i] != addrRef[i] ? " X" : "");
 181     }
 182 }
 183
 184 /* Print values maintained in shadowSpace */
 185 static void selfVerificationDumpState(const u2* pc, Thread* self)
 186 {
 187     ShadowSpace* shadowSpace = self->shadowSpace;
 188     StackSaveArea* stackSave = SAVEAREA_FROM_FP(self->interpSave.curFrame);
 189     int frameBytes = (int) shadowSpace->registerSpace +
 190                      shadowSpace->registerSpaceSize*4 -
 191                      (int) shadowSpace->shadowFP;
 192     int localRegs = 0;
 193     int frameBytes2 = 0;
 194     if ((uintptr_t)self->interpSave.curFrame < (uintptr_t)shadowSpace->fp) {
 195         localRegs = (stackSave->method->registersSize -
 196                      stackSave->method->insSize)*4;
 197         frameBytes2 = (int) shadowSpace->fp -
 198                       (int)self->interpSave.curFrame - localRegs;
 199     }
 200     ALOGD("********** SHADOW STATE DUMP **********");
 201     ALOGD("CurrentPC: %#x, Offset: 0x%04x", (int)pc,
 202         (int)(pc - stackSave->method->insns));
 203     ALOGD("Class: %s", shadowSpace->method->clazz->descriptor);
 204     ALOGD("Method: %s", shadowSpace->method->name);
 205     ALOGD("Dalvik PC: %#x endPC: %#x", (int)shadowSpace->startPC,
 206         (int)shadowSpace->endPC);
 207     ALOGD("Interp FP: %#x endFP: %#x", (int)shadowSpace->fp,
 208         (int)self->interpSave.curFrame);
 209     ALOGD("Shadow FP: %#x endFP: %#x", (int)shadowSpace->shadowFP,
 210         (int)shadowSpace->endShadowFP);
 211     ALOGD("Frame1 Bytes: %d Frame2 Local: %d Bytes: %d", frameBytes,
 212         localRegs, frameBytes2);
 213     ALOGD("Trace length: %d State: %d", shadowSpace->traceLength,
 214         shadowSpace->selfVerificationState);
 215 }
 216
 217 /* Print decoded instructions in the current trace */
 218 static void selfVerificationDumpTrace(const u2* pc, Thread* self)
 219 {
 220     ShadowSpace* shadowSpace = self->shadowSpace;
 221     StackSaveArea* stackSave = SAVEAREA_FROM_FP(self->interpSave.curFrame);
 222     int i, addr, offset;
 223     DecodedInstruction *decInsn;
 224
 225     ALOGD("********** SHADOW TRACE DUMP **********");
 226     for (i = 0; i < shadowSpace->traceLength; i++) {
 227         addr = shadowSpace->trace[i].addr;
 228         offset =  (int)((u2*)addr - stackSave->method->insns);
 229         decInsn = &(shadowSpace->trace[i].decInsn);
 230         /* Not properly decoding instruction, some registers may be garbage */
 231         ALOGD("%#x: (0x%04x) %s",
 232             addr, offset, dexGetOpcodeName(decInsn->opcode));
 233     }
 234 }
 235
 236 /* Code is forced into this spin loop when a divergence is detected */
 237 static void selfVerificationSpinLoop(ShadowSpace *shadowSpace)
 238 {
 239     const u2 *startPC = shadowSpace->startPC;
 240     JitTraceDescription* desc = dvmCopyTraceDescriptor(startPC, NULL);
 241     if (desc) {
 242         dvmCompilerWorkEnqueue(startPC, kWorkOrderTraceDebug, desc);
 243         /*
 244          * This function effectively terminates the VM right here, so not
 245          * freeing the desc pointer when the enqueuing fails is acceptable.
 246          */
 247     }
 248     gDvmJit.selfVerificationSpin = true;
 249     while(gDvmJit.selfVerificationSpin) sleep(10);
 250 }
 251
 252 /*
 253  * If here, we're re-interpreting an instruction that was included
 254  * in a trace that was just executed.  This routine is called for
 255  * each instruction in the original trace, and compares state
 256  * when it reaches the end point.
 257  *
 258  * TUNING: the interpretation mechanism now supports a counted
 259  * single-step mechanism.  If we were to associate an instruction
 260  * count with each trace exit, we could just single-step the right
 261  * number of cycles and then compare.  This would improve detection
 262  * of control divergences, as well as (slightly) simplify this code.
 263  */
 264 void dvmCheckSelfVerification(const u2* pc, Thread* self)
 265 {
 266     ShadowSpace *shadowSpace = self->shadowSpace;
 267     SelfVerificationState state = shadowSpace->selfVerificationState;
 268
 269     DecodedInstruction decInsn;
 270     dexDecodeInstruction(pc, &decInsn);
 271
 272     //ALOGD("### DbgIntp(%d): PC: %#x endPC: %#x state: %d len: %d %s",
 273     //    self->threadId, (int)pc, (int)shadowSpace->endPC, state,
 274     //    shadowSpace->traceLength, dexGetOpcodeName(decInsn.opcode));
 275
 276     if (state == kSVSIdle || state == kSVSStart) {
 277         ALOGD("~~~ DbgIntrp: INCORRECT PREVIOUS STATE(%d): %d",
 278             self->threadId, state);
 279         selfVerificationDumpState(pc, self);
 280         selfVerificationDumpTrace(pc, self);
 281     }
 282
 283     /*
 284      * Generalize the self verification state to kSVSDebugInterp unless the
 285      * entry reason is kSVSBackwardBranch or kSVSSingleStep.
 286      */
 287     if (state != kSVSBackwardBranch && state != kSVSSingleStep) {
 288         shadowSpace->selfVerificationState = kSVSDebugInterp;
 289     }
 290
 291     /*
 292      * Check if the current pc matches the endPC. Only check for non-zero
 293      * trace length when backward branches are involved.
 294      */
 295     if (pc == shadowSpace->endPC &&
 296         (state == kSVSDebugInterp || state == kSVSSingleStep ||
 297          (state == kSVSBackwardBranch && shadowSpace->traceLength != 0))) {
 298
 299         shadowSpace->selfVerificationState = kSVSIdle;
 300
 301         /* Check register space */
 302         int frameBytes = (int) shadowSpace->registerSpace +
 303                          shadowSpace->registerSpaceSize*4 -
 304                          (int) shadowSpace->shadowFP;
 305         if (memcmp(shadowSpace->fp, shadowSpace->shadowFP, frameBytes)) {
 306             if (state == kSVSBackwardBranch) {
 307                 /* State mismatch on backward branch - try one more iteration */
 308                 shadowSpace->selfVerificationState = kSVSDebugInterp;
 309                 goto log_and_continue;
 310             }
 311             ALOGD("~~~ DbgIntp(%d): REGISTERS DIVERGENCE!", self->threadId);
 312             selfVerificationDumpState(pc, self);
 313             selfVerificationDumpTrace(pc, self);
 314             ALOGD("*** Interp Registers: addr: %#x bytes: %d",
 315                 (int)shadowSpace->fp, frameBytes);
 316             selfVerificationPrintRegisters((int*)shadowSpace->fp,
 317                                            (int*)shadowSpace->shadowFP,
 318                                            frameBytes/4);
 319             ALOGD("*** Shadow Registers: addr: %#x bytes: %d",
 320                 (int)shadowSpace->shadowFP, frameBytes);
 321             selfVerificationPrintRegisters((int*)shadowSpace->shadowFP,
 322                                            (int*)shadowSpace->fp,
 323                                            frameBytes/4);
 324             selfVerificationSpinLoop(shadowSpace);
 325         }
 326         /* Check new frame if it exists (invokes only) */
 327         if ((uintptr_t)self->interpSave.curFrame < (uintptr_t)shadowSpace->fp) {
 328             StackSaveArea* stackSave =
 329                 SAVEAREA_FROM_FP(self->interpSave.curFrame);
 330             int localRegs = (stackSave->method->registersSize -
 331                              stackSave->method->insSize)*4;
 332             int frameBytes2 = (int) shadowSpace->fp -
 333                               (int) self->interpSave.curFrame - localRegs;
 334             if (memcmp(((char*)self->interpSave.curFrame)+localRegs,
 335                 ((char*)shadowSpace->endShadowFP)+localRegs, frameBytes2)) {
 336                 if (state == kSVSBackwardBranch) {
 337                     /*
 338                      * State mismatch on backward branch - try one more
 339                      * iteration.
 340                      */
 341                     shadowSpace->selfVerificationState = kSVSDebugInterp;
 342                     goto log_and_continue;
 343                 }
 344                 ALOGD("~~~ DbgIntp(%d): REGISTERS (FRAME2) DIVERGENCE!",
 345                     self->threadId);
 346                 selfVerificationDumpState(pc, self);
 347                 selfVerificationDumpTrace(pc, self);
 348                 ALOGD("*** Interp Registers: addr: %#x l: %d bytes: %d",
 349                     (int)self->interpSave.curFrame, localRegs, frameBytes2);
 350                 selfVerificationPrintRegisters((int*)self->interpSave.curFrame,
 351                                                (int*)shadowSpace->endShadowFP,
 352                                                (frameBytes2+localRegs)/4);
 353                 ALOGD("*** Shadow Registers: addr: %#x l: %d bytes: %d",
 354                     (int)shadowSpace->endShadowFP, localRegs, frameBytes2);
 355                 selfVerificationPrintRegisters((int*)shadowSpace->endShadowFP,
 356                                                (int*)self->interpSave.curFrame,
 357                                                (frameBytes2+localRegs)/4);
 358                 selfVerificationSpinLoop(shadowSpace);
 359             }
 360         }
 361
 362         /* Check memory space */
 363         bool memDiff = false;
 364         ShadowHeap* heapSpacePtr;
 365         for (heapSpacePtr = shadowSpace->heapSpace;
 366              heapSpacePtr != shadowSpace->heapSpaceTail; heapSpacePtr++) {
 367             int memData = *((unsigned int*) heapSpacePtr->addr);
 368             if (heapSpacePtr->data != memData) {
 369                 if (state == kSVSBackwardBranch) {
 370                     /*
 371                      * State mismatch on backward branch - try one more
 372                      * iteration.
 373                      */
 374                     shadowSpace->selfVerificationState = kSVSDebugInterp;
 375                     goto log_and_continue;
 376                 }
 377                 ALOGD("~~~ DbgIntp(%d): MEMORY DIVERGENCE!", self->threadId);
 378                 ALOGD("Addr: %#x Intrp Data: %#x Jit Data: %#x",
 379                     heapSpacePtr->addr, memData, heapSpacePtr->data);
 380                 selfVerificationDumpState(pc, self);
 381                 selfVerificationDumpTrace(pc, self);
 382                 memDiff = true;
 383             }
 384         }
 385         if (memDiff) selfVerificationSpinLoop(shadowSpace);
 386
 387
 388         /*
 389          * Success.  If this shadowed trace included a single-stepped
 390          * instruction, we need to stay in the interpreter for one
 391          * more interpretation before resuming.
 392          */
 393         if (state == kSVSSingleStep) {
 394             assert(self->jitResumeNPC != NULL);
 395             assert(self->singleStepCount == 0);
 396             self->singleStepCount = 1;
 397             dvmEnableSubMode(self, kSubModeCountedStep);
 398         }
 399
 400         /*
 401          * Switch off shadow replay mode.  The next shadowed trace
 402          * execution will turn it back on.
 403          */
 404         dvmDisableSubMode(self, kSubModeJitSV);
 405
 406         self->jitState = kJitDone;
 407         return;
 408     }
 409 log_and_continue:
 410     /* If end not been reached, make sure max length not exceeded */
 411     if (shadowSpace->traceLength >= JIT_MAX_TRACE_LEN) {
 412         ALOGD("~~~ DbgIntp(%d): CONTROL DIVERGENCE!", self->threadId);
 413         ALOGD("startPC: %#x endPC: %#x currPC: %#x",
 414             (int)shadowSpace->startPC, (int)shadowSpace->endPC, (int)pc);
 415         selfVerificationDumpState(pc, self);
 416         selfVerificationDumpTrace(pc, self);
 417         selfVerificationSpinLoop(shadowSpace);
 418         return;
 419     }
 420     /* Log the instruction address and decoded instruction for debug */
 421     shadowSpace->trace[shadowSpace->traceLength].addr = (int)pc;
 422     shadowSpace->trace[shadowSpace->traceLength].decInsn = decInsn;
 423     shadowSpace->traceLength++;
 424 }
 425 #endif
 426
 427 /*
 428  * If one of our fixed tables or the translation buffer fills up,
 429  * call this routine to avoid wasting cycles on future translation requests.
 430  */
 431 void dvmJitStopTranslationRequests()
 432 {
 433     /*
 434      * Note 1: This won't necessarily stop all translation requests, and
 435      * operates on a delayed mechanism.  Running threads look to the copy
 436      * of this value in their private thread structures and won't see
 437      * this change until it is refreshed (which happens on interpreter
 438      * entry).
 439      * Note 2: This is a one-shot memory leak on this table. Because this is a
 440      * permanent off switch for Jit profiling, it is a one-time leak of 1K
 441      * bytes, and no further attempt will be made to re-allocate it.  Can't
 442      * free it because some thread may be holding a reference.
 443      */
 444     gDvmJit.pProfTable = NULL;
 445     dvmJitUpdateThreadStateAll();
 446 }
 447
 448 #if defined(WITH_JIT_TUNING)
 449 /* Convenience function to increment counter from assembly code */
 450 void dvmBumpNoChain(int from)
 451 {
 452     gDvmJit.noChainExit[from]++;
 453 }
 454
 455 /* Convenience function to increment counter from assembly code */
 456 void dvmBumpNormal()
 457 {
 458     gDvmJit.normalExit++;
 459 }
 460
 461 /* Convenience function to increment counter from assembly code */
 462 void dvmBumpPunt(int from)
 463 {
 464     gDvmJit.puntExit++;
 465 }
 466 #endif
 467
 468 /* Dumps debugging & tuning stats to the log */
 469 void dvmJitStats()
 470 {
 471     int i;
 472     int hit;
 473     int not_hit;
 474     int chains;
 475     int stubs;
 476     if (gDvmJit.pJitEntryTable) {
 477         for (i=0, stubs=chains=hit=not_hit=0;
 478              i < (int) gDvmJit.jitTableSize;
 479              i++) {
 480             if (gDvmJit.pJitEntryTable[i].dPC != 0) {
 481                 hit++;
 482                 if (gDvmJit.pJitEntryTable[i].codeAddress ==
 483                       dvmCompilerGetInterpretTemplate())
 484                     stubs++;
 485             } else
 486                 not_hit++;
 487             if (gDvmJit.pJitEntryTable[i].u.info.chain != gDvmJit.jitTableSize)
 488                 chains++;
 489         }
 490         ALOGD("JIT: table size is %d, entries used is %d",
 491              gDvmJit.jitTableSize,  gDvmJit.jitTableEntriesUsed);
 492         ALOGD("JIT: %d traces, %d slots, %d chains, %d thresh, %s",
 493              hit, not_hit + hit, chains, gDvmJit.threshold,
 494              gDvmJit.blockingMode ? "Blocking" : "Non-blocking");
 495
 496 #if defined(WITH_JIT_TUNING)
 497         ALOGD("JIT: Code cache patches: %d", gDvmJit.codeCachePatches);
 498
 499         ALOGD("JIT: Lookups: %d hits, %d misses; %d normal, %d punt",
 500              gDvmJit.addrLookupsFound, gDvmJit.addrLookupsNotFound,
 501              gDvmJit.normalExit, gDvmJit.puntExit);
 502
 503         ALOGD("JIT: ICHits: %d", gDvmICHitCount);
 504
 505         ALOGD("JIT: noChainExit: %d IC miss, %d interp callsite, "
 506              "%d switch overflow",
 507              gDvmJit.noChainExit[kInlineCacheMiss],
 508              gDvmJit.noChainExit[kCallsiteInterpreted],
 509              gDvmJit.noChainExit[kSwitchOverflow]);
 510
 511         ALOGD("JIT: ICPatch: %d init, %d rejected, %d lock-free, %d queued, "
 512              "%d dropped",
 513              gDvmJit.icPatchInit, gDvmJit.icPatchRejected,
 514              gDvmJit.icPatchLockFree, gDvmJit.icPatchQueued,
 515              gDvmJit.icPatchDropped);
 516
 517         ALOGD("JIT: Invoke: %d mono, %d poly, %d native, %d return",
 518              gDvmJit.invokeMonomorphic, gDvmJit.invokePolymorphic,
 519              gDvmJit.invokeNative, gDvmJit.returnOp);
 520         ALOGD("JIT: Inline: %d mgetter, %d msetter, %d pgetter, %d psetter",
 521              gDvmJit.invokeMonoGetterInlined, gDvmJit.invokeMonoSetterInlined,
 522              gDvmJit.invokePolyGetterInlined, gDvmJit.invokePolySetterInlined);
 523         ALOGD("JIT: Total compilation time: %llu ms", gDvmJit.jitTime / 1000);
 524         ALOGD("JIT: Avg unit compilation time: %llu us",
 525              gDvmJit.numCompilations == 0 ? 0 :
 526              gDvmJit.jitTime / gDvmJit.numCompilations);
 527         ALOGD("JIT: Potential GC blocked by compiler: max %llu us / "
 528              "avg %llu us (%d)",
 529              gDvmJit.maxCompilerThreadBlockGCTime,
 530              gDvmJit.numCompilerThreadBlockGC == 0 ?
 531                  0 : gDvmJit.compilerThreadBlockGCTime /
 532                      gDvmJit.numCompilerThreadBlockGC,
 533              gDvmJit.numCompilerThreadBlockGC);
 534 #endif
 535
 536         ALOGD("JIT: %d Translation chains, %d interp stubs",
 537              gDvmJit.translationChains, stubs);
 538         if (gDvmJit.profileMode == kTraceProfilingContinuous) {
 539             dvmCompilerSortAndPrintTraceProfiles();
 540         }
 541     }
 542 }
 543
 544
 545 /* End current trace now & don't include current instruction */
 546 void dvmJitEndTraceSelect(Thread* self, const u2* dPC)
 547 {
 548     if (self->jitState == kJitTSelect) {
 549         self->jitState = kJitTSelectEnd;
 550     }
 551     if (self->jitState == kJitTSelectEnd) {
 552         // Clean up and finish now.
 553         dvmCheckJit(dPC, self);
 554     }
 555 }
 556
 557 /*
 558  * Find an entry in the JitTable, creating if necessary.
 559  * Returns null if table is full.
 560  */
 561 static JitEntry *lookupAndAdd(const u2* dPC, bool callerLocked,
 562                               bool isMethodEntry)
 563 {
 564     u4 chainEndMarker = gDvmJit.jitTableSize;
 565     u4 idx = dvmJitHash(dPC);
 566
 567     /*
 568      * Walk the bucket chain to find an exact match for our PC and trace/method
 569      * type
 570      */
 571     while ((gDvmJit.pJitEntryTable[idx].u.info.chain != chainEndMarker) &&
 572            ((gDvmJit.pJitEntryTable[idx].dPC != dPC) ||
 573             (gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry !=
 574              isMethodEntry))) {
 575         idx = gDvmJit.pJitEntryTable[idx].u.info.chain;
 576     }
 577
 578     if (gDvmJit.pJitEntryTable[idx].dPC != dPC ||
 579         gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry != isMethodEntry) {
 580         /*
 581          * No match.  Aquire jitTableLock and find the last
 582          * slot in the chain. Possibly continue the chain walk in case
 583          * some other thread allocated the slot we were looking
 584          * at previuosly (perhaps even the dPC we're trying to enter).
 585          */
 586         if (!callerLocked)
 587             dvmLockMutex(&gDvmJit.tableLock);
 588         /*
 589          * At this point, if .dPC is NULL, then the slot we're
 590          * looking at is the target slot from the primary hash
 591          * (the simple, and common case).  Otherwise we're going
 592          * to have to find a free slot and chain it.
 593          */
 594         ANDROID_MEMBAR_FULL(); /* Make sure we reload [].dPC after lock */
 595         if (gDvmJit.pJitEntryTable[idx].dPC != NULL) {
 596             u4 prev;
 597             while (gDvmJit.pJitEntryTable[idx].u.info.chain != chainEndMarker) {
 598                 if (gDvmJit.pJitEntryTable[idx].dPC == dPC &&
 599                     gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry ==
 600                         isMethodEntry) {
 601                     /* Another thread got there first for this dPC */
 602                     if (!callerLocked)
 603                         dvmUnlockMutex(&gDvmJit.tableLock);
 604                     return &gDvmJit.pJitEntryTable[idx];
 605                 }
 606                 idx = gDvmJit.pJitEntryTable[idx].u.info.chain;
 607             }
 608             /* Here, idx should be pointing to the last cell of an
 609              * active chain whose last member contains a valid dPC */
 610             assert(gDvmJit.pJitEntryTable[idx].dPC != NULL);
 611             /* Linear walk to find a free cell and add it to the end */
 612             prev = idx;
 613             while (true) {
 614                 idx++;
 615                 if (idx == chainEndMarker)
 616                     idx = 0;  /* Wraparound */
 617                 if ((gDvmJit.pJitEntryTable[idx].dPC == NULL) ||
 618                     (idx == prev))
 619                     break;
 620             }
 621             if (idx != prev) {
 622                 JitEntryInfoUnion oldValue;
 623                 JitEntryInfoUnion newValue;
 624                 /*
 625                  * Although we hold the lock so that noone else will
 626                  * be trying to update a chain field, the other fields
 627                  * packed into the word may be in use by other threads.
 628                  */
 629                 do {
 630                     oldValue = gDvmJit.pJitEntryTable[prev].u;
 631                     newValue = oldValue;
 632                     newValue.info.chain = idx;
 633                 } while (android_atomic_release_cas(oldValue.infoWord,
 634                         newValue.infoWord,
 635                         &gDvmJit.pJitEntryTable[prev].u.infoWord) != 0);
 636             }
 637         }
 638         if (gDvmJit.pJitEntryTable[idx].dPC == NULL) {
 639             gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry = isMethodEntry;
 640             /*
 641              * Initialize codeAddress and allocate the slot.  Must
 642              * happen in this order (since dPC is set, the entry is live.
 643              */
 644             android_atomic_release_store((int32_t)dPC,
 645                  (volatile int32_t *)(void *)&gDvmJit.pJitEntryTable[idx].dPC);
 646             /* for simulator mode, we need to initialized codeAddress to null */
 647             gDvmJit.pJitEntryTable[idx].codeAddress = NULL;
 648             gDvmJit.pJitEntryTable[idx].dPC = dPC;
 649             gDvmJit.jitTableEntriesUsed++;
 650         } else {
 651             /* Table is full */
 652             idx = chainEndMarker;
 653         }
 654         if (!callerLocked)
 655             dvmUnlockMutex(&gDvmJit.tableLock);
 656     }
 657     return (idx == chainEndMarker) ? NULL : &gDvmJit.pJitEntryTable[idx];
 658 }
 659
 660 /* Dump a trace description */
 661 void dvmJitDumpTraceDesc(JitTraceDescription *trace)
 662 {
 663     int i;
 664     bool done = false;
 665     const u2* dpc;
 666     const u2* dpcBase;
 667     int curFrag = 0;
 668     ALOGD("===========================================");
 669     ALOGD("Trace dump %#x, Method %s off %#x",(int)trace,
 670          trace->method->name,trace->trace[curFrag].info.frag.startOffset);
 671     dpcBase = trace->method->insns;
 672     while (!done) {
 673         DecodedInstruction decInsn;
 674         if (trace->trace[curFrag].isCode) {
 675             ALOGD("Frag[%d]- Insts: %d, start: %#x, hint: %#x, end: %d",
 676                  curFrag, trace->trace[curFrag].info.frag.numInsts,
 677                  trace->trace[curFrag].info.frag.startOffset,
 678                  trace->trace[curFrag].info.frag.hint,
 679                  trace->trace[curFrag].info.frag.runEnd);
 680             dpc = dpcBase + trace->trace[curFrag].info.frag.startOffset;
 681             for (i=0; i<trace->trace[curFrag].info.frag.numInsts; i++) {
 682                 dexDecodeInstruction(dpc, &decInsn);
 683                 ALOGD("    0x%04x - %s %#x",(dpc-dpcBase),
 684                      dexGetOpcodeName(decInsn.opcode),(int)dpc);
 685                 dpc += dexGetWidthFromOpcode(decInsn.opcode);
 686             }
 687             if (trace->trace[curFrag].info.frag.runEnd) {
 688                 done = true;
 689             }
 690         } else {
 691             ALOGD("Frag[%d]- META info: 0x%08x", curFrag,
 692                  (int)trace->trace[curFrag].info.meta);
 693         }
 694         curFrag++;
 695     }
 696     ALOGD("-------------------------------------------");
 697 }
 698
 699 /*
 700  * Append the class ptr of "this" and the current method ptr to the current
 701  * trace. That is, the trace runs will contain the following components:
 702  *  + trace run that ends with an invoke (existing entry)
 703  *  + thisClass (new)
 704  *  + calleeMethod (new)
 705  */
 706 static void insertClassMethodInfo(Thread* self,
 707                                   const ClassObject* thisClass,
 708                                   const Method* calleeMethod,
 709                                   const DecodedInstruction* insn)
 710 {
 711     int currTraceRun = ++self->currTraceRun;
 712     self->trace[currTraceRun].info.meta = thisClass ?
 713                                     (void *) thisClass->descriptor : NULL;
 714     self->trace[currTraceRun].isCode = false;
 715
 716     currTraceRun = ++self->currTraceRun;
 717     self->trace[currTraceRun].info.meta = thisClass ?
 718                                     (void *) thisClass->classLoader : NULL;
 719     self->trace[currTraceRun].isCode = false;
 720
 721     currTraceRun = ++self->currTraceRun;
 722     self->trace[currTraceRun].info.meta = (void *) calleeMethod;
 723     self->trace[currTraceRun].isCode = false;
 724 }
 725
 726 /*
 727  * Check if the next instruction following the invoke is a move-result and if
 728  * so add it to the trace. That is, this will add the trace run that includes
 729  * the move-result to the trace list.
 730  *
 731  *  + trace run that ends with an invoke (existing entry)
 732  *  + thisClass (existing entry)
 733  *  + calleeMethod (existing entry)
 734  *  + move result (new)
 735  *
 736  * lastPC, len, offset are all from the preceding invoke instruction
 737  */
 738 static void insertMoveResult(const u2 *lastPC, int len, int offset,
 739                              Thread *self)
 740 {
 741     DecodedInstruction nextDecInsn;
 742     const u2 *moveResultPC = lastPC + len;
 743
 744     dexDecodeInstruction(moveResultPC, &nextDecInsn);
 745     if ((nextDecInsn.opcode != OP_MOVE_RESULT) &&
 746         (nextDecInsn.opcode != OP_MOVE_RESULT_WIDE) &&
 747         (nextDecInsn.opcode != OP_MOVE_RESULT_OBJECT))
 748         return;
 749
 750     /* We need to start a new trace run */
 751     int currTraceRun = ++self->currTraceRun;
 752     self->currRunHead = moveResultPC;
 753     self->trace[currTraceRun].info.frag.startOffset = offset + len;
 754     self->trace[currTraceRun].info.frag.numInsts = 1;
 755     self->trace[currTraceRun].info.frag.runEnd = false;
 756     self->trace[currTraceRun].info.frag.hint = kJitHintNone;
 757     self->trace[currTraceRun].isCode = true;
 758     self->totalTraceLen++;
 759
 760     self->currRunLen = dexGetWidthFromInstruction(moveResultPC);
 761 }
 762
 763 /*
 764  * Adds to the current trace request one instruction at a time, just
 765  * before that instruction is interpreted.  This is the primary trace
 766  * selection function.  NOTE: return instruction are handled a little
 767  * differently.  In general, instructions are "proposed" to be added
 768  * to the current trace prior to interpretation.  If the interpreter
 769  * then successfully completes the instruction, is will be considered
 770  * part of the request.  This allows us to examine machine state prior
 771  * to interpretation, and also abort the trace request if the instruction
 772  * throws or does something unexpected.  However, return instructions
 773  * will cause an immediate end to the translation request - which will
 774  * be passed to the compiler before the return completes.  This is done
 775  * in response to special handling of returns by the interpreter (and
 776  * because returns cannot throw in a way that causes problems for the
 777  * translated code.
 778  */
 779 void dvmCheckJit(const u2* pc, Thread* self)
 780 {
 781     const ClassObject *thisClass = self->callsiteClass;
 782     const Method* curMethod = self->methodToCall;
 783     int flags, len;
 784     int allDone = false;
 785     /* Stay in break/single-stop mode for the next instruction */
 786     bool stayOneMoreInst = false;
 787
 788     /* Prepare to handle last PC and stage the current PC & method*/
 789     const u2 *lastPC = self->lastPC;
 790
 791     self->lastPC = pc;
 792
 793     switch (self->jitState) {
 794         int offset;
 795         DecodedInstruction decInsn;
 796         case kJitTSelect:
 797             /* First instruction - just remember the PC and exit */
 798             if (lastPC == NULL) break;
 799             /* Grow the trace around the last PC if jitState is kJitTSelect */
 800             dexDecodeInstruction(lastPC, &decInsn);
 801 #if TRACE_OPCODE_FILTER
 802             /* Only add JIT support opcode to trace. End the trace if
 803              * this opcode is not supported.
 804              */
 805             if (!dvmIsOpcodeSupportedByJit(decInsn.opcode)) {
 806                 self->jitState = kJitTSelectEnd;
 807                 break;
 808             }
 809 #endif
 810             /*
 811              * Treat {PACKED,SPARSE}_SWITCH as trace-ending instructions due
 812              * to the amount of space it takes to generate the chaining
 813              * cells.
 814              */
 815             if (self->totalTraceLen != 0 &&
 816                 (decInsn.opcode == OP_PACKED_SWITCH ||
 817                  decInsn.opcode == OP_SPARSE_SWITCH)) {
 818                 self->jitState = kJitTSelectEnd;
 819                 break;
 820             }
 821
 822 #if defined(SHOW_TRACE)
 823             ALOGD("TraceGen: adding %s. lpc:%#x, pc:%#x",
 824                  dexGetOpcodeName(decInsn.opcode), (int)lastPC, (int)pc);
 825 #endif
 826             flags = dexGetFlagsFromOpcode(decInsn.opcode);
 827             len = dexGetWidthFromInstruction(lastPC);
 828             offset = lastPC - self->traceMethod->insns;
 829             assert((unsigned) offset <
 830                    dvmGetMethodInsnsSize(self->traceMethod));
 831             if (lastPC != self->currRunHead + self->currRunLen) {
 832                 int currTraceRun;
 833                 /* We need to start a new trace run */
 834                 currTraceRun = ++self->currTraceRun;
 835                 self->currRunLen = 0;
 836                 self->currRunHead = (u2*)lastPC;
 837                 self->trace[currTraceRun].info.frag.startOffset = offset;
 838                 self->trace[currTraceRun].info.frag.numInsts = 0;
 839                 self->trace[currTraceRun].info.frag.runEnd = false;
 840                 self->trace[currTraceRun].info.frag.hint = kJitHintNone;
 841                 self->trace[currTraceRun].isCode = true;
 842             }
 843             self->trace[self->currTraceRun].info.frag.numInsts++;
 844             self->totalTraceLen++;
 845             self->currRunLen += len;
 846
 847             /*
 848              * If the last instruction is an invoke, we will try to sneak in
 849              * the move-result* (if existent) into a separate trace run.
 850              */
 851             {
 852               int needReservedRun = (flags & kInstrInvoke) ? 1 : 0;
 853
 854               /* Will probably never hit this with the current trace builder */
 855               if (self->currTraceRun ==
 856                    (MAX_JIT_RUN_LEN - 1 - needReservedRun)) {
 857                 self->jitState = kJitTSelectEnd;
 858               }
 859             }
 860
 861             if (!dexIsGoto(flags) &&
 862                   ((flags & (kInstrCanBranch |
 863                              kInstrCanSwitch |
 864                              kInstrCanReturn |
 865                              kInstrInvoke)) != 0)) {
 866                     self->jitState = kJitTSelectEnd;
 867 #if defined(SHOW_TRACE)
 868                 ALOGD("TraceGen: ending on %s, basic block end",
 869                      dexGetOpcodeName(decInsn.opcode));
 870 #endif
 871
 872                 /*
 873                  * If the current invoke is a {virtual,interface}, get the
 874                  * current class/method pair into the trace as well.
 875                  * If the next instruction is a variant of move-result, insert
 876                  * it to the trace too.
 877                  */
 878                 if (flags & kInstrInvoke) {
 879                     insertClassMethodInfo(self, thisClass, curMethod,
 880                                           &decInsn);
 881                     insertMoveResult(lastPC, len, offset, self);
 882                 }
 883             }
 884             /* Break on throw or self-loop */
 885             if ((decInsn.opcode == OP_THROW) || (lastPC == pc)){
 886                 self->jitState = kJitTSelectEnd;
 887             }
 888             if (self->totalTraceLen >= JIT_MAX_TRACE_LEN) {
 889                 self->jitState = kJitTSelectEnd;
 890             }
 891             if ((flags & kInstrCanReturn) != kInstrCanReturn) {
 892                 break;
 893             }
 894             else {
 895                 /*
 896                  * Last instruction is a return - stay in the dbg interpreter
 897                  * for one more instruction if it is a non-void return, since
 898                  * we don't want to start a trace with move-result as the first
 899                  * instruction (which is already included in the trace
 900                  * containing the invoke.
 901                  */
 902                 if (decInsn.opcode != OP_RETURN_VOID) {
 903                     stayOneMoreInst = true;
 904                 }
 905             }
 906             /* NOTE: intentional fallthrough for returns */
 907         case kJitTSelectEnd:
 908             {
 909                 /* Empty trace - set to bail to interpreter */
 910                 if (self->totalTraceLen == 0) {
 911                     dvmJitSetCodeAddr(self->currTraceHead,
 912                                       dvmCompilerGetInterpretTemplate(),
 913                                       dvmCompilerGetInterpretTemplateSet(),
 914                                       false /* Not method entry */, 0);
 915                     self->jitState = kJitDone;
 916                     allDone = true;
 917                     break;
 918                 }
 919
 920                 int lastTraceDesc = self->currTraceRun;
 921
 922                 /* Extend a new empty desc if the last slot is meta info */
 923                 if (!self->trace[lastTraceDesc].isCode) {
 924                     lastTraceDesc = ++self->currTraceRun;
 925                     self->trace[lastTraceDesc].info.frag.startOffset = 0;
 926                     self->trace[lastTraceDesc].info.frag.numInsts = 0;
 927                     self->trace[lastTraceDesc].info.frag.hint = kJitHintNone;
 928                     self->trace[lastTraceDesc].isCode = true;
 929                 }
 930
 931                 /* Mark the end of the trace runs */
 932                 self->trace[lastTraceDesc].info.frag.runEnd = true;
 933
 934                 JitTraceDescription* desc =
 935                    (JitTraceDescription*)malloc(sizeof(JitTraceDescription) +
 936                      sizeof(JitTraceRun) * (self->currTraceRun+1));
 937
 938                 if (desc == NULL) {
 939                     ALOGE("Out of memory in trace selection");
 940                     dvmJitStopTranslationRequests();
 941                     self->jitState = kJitDone;
 942                     allDone = true;
 943                     break;
 944                 }
 945
 946                 desc->method = self->traceMethod;
 947                 memcpy((char*)&(desc->trace[0]),
 948                     (char*)&(self->trace[0]),
 949                     sizeof(JitTraceRun) * (self->currTraceRun+1));
 950 #if defined(SHOW_TRACE)
 951                 ALOGD("TraceGen:  trace done, adding to queue");
 952                 dvmJitDumpTraceDesc(desc);
 953 #endif
 954                 if (dvmCompilerWorkEnqueue(
 955                        self->currTraceHead,kWorkOrderTrace,desc)) {
 956                     /* Work order successfully enqueued */
 957                     if (gDvmJit.blockingMode) {
 958                         dvmCompilerDrainQueue();
 959                     }
 960                 } else {
 961                     /*
 962                      * Make sure the descriptor for the abandoned work order is
 963                      * freed.
 964                      */
 965                     free(desc);
 966                 }
 967                 self->jitState = kJitDone;
 968                 allDone = true;
 969             }
 970             break;
 971         case kJitDone:
 972             allDone = true;
 973             break;
 974         case kJitNot:
 975             allDone = true;
 976             break;
 977         default:
 978             ALOGE("Unexpected JIT state: %d", self->jitState);
 979             dvmAbort();
 980             break;
 981     }
 982
 983     /*
 984      * If we're done with trace selection, switch off the control flags.
 985      */
 986      if (allDone) {
 987          dvmDisableSubMode(self, kSubModeJitTraceBuild);
 988          if (stayOneMoreInst) {
 989              // Clear jitResumeNPC explicitly since we know we don't need it
 990              // here.
 991              self->jitResumeNPC = NULL;
 992              // Keep going in single-step mode for at least one more inst
 993              if (self->singleStepCount == 0)
 994                  self->singleStepCount = 1;
 995              dvmEnableSubMode(self, kSubModeCountedStep);
 996          }
 997      }
 998      return;
 999 }
1000
1001 JitEntry *dvmJitFindEntry(const u2* pc, bool isMethodEntry)
1002 {
1003     int idx = dvmJitHash(pc);
1004
1005     /* Expect a high hit rate on 1st shot */
1006     if ((gDvmJit.pJitEntryTable[idx].dPC == pc) &&
1007         (gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry == isMethodEntry))
1008         return &gDvmJit.pJitEntryTable[idx];
1009     else {
1010         int chainEndMarker = gDvmJit.jitTableSize;
1011         while (gDvmJit.pJitEntryTable[idx].u.info.chain != chainEndMarker) {
1012             idx = gDvmJit.pJitEntryTable[idx].u.info.chain;
1013             if ((gDvmJit.pJitEntryTable[idx].dPC == pc) &&
1014                 (gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry ==
1015                 isMethodEntry))
1016                 return &gDvmJit.pJitEntryTable[idx];
1017         }
1018     }
1019     return NULL;
1020 }
1021
1022 /*
1023  * Walk through the JIT profile table and find the corresponding JIT code, in
1024  * the specified format (ie trace vs method). This routine needs to be fast.
1025  */
1026 void* getCodeAddrCommon(const u2* dPC, bool methodEntry)
1027 {
1028     int idx = dvmJitHash(dPC);
1029     const u2* pc = gDvmJit.pJitEntryTable[idx].dPC;
1030     if (pc != NULL) {
1031         bool hideTranslation = dvmJitHideTranslation();
1032         if (pc == dPC &&
1033             gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry == methodEntry) {
1034             int offset = (gDvmJit.profileMode >= kTraceProfilingContinuous) ?
1035                  0 : gDvmJit.pJitEntryTable[idx].u.info.profileOffset;
1036             intptr_t codeAddress =
1037                 (intptr_t)gDvmJit.pJitEntryTable[idx].codeAddress;
1038 #if defined(WITH_JIT_TUNING)
1039             gDvmJit.addrLookupsFound++;
1040 #endif
1041             return hideTranslation || !codeAddress ?  NULL :
1042                   (void *)(codeAddress + offset);
1043         } else {
1044             int chainEndMarker = gDvmJit.jitTableSize;
1045             while (gDvmJit.pJitEntryTable[idx].u.info.chain != chainEndMarker) {
1046                 idx = gDvmJit.pJitEntryTable[idx].u.info.chain;
1047                 if (gDvmJit.pJitEntryTable[idx].dPC == dPC &&
1048                     gDvmJit.pJitEntryTable[idx].u.info.isMethodEntry ==
1049                         methodEntry) {
1050                     int offset = (gDvmJit.profileMode >=
1051                         kTraceProfilingContinuous) ? 0 :
1052                         gDvmJit.pJitEntryTable[idx].u.info.profileOffset;
1053                     intptr_t codeAddress =
1054                         (intptr_t)gDvmJit.pJitEntryTable[idx].codeAddress;
1055 #if defined(WITH_JIT_TUNING)
1056                     gDvmJit.addrLookupsFound++;
1057 #endif
1058                     return hideTranslation || !codeAddress ? NULL :
1059                         (void *)(codeAddress + offset);
1060                 }
1061             }
1062         }
1063     }
1064 #if defined(WITH_JIT_TUNING)
1065     gDvmJit.addrLookupsNotFound++;
1066 #endif
1067     return NULL;
1068 }
1069
1070 /*
1071  * If a translated code address, in trace format, exists for the davik byte code
1072  * pointer return it.
1073  */
1074 void* dvmJitGetTraceAddr(const u2* dPC)
1075 {
1076     return getCodeAddrCommon(dPC, false /* method entry */);
1077 }
1078
1079 /*
1080  * If a translated code address, in whole-method format, exists for the davik
1081  * byte code pointer return it.
1082  */
1083 void* dvmJitGetMethodAddr(const u2* dPC)
1084 {
1085     return getCodeAddrCommon(dPC, true /* method entry */);
1086 }
1087
1088 /*
1089  * Similar to dvmJitGetTraceAddr, but returns null if the calling
1090  * thread is in a single-step mode.
1091  */
1092 void* dvmJitGetTraceAddrThread(const u2* dPC, Thread* self)
1093 {
1094     return (self->interpBreak.ctl.breakFlags != 0) ? NULL :
1095             getCodeAddrCommon(dPC, false /* method entry */);
1096 }
1097
1098 /*
1099  * Similar to dvmJitGetMethodAddr, but returns null if the calling
1100  * thread is in a single-step mode.
1101  */
1102 void* dvmJitGetMethodAddrThread(const u2* dPC, Thread* self)
1103 {
1104     return (self->interpBreak.ctl.breakFlags != 0) ? NULL :
1105             getCodeAddrCommon(dPC, true /* method entry */);
1106 }
1107
1108 /*
1109  * Register the translated code pointer into the JitTable.
1110  * NOTE: Once a codeAddress field transitions from initial state to
1111  * JIT'd code, it must not be altered without first halting all
1112  * threads.  We defer the setting of the profile prefix size until
1113  * after the new code address is set to ensure that the prefix offset
1114  * is never applied to the initial interpret-only translation.  All
1115  * translations with non-zero profile prefixes will still be correct
1116  * if entered as if the profile offset is 0, but the interpret-only
1117  * template cannot handle a non-zero prefix.
1118  * NOTE: JitTable must not be in danger of reset while this
1119  * code is executing. see Issue 4271784 for details.
1120  */
1121 void dvmJitSetCodeAddr(const u2* dPC, void *nPC, JitInstructionSetType set,
1122                        bool isMethodEntry, int profilePrefixSize)
1123 {
1124     JitEntryInfoUnion oldValue;
1125     JitEntryInfoUnion newValue;
1126     /*
1127      * Get the JitTable slot for this dPC (or create one if JitTable
1128      * has been reset between the time the trace was requested and
1129      * now.
1130      */
1131     JitEntry *jitEntry = isMethodEntry ?
1132         lookupAndAdd(dPC, false /* caller holds tableLock */, isMethodEntry) :
1133                      dvmJitFindEntry(dPC, isMethodEntry);
1134     assert(jitEntry);
1135     /* Note: order of update is important */
1136     do {
1137         oldValue = jitEntry->u;
1138         newValue = oldValue;
1139         newValue.info.isMethodEntry = isMethodEntry;
1140         newValue.info.instructionSet = set;
1141         newValue.info.profileOffset = profilePrefixSize;
1142     } while (android_atomic_release_cas(
1143              oldValue.infoWord, newValue.infoWord,
1144              &jitEntry->u.infoWord) != 0);
1145     jitEntry->codeAddress = nPC;
1146 }
1147
1148 /*
1149  * Determine if valid trace-bulding request is active.  If so, set
1150  * the proper flags in interpBreak and return.  Trace selection will
1151  * then begin normally via dvmCheckBefore.
1152  */
1153 void dvmJitCheckTraceRequest(Thread* self)
1154 {
1155     int i;
1156     /*
1157      * A note on trace "hotness" filtering:
1158      *
1159      * Our first level trigger is intentionally loose - we need it to
1160      * fire easily not just to identify potential traces to compile, but
1161      * also to allow re-entry into the code cache.
1162      *
1163      * The 2nd level filter (done here) exists to be selective about
1164      * what we actually compile.  It works by requiring the same
1165      * trace head "key" (defined as filterKey below) to appear twice in
1166      * a relatively short period of time.   The difficulty is defining the
1167      * shape of the filterKey.  Unfortunately, there is no "one size fits
1168      * all" approach.
1169      *
1170      * For spiky execution profiles dominated by a smallish
1171      * number of very hot loops, we would want the second-level filter
1172      * to be very selective.  A good selective filter is requiring an
1173      * exact match of the Dalvik PC.  In other words, defining filterKey as:
1174      *     intptr_t filterKey = (intptr_t)self->interpSave.pc
1175      *
1176      * However, for flat execution profiles we do best when aggressively
1177      * translating.  A heuristically decent proxy for this is to use
1178      * the value of the method pointer containing the trace as the filterKey.
1179      * Intuitively, this is saying that once any trace in a method appears hot,
1180      * immediately translate any other trace from that same method that
1181      * survives the first-level filter.  Here, filterKey would be defined as:
1182      *     intptr_t filterKey = (intptr_t)self->interpSave.method
1183      *
1184      * The problem is that we can't easily detect whether we're dealing
1185      * with a spiky or flat profile.  If we go with the "pc" match approach,
1186      * flat profiles perform poorly.  If we go with the loose "method" match,
1187      * we end up generating a lot of useless translations.  Probably the
1188      * best approach in the future will be to retain profile information
1189      * across runs of each application in order to determine it's profile,
1190      * and then choose once we have enough history.
1191      *
1192      * However, for now we've decided to chose a compromise filter scheme that
1193      * includes elements of both.  The high order bits of the filter key
1194      * are drawn from the enclosing method, and are combined with a slice
1195      * of the low-order bits of the Dalvik pc of the trace head.  The
1196      * looseness of the filter can be adjusted by changing with width of
1197      * the Dalvik pc slice (JIT_TRACE_THRESH_FILTER_PC_BITS).  The wider
1198      * the slice, the tighter the filter.
1199      *
1200      * Note: the fixed shifts in the function below reflect assumed word
1201      * alignment for method pointers, and half-word alignment of the Dalvik pc.
1202      * for method pointers and half-word alignment for dalvik pc.
1203      */
1204     u4 methodKey = (u4)self->interpSave.method <<
1205                    (JIT_TRACE_THRESH_FILTER_PC_BITS - 2);
1206     u4 pcKey = ((u4)self->interpSave.pc >> 1) &
1207                ((1 << JIT_TRACE_THRESH_FILTER_PC_BITS) - 1);
1208     intptr_t filterKey = (intptr_t)(methodKey | pcKey);
1209
1210     // Shouldn't be here if already building a trace.
1211     assert((self->interpBreak.ctl.subMode & kSubModeJitTraceBuild)==0);
1212
1213     /* Check if the JIT request can be handled now */
1214     if ((gDvmJit.pJitEntryTable != NULL) &&
1215         ((self->interpBreak.ctl.breakFlags & kInterpSingleStep) == 0)){
1216         /* Bypass the filter for hot trace requests or during stress mode */
1217         if (self->jitState == kJitTSelectRequest &&
1218             gDvmJit.threshold > 6) {
1219             /* Two-level filtering scheme */
1220             for (i=0; i< JIT_TRACE_THRESH_FILTER_SIZE; i++) {
1221                 if (filterKey == self->threshFilter[i]) {
1222                     self->threshFilter[i] = 0; // Reset filter entry
1223                     break;
1224                 }
1225             }
1226             if (i == JIT_TRACE_THRESH_FILTER_SIZE) {
1227                 /*
1228                  * Use random replacement policy - otherwise we could miss a
1229                  * large loop that contains more traces than the size of our
1230                  * filter array.
1231                  */
1232                 i = rand() % JIT_TRACE_THRESH_FILTER_SIZE;
1233                 self->threshFilter[i] = filterKey;
1234                 self->jitState = kJitDone;
1235             }
1236         }
1237
1238         /* If the compiler is backlogged, cancel any JIT actions */
1239         if (gDvmJit.compilerQueueLength >= gDvmJit.compilerHighWater) {
1240             self->jitState = kJitDone;
1241         }
1242
1243         /*
1244          * Check for additional reasons that might force the trace select
1245          * request to be dropped
1246          */
1247         if (self->jitState == kJitTSelectRequest ||
1248             self->jitState == kJitTSelectRequestHot) {
1249             if (dvmJitFindEntry(self->interpSave.pc, false)) {
1250                 /* In progress - nothing do do */
1251                self->jitState = kJitDone;
1252             } else {
1253                 JitEntry *slot = lookupAndAdd(self->interpSave.pc,
1254                                               false /* lock */,
1255                                               false /* method entry */);
1256                 if (slot == NULL) {
1257                     /*
1258                      * Table is full.  This should have been
1259                      * detected by the compiler thread and the table
1260                      * resized before we run into it here.  Assume bad things
1261                      * are afoot and disable profiling.
1262                      */
1263                     self->jitState = kJitDone;
1264                     ALOGD("JIT: JitTable full, disabling profiling");
1265                     dvmJitStopTranslationRequests();
1266                 }
1267             }
1268         }
1269
1270         switch (self->jitState) {
1271             case kJitTSelectRequest:
1272             case kJitTSelectRequestHot:
1273                 self->jitState = kJitTSelect;
1274                 self->traceMethod = self->interpSave.method;
1275                 self->currTraceHead = self->interpSave.pc;
1276                 self->currTraceRun = 0;
1277                 self->totalTraceLen = 0;
1278                 self->currRunHead = self->interpSave.pc;
1279                 self->currRunLen = 0;
1280                 self->trace[0].info.frag.startOffset =
1281                      self->interpSave.pc - self->interpSave.method->insns;
1282                 self->trace[0].info.frag.numInsts = 0;
1283                 self->trace[0].info.frag.runEnd = false;
1284                 self->trace[0].info.frag.hint = kJitHintNone;
1285                 self->trace[0].isCode = true;
1286                 self->lastPC = 0;
1287                 /* Turn on trace selection mode */
1288                 dvmEnableSubMode(self, kSubModeJitTraceBuild);
1289 #if defined(SHOW_TRACE)
1290                 ALOGD("Starting trace for %s at %#x",
1291                      self->interpSave.method->name, (int)self->interpSave.pc);
1292 #endif
1293                 break;
1294             case kJitDone:
1295                 break;
1296             default:
1297                 ALOGE("Unexpected JIT state: %d", self->jitState);
1298                 dvmAbort();
1299         }
1300     } else {
1301         /* Cannot build trace this time */
1302         self->jitState = kJitDone;
1303     }
1304 }
1305
1306 /*
1307  * Resizes the JitTable.  Must be a power of 2, and returns true on failure.
1308  * Stops all threads, and thus is a heavyweight operation. May only be called
1309  * by the compiler thread.
1310  */
1311 bool dvmJitResizeJitTable( unsigned int size )
1312 {
1313     JitEntry *pNewTable;
1314     JitEntry *pOldTable;
1315     JitEntry tempEntry;
1316     unsigned int oldSize;
1317     unsigned int i;
1318
1319     assert(gDvmJit.pJitEntryTable != NULL);
1320     assert(size && !(size & (size - 1)));   /* Is power of 2? */
1321
1322     ALOGI("Jit: resizing JitTable from %d to %d", gDvmJit.jitTableSize, size);
1323
1324     if (size <= gDvmJit.jitTableSize) {
1325         return true;
1326     }
1327
1328     /* Make sure requested size is compatible with chain field width */
1329     tempEntry.u.info.chain = size;
1330     if (tempEntry.u.info.chain != size) {
1331         ALOGD("Jit: JitTable request of %d too big", size);
1332         return true;
1333     }
1334
1335     pNewTable = (JitEntry*)calloc(size, sizeof(*pNewTable));
1336     if (pNewTable == NULL) {
1337         return true;
1338     }
1339     for (i=0; i< size; i++) {
1340         pNewTable[i].u.info.chain = size;  /* Initialize chain termination */
1341     }
1342
1343     /* Stop all other interpreting/jit'ng threads */
1344     dvmSuspendAllThreads(SUSPEND_FOR_TBL_RESIZE);
1345
1346     pOldTable = gDvmJit.pJitEntryTable;
1347     oldSize = gDvmJit.jitTableSize;
1348
1349     dvmLockMutex(&gDvmJit.tableLock);
1350     gDvmJit.pJitEntryTable = pNewTable;
1351     gDvmJit.jitTableSize = size;
1352     gDvmJit.jitTableMask = size - 1;
1353     gDvmJit.jitTableEntriesUsed = 0;
1354
1355     for (i=0; i < oldSize; i++) {
1356         if (pOldTable[i].dPC) {
1357             JitEntry *p;
1358             u2 chain;
1359             p = lookupAndAdd(pOldTable[i].dPC, true /* holds tableLock*/,
1360                              pOldTable[i].u.info.isMethodEntry);
1361             p->codeAddress = pOldTable[i].codeAddress;
1362             /* We need to preserve the new chain field, but copy the rest */
1363             chain = p->u.info.chain;
1364             p->u = pOldTable[i].u;
1365             p->u.info.chain = chain;
1366         }
1367     }
1368
1369     dvmUnlockMutex(&gDvmJit.tableLock);
1370
1371     free(pOldTable);
1372
1373     /* Restart the world */
1374     dvmResumeAllThreads(SUSPEND_FOR_TBL_RESIZE);
1375
1376     return false;
1377 }
1378
1379 /*
1380  * Reset the JitTable to the initial clean state.
1381  */
1382 void dvmJitResetTable()
1383 {
1384     JitEntry *jitEntry = gDvmJit.pJitEntryTable;
1385     unsigned int size = gDvmJit.jitTableSize;
1386     unsigned int i;
1387
1388     dvmLockMutex(&gDvmJit.tableLock);
1389
1390     /* Note: If need to preserve any existing counts. Do so here. */
1391     if (gDvmJit.pJitTraceProfCounters) {
1392         for (i=0; i < JIT_PROF_BLOCK_BUCKETS; i++) {
1393             if (gDvmJit.pJitTraceProfCounters->buckets[i])
1394                 memset((void *) gDvmJit.pJitTraceProfCounters->buckets[i],
1395                        0, sizeof(JitTraceCounter_t) * JIT_PROF_BLOCK_ENTRIES);
1396         }
1397         gDvmJit.pJitTraceProfCounters->next = 0;
1398     }
1399
1400     memset((void *) jitEntry, 0, sizeof(JitEntry) * size);
1401     for (i=0; i< size; i++) {
1402         jitEntry[i].u.info.chain = size;  /* Initialize chain termination */
1403     }
1404     gDvmJit.jitTableEntriesUsed = 0;
1405     dvmUnlockMutex(&gDvmJit.tableLock);
1406 }
1407
1408 /*
1409  * Return the address of the next trace profile counter.  This address
1410  * will be embedded in the generated code for the trace, and thus cannot
1411  * change while the trace exists.
1412  */
1413 JitTraceCounter_t *dvmJitNextTraceCounter()
1414 {
1415     int idx = gDvmJit.pJitTraceProfCounters->next / JIT_PROF_BLOCK_ENTRIES;
1416     int elem = gDvmJit.pJitTraceProfCounters->next % JIT_PROF_BLOCK_ENTRIES;
1417     JitTraceCounter_t *res;
1418     /* Lazily allocate blocks of counters */
1419     if (!gDvmJit.pJitTraceProfCounters->buckets[idx]) {
1420         JitTraceCounter_t *p =
1421               (JitTraceCounter_t*) calloc(JIT_PROF_BLOCK_ENTRIES, sizeof(*p));
1422         if (!p) {
1423             ALOGE("Failed to allocate block of trace profile counters");
1424             dvmAbort();
1425         }
1426         gDvmJit.pJitTraceProfCounters->buckets[idx] = p;
1427     }
1428     res = &gDvmJit.pJitTraceProfCounters->buckets[idx][elem];
1429     gDvmJit.pJitTraceProfCounters->next++;
1430     return res;
1431 }
1432
1433 /*
1434  * Float/double conversion requires clamping to min and max of integer form.  If
1435  * target doesn't support this normally, use these.
1436  */
1437 s8 dvmJitd2l(double d)
1438 {
1439     static const double kMaxLong = (double)(s8)0x7fffffffffffffffULL;
1440     static const double kMinLong = (double)(s8)0x8000000000000000ULL;
1441     if (d >= kMaxLong)
1442         return (s8)0x7fffffffffffffffULL;
1443     else if (d <= kMinLong)
1444         return (s8)0x8000000000000000ULL;
1445     else if (d != d) // NaN case
1446         return 0;
1447     else
1448         return (s8)d;
1449 }
1450
1451 s8 dvmJitf2l(float f)
1452 {
1453     static const float kMaxLong = (float)(s8)0x7fffffffffffffffULL;
1454     static const float kMinLong = (float)(s8)0x8000000000000000ULL;
1455     if (f >= kMaxLong)
1456         return (s8)0x7fffffffffffffffULL;
1457     else if (f <= kMinLong)
1458         return (s8)0x8000000000000000ULL;
1459     else if (f != f) // NaN case
1460         return 0;
1461     else
1462         return (s8)f;
1463 }
1464
1465 /* Should only be called by the compiler thread */
1466 void dvmJitChangeProfileMode(TraceProfilingModes newState)
1467 {
1468     if (gDvmJit.profileMode != newState) {
1469         gDvmJit.profileMode = newState;
1470         dvmJitUnchainAll();
1471     }
1472 }
1473
1474 void dvmJitTraceProfilingOn()
1475 {
1476     if (gDvmJit.profileMode == kTraceProfilingPeriodicOff)
1477         dvmCompilerForceWorkEnqueue(NULL, kWorkOrderProfileMode,
1478                                     (void*) kTraceProfilingPeriodicOn);
1479     else if (gDvmJit.profileMode == kTraceProfilingDisabled)
1480         dvmCompilerForceWorkEnqueue(NULL, kWorkOrderProfileMode,
1481                                     (void*) kTraceProfilingContinuous);
1482 }
1483
1484 void dvmJitTraceProfilingOff()
1485 {
1486     if (gDvmJit.profileMode == kTraceProfilingPeriodicOn)
1487         dvmCompilerForceWorkEnqueue(NULL, kWorkOrderProfileMode,
1488                                     (void*) kTraceProfilingPeriodicOff);
1489     else if (gDvmJit.profileMode == kTraceProfilingContinuous)
1490         dvmCompilerForceWorkEnqueue(NULL, kWorkOrderProfileMode,
1491                                     (void*) kTraceProfilingDisabled);
1492 }
1493
1494 /*
1495  * Update JIT-specific info in Thread structure for a single thread
1496  */
1497 void dvmJitUpdateThreadStateSingle(Thread* thread)
1498 {
1499     thread->pJitProfTable = gDvmJit.pProfTable;
1500     thread->jitThreshold = gDvmJit.threshold;
1501 }
1502
1503 /*
1504  * Walk through the thread list and refresh all local copies of
1505  * JIT global state (which was placed there for fast access).
1506  */
1507 void dvmJitUpdateThreadStateAll()
1508 {
1509     Thread* self = dvmThreadSelf();
1510     Thread* thread;
1511
1512     dvmLockThreadList(self);
1513     for (thread = gDvm.threadList; thread != NULL; thread = thread->next) {
1514         dvmJitUpdateThreadStateSingle(thread);
1515     }
1516     dvmUnlockThreadList();
1517
1518 }
1519 #endif /* WITH_JIT */