2 * Copyright (C) 2008 The Android Open Source Project
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
19 * Target independent portion of Android's Jit
26 #include "dexdump/OpCodeNames.h"
31 #include "compiler/Compiler.h"
32 #include "compiler/CompilerUtility.h"
33 #include "compiler/CompilerIR.h"
36 #if defined(WITH_SELF_VERIFICATION)
37 /* Allocate space for per-thread ShadowSpace data structures */
38 void* dvmSelfVerificationShadowSpaceAlloc(Thread* self)
40 self->shadowSpace = (ShadowSpace*) calloc(1, sizeof(ShadowSpace));
41 if (self->shadowSpace == NULL)
44 self->shadowSpace->registerSpaceSize = REG_SPACE;
45 self->shadowSpace->registerSpace =
46 (int*) calloc(self->shadowSpace->registerSpaceSize, sizeof(int));
48 return self->shadowSpace->registerSpace;
51 /* Free per-thread ShadowSpace data structures */
52 void dvmSelfVerificationShadowSpaceFree(Thread* self)
54 free(self->shadowSpace->registerSpace);
55 free(self->shadowSpace);
59 * Save out PC, FP, InterpState, and registers to shadow space.
60 * Return a pointer to the shadow space for JIT to use.
62 void* dvmSelfVerificationSaveState(const u2* pc, const void* fp,
63 InterpState* interpState, int targetTrace)
65 Thread *self = dvmThreadSelf();
66 ShadowSpace *shadowSpace = self->shadowSpace;
67 int preBytes = interpState->method->outsSize*4 + sizeof(StackSaveArea);
68 int postBytes = interpState->method->registersSize*4;
70 //LOGD("### selfVerificationSaveState(%d) pc: 0x%x fp: 0x%x",
71 // self->threadId, (int)pc, (int)fp);
73 if (shadowSpace->selfVerificationState != kSVSIdle) {
74 LOGD("~~~ Save: INCORRECT PREVIOUS STATE(%d): %d",
75 self->threadId, shadowSpace->selfVerificationState);
76 LOGD("********** SHADOW STATE DUMP **********");
77 LOGD("PC: 0x%x FP: 0x%x", (int)pc, (int)fp);
79 shadowSpace->selfVerificationState = kSVSStart;
81 // Dynamically grow shadow register space if necessary
82 while (preBytes + postBytes > shadowSpace->registerSpaceSize) {
83 shadowSpace->registerSpaceSize *= 2;
84 free(shadowSpace->registerSpace);
85 shadowSpace->registerSpace =
86 (int*) calloc(shadowSpace->registerSpaceSize, sizeof(int));
89 // Remember original state
90 shadowSpace->startPC = pc;
92 shadowSpace->glue = interpState;
94 * Store the original method here in case the trace ends with a
95 * return/invoke, the last method.
97 shadowSpace->method = interpState->method;
98 shadowSpace->shadowFP = shadowSpace->registerSpace +
99 shadowSpace->registerSpaceSize - postBytes/4;
101 // Create a copy of the InterpState
102 //shadowSpace->interpState = *interpState;
103 memcpy(&(shadowSpace->interpState), interpState, sizeof(InterpState));
104 shadowSpace->interpState.fp = shadowSpace->shadowFP;
105 shadowSpace->interpState.interpStackEnd = (u1*)shadowSpace->registerSpace;
107 // Create a copy of the stack
108 memcpy(((char*)shadowSpace->shadowFP)-preBytes, ((char*)fp)-preBytes,
111 // Setup the shadowed heap space
112 shadowSpace->heapSpaceTail = shadowSpace->heapSpace;
114 // Reset trace length
115 shadowSpace->traceLength = 0;
121 * Save ending PC, FP and compiled code exit point to shadow space.
122 * Return a pointer to the shadow space for JIT to restore state.
124 void* dvmSelfVerificationRestoreState(const u2* pc, const void* fp,
125 SelfVerificationState exitPoint)
127 Thread *self = dvmThreadSelf();
128 ShadowSpace *shadowSpace = self->shadowSpace;
129 shadowSpace->endPC = pc;
130 shadowSpace->endShadowFP = fp;
132 //LOGD("### selfVerificationRestoreState(%d) pc: 0x%x fp: 0x%x endPC: 0x%x",
133 // self->threadId, (int)shadowSpace->startPC, (int)shadowSpace->fp,
136 if (shadowSpace->selfVerificationState != kSVSStart) {
137 LOGD("~~~ Restore: INCORRECT PREVIOUS STATE(%d): %d",
138 self->threadId, shadowSpace->selfVerificationState);
139 LOGD("********** SHADOW STATE DUMP **********");
140 LOGD("Dalvik PC: 0x%x endPC: 0x%x", (int)shadowSpace->startPC,
141 (int)shadowSpace->endPC);
142 LOGD("Interp FP: 0x%x", (int)shadowSpace->fp);
143 LOGD("Shadow FP: 0x%x endFP: 0x%x", (int)shadowSpace->shadowFP,
144 (int)shadowSpace->endShadowFP);
147 // Special case when punting after a single instruction
148 if (exitPoint == kSVSPunt && pc == shadowSpace->startPC) {
149 shadowSpace->selfVerificationState = kSVSIdle;
151 shadowSpace->selfVerificationState = exitPoint;
157 /* Print contents of virtual registers */
158 static void selfVerificationPrintRegisters(int* addr, int* addrRef,
162 for (i = 0; i < numWords; i++) {
163 LOGD("(v%d) 0x%8x%s", i, addr[i], addr[i] != addrRef[i] ? " X" : "");
167 /* Print values maintained in shadowSpace */
168 static void selfVerificationDumpState(const u2* pc, Thread* self)
170 ShadowSpace* shadowSpace = self->shadowSpace;
171 StackSaveArea* stackSave = SAVEAREA_FROM_FP(self->curFrame);
172 int frameBytes = (int) shadowSpace->registerSpace +
173 shadowSpace->registerSpaceSize*4 -
174 (int) shadowSpace->shadowFP;
177 if (self->curFrame < shadowSpace->fp) {
178 localRegs = (stackSave->method->registersSize -
179 stackSave->method->insSize)*4;
180 frameBytes2 = (int) shadowSpace->fp - (int) self->curFrame - localRegs;
182 LOGD("********** SHADOW STATE DUMP **********");
183 LOGD("CurrentPC: 0x%x, Offset: 0x%04x", (int)pc,
184 (int)(pc - stackSave->method->insns));
185 LOGD("Class: %s", shadowSpace->method->clazz->descriptor);
186 LOGD("Method: %s", shadowSpace->method->name);
187 LOGD("Dalvik PC: 0x%x endPC: 0x%x", (int)shadowSpace->startPC,
188 (int)shadowSpace->endPC);
189 LOGD("Interp FP: 0x%x endFP: 0x%x", (int)shadowSpace->fp,
190 (int)self->curFrame);
191 LOGD("Shadow FP: 0x%x endFP: 0x%x", (int)shadowSpace->shadowFP,
192 (int)shadowSpace->endShadowFP);
193 LOGD("Frame1 Bytes: %d Frame2 Local: %d Bytes: %d", frameBytes,
194 localRegs, frameBytes2);
195 LOGD("Trace length: %d State: %d", shadowSpace->traceLength,
196 shadowSpace->selfVerificationState);
199 /* Print decoded instructions in the current trace */
200 static void selfVerificationDumpTrace(const u2* pc, Thread* self)
202 ShadowSpace* shadowSpace = self->shadowSpace;
203 StackSaveArea* stackSave = SAVEAREA_FROM_FP(self->curFrame);
205 DecodedInstruction *decInsn;
207 LOGD("********** SHADOW TRACE DUMP **********");
208 for (i = 0; i < shadowSpace->traceLength; i++) {
209 addr = shadowSpace->trace[i].addr;
210 offset = (int)((u2*)addr - stackSave->method->insns);
211 decInsn = &(shadowSpace->trace[i].decInsn);
212 /* Not properly decoding instruction, some registers may be garbage */
213 LOGD("0x%x: (0x%04x) %s", addr, offset, getOpcodeName(decInsn->opCode));
217 /* Code is forced into this spin loop when a divergence is detected */
218 static void selfVerificationSpinLoop(ShadowSpace *shadowSpace)
220 const u2 *startPC = shadowSpace->startPC;
221 JitTraceDescription* desc = dvmCopyTraceDescriptor(startPC);
223 dvmCompilerWorkEnqueue(startPC, kWorkOrderTraceDebug, desc);
225 gDvmJit.selfVerificationSpin = true;
226 while(gDvmJit.selfVerificationSpin) sleep(10);
229 /* Manage self verification while in the debug interpreter */
230 static bool selfVerificationDebugInterp(const u2* pc, Thread* self)
232 ShadowSpace *shadowSpace = self->shadowSpace;
233 SelfVerificationState state = shadowSpace->selfVerificationState;
235 DecodedInstruction decInsn;
236 dexDecodeInstruction(gDvm.instrFormat, pc, &decInsn);
238 //LOGD("### DbgIntp(%d): PC: 0x%x endPC: 0x%x state: %d len: %d %s",
239 // self->threadId, (int)pc, (int)shadowSpace->endPC, state,
240 // shadowSpace->traceLength, getOpcodeName(decInsn.opCode));
242 if (state == kSVSIdle || state == kSVSStart) {
243 LOGD("~~~ DbgIntrp: INCORRECT PREVIOUS STATE(%d): %d",
244 self->threadId, state);
245 selfVerificationDumpState(pc, self);
246 selfVerificationDumpTrace(pc, self);
249 /* Skip endPC once when trace has a backward branch */
250 if ((state == kSVSBackwardBranch && pc == shadowSpace->endPC) ||
251 state != kSVSBackwardBranch) {
252 shadowSpace->selfVerificationState = kSVSDebugInterp;
255 /* Check that the current pc is the end of the trace */
256 if ((state == kSVSSingleStep || state == kSVSDebugInterp) &&
257 pc == shadowSpace->endPC) {
259 shadowSpace->selfVerificationState = kSVSIdle;
261 /* Check register space */
262 int frameBytes = (int) shadowSpace->registerSpace +
263 shadowSpace->registerSpaceSize*4 -
264 (int) shadowSpace->shadowFP;
265 if (memcmp(shadowSpace->fp, shadowSpace->shadowFP, frameBytes)) {
266 LOGD("~~~ DbgIntp(%d): REGISTERS DIVERGENCE!", self->threadId);
267 selfVerificationDumpState(pc, self);
268 selfVerificationDumpTrace(pc, self);
269 LOGD("*** Interp Registers: addr: 0x%x bytes: %d",
270 (int)shadowSpace->fp, frameBytes);
271 selfVerificationPrintRegisters((int*)shadowSpace->fp,
272 (int*)shadowSpace->shadowFP,
274 LOGD("*** Shadow Registers: addr: 0x%x bytes: %d",
275 (int)shadowSpace->shadowFP, frameBytes);
276 selfVerificationPrintRegisters((int*)shadowSpace->shadowFP,
277 (int*)shadowSpace->fp,
279 selfVerificationSpinLoop(shadowSpace);
281 /* Check new frame if it exists (invokes only) */
282 if (self->curFrame < shadowSpace->fp) {
283 StackSaveArea* stackSave = SAVEAREA_FROM_FP(self->curFrame);
284 int localRegs = (stackSave->method->registersSize -
285 stackSave->method->insSize)*4;
286 int frameBytes2 = (int) shadowSpace->fp -
287 (int) self->curFrame - localRegs;
288 if (memcmp(((char*)self->curFrame)+localRegs,
289 ((char*)shadowSpace->endShadowFP)+localRegs, frameBytes2)) {
290 LOGD("~~~ DbgIntp(%d): REGISTERS (FRAME2) DIVERGENCE!",
292 selfVerificationDumpState(pc, self);
293 selfVerificationDumpTrace(pc, self);
294 LOGD("*** Interp Registers: addr: 0x%x l: %d bytes: %d",
295 (int)self->curFrame, localRegs, frameBytes2);
296 selfVerificationPrintRegisters((int*)self->curFrame,
297 (int*)shadowSpace->endShadowFP,
298 (frameBytes2+localRegs)/4);
299 LOGD("*** Shadow Registers: addr: 0x%x l: %d bytes: %d",
300 (int)shadowSpace->endShadowFP, localRegs, frameBytes2);
301 selfVerificationPrintRegisters((int*)shadowSpace->endShadowFP,
302 (int*)self->curFrame,
303 (frameBytes2+localRegs)/4);
304 selfVerificationSpinLoop(shadowSpace);
308 /* Check memory space */
309 bool memDiff = false;
310 ShadowHeap* heapSpacePtr;
311 for (heapSpacePtr = shadowSpace->heapSpace;
312 heapSpacePtr != shadowSpace->heapSpaceTail; heapSpacePtr++) {
313 int memData = *((unsigned int*) heapSpacePtr->addr);
314 if (heapSpacePtr->data != memData) {
315 LOGD("~~~ DbgIntp(%d): MEMORY DIVERGENCE!", self->threadId);
316 LOGD("Addr: 0x%x Intrp Data: 0x%x Jit Data: 0x%x",
317 heapSpacePtr->addr, memData, heapSpacePtr->data);
318 selfVerificationDumpState(pc, self);
319 selfVerificationDumpTrace(pc, self);
323 if (memDiff) selfVerificationSpinLoop(shadowSpace);
326 /* If end not been reached, make sure max length not exceeded */
327 } else if (shadowSpace->traceLength >= JIT_MAX_TRACE_LEN) {
328 LOGD("~~~ DbgIntp(%d): CONTROL DIVERGENCE!", self->threadId);
329 LOGD("startPC: 0x%x endPC: 0x%x currPC: 0x%x",
330 (int)shadowSpace->startPC, (int)shadowSpace->endPC, (int)pc);
331 selfVerificationDumpState(pc, self);
332 selfVerificationDumpTrace(pc, self);
333 selfVerificationSpinLoop(shadowSpace);
337 /* Log the instruction address and decoded instruction for debug */
338 shadowSpace->trace[shadowSpace->traceLength].addr = (int)pc;
339 shadowSpace->trace[shadowSpace->traceLength].decInsn = decInsn;
340 shadowSpace->traceLength++;
346 int dvmJitStartup(void)
349 bool res = true; /* Assume success */
351 // Create the compiler thread and setup miscellaneous chores */
352 res &= dvmCompilerStartup();
354 dvmInitMutex(&gDvmJit.tableLock);
355 if (res && gDvm.executionMode == kExecutionModeJit) {
356 JitEntry *pJitTable = NULL;
357 unsigned char *pJitProfTable = NULL;
359 assert(gDvmJit.jitTableSize &&
360 !(gDvmJit.jitTableSize & (gDvmJit.jitTableSize - 1)));
361 dvmLockMutex(&gDvmJit.tableLock);
362 pJitTable = (JitEntry*)
363 calloc(gDvmJit.jitTableSize, sizeof(*pJitTable));
365 LOGE("jit table allocation failed\n");
370 * NOTE: the profile table must only be allocated once, globally.
371 * Profiling is turned on and off by nulling out gDvm.pJitProfTable
372 * and then restoring its original value. However, this action
373 * is not syncronized for speed so threads may continue to hold
374 * and update the profile table after profiling has been turned
375 * off by null'ng the global pointer. Be aware.
377 pJitProfTable = (unsigned char *)malloc(JIT_PROF_SIZE);
378 if (!pJitProfTable) {
379 LOGE("jit prof table allocation failed\n");
383 memset(pJitProfTable, gDvmJit.threshold, JIT_PROF_SIZE);
384 for (i=0; i < gDvmJit.jitTableSize; i++) {
385 pJitTable[i].u.info.chain = gDvmJit.jitTableSize;
387 /* Is chain field wide enough for termination pattern? */
388 assert(pJitTable[0].u.info.chain == gDvmJit.jitTableSize);
391 gDvmJit.pJitEntryTable = pJitTable;
392 gDvmJit.jitTableMask = gDvmJit.jitTableSize - 1;
393 gDvmJit.jitTableEntriesUsed = 0;
394 gDvmJit.pProfTable = pJitProfTable;
395 dvmUnlockMutex(&gDvmJit.tableLock);
401 * If one of our fixed tables or the translation buffer fills up,
402 * call this routine to avoid wasting cycles on future translation requests.
404 void dvmJitStopTranslationRequests()
407 * Note 1: This won't necessarily stop all translation requests, and
408 * operates on a delayed mechanism. Running threads look to the copy
409 * of this value in their private InterpState structures and won't see
410 * this change until it is refreshed (which happens on interpreter
412 * Note 2: This is a one-shot memory leak on this table. Because this is a
413 * permanent off switch for Jit profiling, it is a one-time leak of 1K
414 * bytes, and no further attempt will be made to re-allocate it. Can't
415 * free it because some thread may be holding a reference.
417 gDvmJit.pProfTable = NULL;
420 #if defined(EXIT_STATS)
421 /* Convenience function to increment counter from assembly code */
422 void dvmBumpNoChain(int from)
424 gDvmJit.noChainExit[from]++;
427 /* Convenience function to increment counter from assembly code */
430 gDvmJit.normalExit++;
433 /* Convenience function to increment counter from assembly code */
434 void dvmBumpPunt(int from)
440 /* Dumps debugging & tuning stats to the log */
448 if (gDvmJit.pJitEntryTable) {
449 for (i=0, stubs=chains=hit=not_hit=0;
450 i < (int) gDvmJit.jitTableSize;
452 if (gDvmJit.pJitEntryTable[i].dPC != 0) {
454 if (gDvmJit.pJitEntryTable[i].codeAddress ==
455 gDvmJit.interpretTemplate)
459 if (gDvmJit.pJitEntryTable[i].u.info.chain != gDvmJit.jitTableSize)
463 "JIT: %d traces, %d slots, %d chains, %d maxQ, %d thresh, %s",
464 hit, not_hit + hit, chains, gDvmJit.compilerMaxQueued,
465 gDvmJit.threshold, gDvmJit.blockingMode ? "Blocking" : "Non-blocking");
466 #if defined(EXIT_STATS)
468 "JIT: Lookups: %d hits, %d misses; %d normal, %d punt",
469 gDvmJit.addrLookupsFound, gDvmJit.addrLookupsNotFound,
470 gDvmJit.normalExit, gDvmJit.puntExit);
472 "JIT: noChainExit: %d IC miss, %d interp callsite, %d switch overflow",
473 gDvmJit.noChainExit[kInlineCacheMiss],
474 gDvmJit.noChainExit[kCallsiteInterpreted],
475 gDvmJit.noChainExit[kSwitchOverflow]);
477 LOGD("JIT: %d Translation chains, %d interp stubs",
478 gDvmJit.translationChains, stubs);
479 #if defined(INVOKE_STATS)
480 LOGD("JIT: Invoke: %d chainable, %d pred. chain, %d native, "
482 gDvmJit.invokeChain, gDvmJit.invokePredictedChain,
483 gDvmJit.invokeNative, gDvmJit.returnOp);
485 if (gDvmJit.profile) {
486 dvmCompilerSortAndPrintTraceProfiles();
493 * Final JIT shutdown. Only do this once, and do not attempt to restart
496 void dvmJitShutdown(void)
498 /* Shutdown the compiler thread */
500 dvmCompilerShutdown();
502 dvmCompilerDumpStats();
504 dvmDestroyMutex(&gDvmJit.tableLock);
506 if (gDvmJit.pJitEntryTable) {
507 free(gDvmJit.pJitEntryTable);
508 gDvmJit.pJitEntryTable = NULL;
511 if (gDvmJit.pProfTable) {
512 free(gDvmJit.pProfTable);
513 gDvmJit.pProfTable = NULL;
517 void setTraceConstruction(JitEntry *slot, bool value)
520 JitEntryInfoUnion oldValue;
521 JitEntryInfoUnion newValue;
525 newValue.info.traceConstruction = value;
526 } while (!ATOMIC_CMP_SWAP( &slot->u.infoWord,
527 oldValue.infoWord, newValue.infoWord));
530 void resetTracehead(InterpState* interpState, JitEntry *slot)
532 slot->codeAddress = gDvmJit.interpretTemplate;
533 setTraceConstruction(slot, false);
536 /* Clean up any pending trace builds */
537 void dvmJitAbortTraceSelect(InterpState* interpState)
539 if (interpState->jitState == kJitTSelect)
540 interpState->jitState = kJitTSelectAbort;
543 #if defined(WITH_SELF_VERIFICATION)
544 static bool selfVerificationPuntOps(DecodedInstruction *decInsn)
546 OpCode op = decInsn->opCode;
547 int flags = dexGetInstrFlags(gDvm.instrFlags, op);
548 return (op == OP_MONITOR_ENTER || op == OP_MONITOR_EXIT ||
549 op == OP_NEW_INSTANCE || op == OP_NEW_ARRAY ||
550 op == OP_CHECK_CAST || op == OP_MOVE_EXCEPTION ||
551 (flags & kInstrInvoke));
556 * Adds to the current trace request one instruction at a time, just
557 * before that instruction is interpreted. This is the primary trace
558 * selection function. NOTE: return instruction are handled a little
559 * differently. In general, instructions are "proposed" to be added
560 * to the current trace prior to interpretation. If the interpreter
561 * then successfully completes the instruction, is will be considered
562 * part of the request. This allows us to examine machine state prior
563 * to interpretation, and also abort the trace request if the instruction
564 * throws or does something unexpected. However, return instructions
565 * will cause an immediate end to the translation request - which will
566 * be passed to the compiler before the return completes. This is done
567 * in response to special handling of returns by the interpreter (and
568 * because returns cannot throw in a way that causes problems for the
571 int dvmCheckJit(const u2* pc, Thread* self, InterpState* interpState)
574 int switchInterp = false;
575 int debugOrProfile = (gDvm.debuggerActive || self->suspendCount
576 #if defined(WITH_PROFILER)
577 || gDvm.activeProfilers
581 /* Prepare to handle last PC and stage the current PC */
582 const u2 *lastPC = interpState->lastPC;
583 interpState->lastPC = pc;
585 #if defined(WITH_SELF_VERIFICATION)
587 * We can't allow some instructions to be executed twice, and so they
588 * must not appear in any translations. End the trace before they
591 if (lastPC && interpState->jitState == kJitTSelect) {
592 DecodedInstruction decInsn;
593 dexDecodeInstruction(gDvm.instrFormat, lastPC, &decInsn);
594 if (selfVerificationPuntOps(&decInsn)) {
595 interpState->jitState = kJitTSelectEnd;
600 switch (interpState->jitState) {
604 DecodedInstruction decInsn;
606 /* First instruction - just remember the PC and exit */
607 if (lastPC == NULL) break;
608 /* Grow the trace around the last PC if jitState is kJitTSelect */
609 dexDecodeInstruction(gDvm.instrFormat, lastPC, &decInsn);
612 * Treat {PACKED,SPARSE}_SWITCH as trace-ending instructions due
613 * to the amount of space it takes to generate the chaining
616 if (interpState->totalTraceLen != 0 &&
617 (decInsn.opCode == OP_PACKED_SWITCH ||
618 decInsn.opCode == OP_SPARSE_SWITCH)) {
619 interpState->jitState = kJitTSelectEnd;
624 #if defined(SHOW_TRACE)
625 LOGD("TraceGen: adding %s",getOpcodeName(decInsn.opCode));
627 flags = dexGetInstrFlags(gDvm.instrFlags, decInsn.opCode);
628 len = dexGetInstrOrTableWidthAbs(gDvm.instrWidth, lastPC);
629 offset = lastPC - interpState->method->insns;
630 assert((unsigned) offset <
631 dvmGetMethodInsnsSize(interpState->method));
632 if (lastPC != interpState->currRunHead + interpState->currRunLen) {
634 /* We need to start a new trace run */
635 currTraceRun = ++interpState->currTraceRun;
636 interpState->currRunLen = 0;
637 interpState->currRunHead = (u2*)lastPC;
638 interpState->trace[currTraceRun].frag.startOffset = offset;
639 interpState->trace[currTraceRun].frag.numInsts = 0;
640 interpState->trace[currTraceRun].frag.runEnd = false;
641 interpState->trace[currTraceRun].frag.hint = kJitHintNone;
643 interpState->trace[interpState->currTraceRun].frag.numInsts++;
644 interpState->totalTraceLen++;
645 interpState->currRunLen += len;
647 /* Will probably never hit this with the current trace buildier */
648 if (interpState->currTraceRun == (MAX_JIT_RUN_LEN - 1)) {
649 interpState->jitState = kJitTSelectEnd;
652 if ( ((flags & kInstrUnconditional) == 0) &&
653 /* don't end trace on INVOKE_DIRECT_EMPTY */
654 (decInsn.opCode != OP_INVOKE_DIRECT_EMPTY) &&
655 ((flags & (kInstrCanBranch |
658 kInstrInvoke)) != 0)) {
659 interpState->jitState = kJitTSelectEnd;
660 #if defined(SHOW_TRACE)
661 LOGD("TraceGen: ending on %s, basic block end",
662 getOpcodeName(decInsn.opCode));
665 /* Break on throw or self-loop */
666 if ((decInsn.opCode == OP_THROW) || (lastPC == pc)){
667 interpState->jitState = kJitTSelectEnd;
669 if (interpState->totalTraceLen >= JIT_MAX_TRACE_LEN) {
670 interpState->jitState = kJitTSelectEnd;
672 if (debugOrProfile) {
673 interpState->jitState = kJitTSelectAbort;
674 switchInterp = !debugOrProfile;
677 if ((flags & kInstrCanReturn) != kInstrCanReturn) {
680 /* NOTE: intentional fallthrough for returns */
683 if (interpState->totalTraceLen == 0) {
684 /* Bad trace - mark as untranslatable */
685 dvmJitAbortTraceSelect(interpState);
686 switchInterp = !debugOrProfile;
689 JitTraceDescription* desc =
690 (JitTraceDescription*)malloc(sizeof(JitTraceDescription) +
691 sizeof(JitTraceRun) * (interpState->currTraceRun+1));
693 LOGE("Out of memory in trace selection");
694 dvmJitStopTranslationRequests();
695 interpState->jitState = kJitTSelectAbort;
696 dvmJitAbortTraceSelect(interpState);
697 switchInterp = !debugOrProfile;
700 interpState->trace[interpState->currTraceRun].frag.runEnd =
702 interpState->jitState = kJitNormal;
703 desc->method = interpState->method;
704 memcpy((char*)&(desc->trace[0]),
705 (char*)&(interpState->trace[0]),
706 sizeof(JitTraceRun) * (interpState->currTraceRun+1));
707 #if defined(SHOW_TRACE)
708 LOGD("TraceGen: trace done, adding to queue");
710 dvmCompilerWorkEnqueue(
711 interpState->currTraceHead,kWorkOrderTrace,desc);
712 setTraceConstruction(
713 dvmJitLookupAndAdd(interpState->currTraceHead), false);
714 if (gDvmJit.blockingMode) {
715 dvmCompilerDrainQueue();
717 switchInterp = !debugOrProfile;
721 interpState->jitState = kJitSingleStepEnd;
723 case kJitSingleStepEnd:
724 interpState->entryPoint = kInterpEntryResume;
725 switchInterp = !debugOrProfile;
727 case kJitTSelectAbort:
728 #if defined(SHOW_TRACE)
729 LOGD("TraceGen: trace abort");
731 dvmJitAbortTraceSelect(interpState);
732 interpState->jitState = kJitNormal;
733 switchInterp = !debugOrProfile;
736 switchInterp = !debugOrProfile;
738 #if defined(WITH_SELF_VERIFICATION)
739 case kJitSelfVerification:
740 if (selfVerificationDebugInterp(pc, self)) {
741 interpState->jitState = kJitNormal;
742 switchInterp = !debugOrProfile;
746 /* If JIT is off stay out of interpreter selections */
750 if (!debugOrProfile) {
751 LOGE("Unexpected JIT state: %d", interpState->jitState);
759 JitEntry *dvmFindJitEntry(const u2* pc)
761 int idx = dvmJitHash(pc);
763 /* Expect a high hit rate on 1st shot */
764 if (gDvmJit.pJitEntryTable[idx].dPC == pc)
765 return &gDvmJit.pJitEntryTable[idx];
767 int chainEndMarker = gDvmJit.jitTableSize;
768 while (gDvmJit.pJitEntryTable[idx].u.info.chain != chainEndMarker) {
769 idx = gDvmJit.pJitEntryTable[idx].u.info.chain;
770 if (gDvmJit.pJitEntryTable[idx].dPC == pc)
771 return &gDvmJit.pJitEntryTable[idx];
778 * If a translated code address exists for the davik byte code
779 * pointer return it. This routine needs to be fast.
781 void* dvmJitGetCodeAddr(const u2* dPC)
783 int idx = dvmJitHash(dPC);
784 u2* npc = gDvmJit.pJitEntryTable[idx].dPC;
788 #if defined(EXIT_STATS)
789 gDvmJit.addrLookupsFound++;
791 return gDvm.sumThreadSuspendCount ? NULL :
792 gDvmJit.pJitEntryTable[idx].codeAddress;
794 int chainEndMarker = gDvmJit.jitTableSize;
795 while (gDvmJit.pJitEntryTable[idx].u.info.chain != chainEndMarker) {
796 idx = gDvmJit.pJitEntryTable[idx].u.info.chain;
797 if (gDvmJit.pJitEntryTable[idx].dPC == dPC) {
798 #if defined(EXIT_STATS)
799 gDvmJit.addrLookupsFound++;
801 return gDvm.sumThreadSuspendCount ? NULL :
802 gDvmJit.pJitEntryTable[idx].codeAddress;
807 #if defined(EXIT_STATS)
808 gDvmJit.addrLookupsNotFound++;
814 * Find an entry in the JitTable, creating if necessary.
815 * Returns null if table is full.
817 JitEntry *dvmJitLookupAndAdd(const u2* dPC)
819 u4 chainEndMarker = gDvmJit.jitTableSize;
820 u4 idx = dvmJitHash(dPC);
822 /* Walk the bucket chain to find an exact match for our PC */
823 while ((gDvmJit.pJitEntryTable[idx].u.info.chain != chainEndMarker) &&
824 (gDvmJit.pJitEntryTable[idx].dPC != dPC)) {
825 idx = gDvmJit.pJitEntryTable[idx].u.info.chain;
828 if (gDvmJit.pJitEntryTable[idx].dPC != dPC) {
830 * No match. Aquire jitTableLock and find the last
831 * slot in the chain. Possibly continue the chain walk in case
832 * some other thread allocated the slot we were looking
833 * at previuosly (perhaps even the dPC we're trying to enter).
835 dvmLockMutex(&gDvmJit.tableLock);
837 * At this point, if .dPC is NULL, then the slot we're
838 * looking at is the target slot from the primary hash
839 * (the simple, and common case). Otherwise we're going
840 * to have to find a free slot and chain it.
842 MEM_BARRIER(); /* Make sure we reload [].dPC after lock */
843 if (gDvmJit.pJitEntryTable[idx].dPC != NULL) {
845 while (gDvmJit.pJitEntryTable[idx].u.info.chain != chainEndMarker) {
846 if (gDvmJit.pJitEntryTable[idx].dPC == dPC) {
847 /* Another thread got there first for this dPC */
848 dvmUnlockMutex(&gDvmJit.tableLock);
849 return &gDvmJit.pJitEntryTable[idx];
851 idx = gDvmJit.pJitEntryTable[idx].u.info.chain;
853 /* Here, idx should be pointing to the last cell of an
854 * active chain whose last member contains a valid dPC */
855 assert(gDvmJit.pJitEntryTable[idx].dPC != NULL);
856 /* Linear walk to find a free cell and add it to the end */
860 if (idx == chainEndMarker)
861 idx = 0; /* Wraparound */
862 if ((gDvmJit.pJitEntryTable[idx].dPC == NULL) ||
867 JitEntryInfoUnion oldValue;
868 JitEntryInfoUnion newValue;
870 * Although we hold the lock so that noone else will
871 * be trying to update a chain field, the other fields
872 * packed into the word may be in use by other threads.
875 oldValue = gDvmJit.pJitEntryTable[prev].u;
877 newValue.info.chain = idx;
878 } while (!ATOMIC_CMP_SWAP(
879 &gDvmJit.pJitEntryTable[prev].u.infoWord,
880 oldValue.infoWord, newValue.infoWord));
883 if (gDvmJit.pJitEntryTable[idx].dPC == NULL) {
885 * Initialize codeAddress and allocate the slot. Must
886 * happen in this order (since dPC is set, the entry is live.
888 gDvmJit.pJitEntryTable[idx].dPC = dPC;
889 gDvmJit.jitTableEntriesUsed++;
892 idx = chainEndMarker;
894 dvmUnlockMutex(&gDvmJit.tableLock);
896 return (idx == chainEndMarker) ? NULL : &gDvmJit.pJitEntryTable[idx];
899 * Register the translated code pointer into the JitTable.
900 * NOTE: Once a codeAddress field transitions from initial state to
901 * JIT'd code, it must not be altered without first halting all
902 * threads. This routine should only be called by the compiler
905 void dvmJitSetCodeAddr(const u2* dPC, void *nPC, JitInstructionSetType set) {
906 JitEntryInfoUnion oldValue;
907 JitEntryInfoUnion newValue;
908 JitEntry *jitEntry = dvmJitLookupAndAdd(dPC);
910 /* Note: order of update is important */
912 oldValue = jitEntry->u;
914 newValue.info.instructionSet = set;
915 } while (!ATOMIC_CMP_SWAP(
916 &jitEntry->u.infoWord,
917 oldValue.infoWord, newValue.infoWord));
918 jitEntry->codeAddress = nPC;
922 * Determine if valid trace-bulding request is active. Return true
923 * if we need to abort and switch back to the fast interpreter, false
924 * otherwise. NOTE: may be called even when trace selection is not being
927 bool dvmJitCheckTraceRequest(Thread* self, InterpState* interpState)
929 bool res = false; /* Assume success */
932 * If previous trace-building attempt failed, force it's head to be
935 if (gDvmJit.pJitEntryTable != NULL) {
936 /* Two-level filtering scheme */
937 for (i=0; i< JIT_TRACE_THRESH_FILTER_SIZE; i++) {
938 if (interpState->pc == interpState->threshFilter[i]) {
942 if (i == JIT_TRACE_THRESH_FILTER_SIZE) {
944 * Use random replacement policy - otherwise we could miss a large
945 * loop that contains more traces than the size of our filter array.
947 i = rand() % JIT_TRACE_THRESH_FILTER_SIZE;
948 interpState->threshFilter[i] = interpState->pc;
952 /* If stress mode (threshold <= 6), always translate */
953 res &= (gDvmJit.threshold > 6);
956 * If the compiler is backlogged, or if a debugger or profiler is
957 * active, cancel any JIT actions
959 if (res || (gDvmJit.compilerQueueLength >= gDvmJit.compilerHighWater)
960 || gDvm.debuggerActive || self->suspendCount
961 #if defined(WITH_PROFILER)
962 || gDvm.activeProfilers
965 if (interpState->jitState != kJitOff) {
966 interpState->jitState = kJitNormal;
968 } else if (interpState->jitState == kJitTSelectRequest) {
969 JitEntry *slot = dvmJitLookupAndAdd(interpState->pc);
972 * Table is full. This should have been
973 * detected by the compiler thread and the table
974 * resized before we run into it here. Assume bad things
975 * are afoot and disable profiling.
977 interpState->jitState = kJitTSelectAbort;
978 LOGD("JIT: JitTable full, disabling profiling");
979 dvmJitStopTranslationRequests();
980 } else if (slot->u.info.traceConstruction) {
982 * Trace request already in progress, but most likely it
983 * aborted without cleaning up. Assume the worst and
984 * mark trace head as untranslatable. If we're wrong,
985 * the compiler thread will correct the entry when the
986 * translation is completed. The downside here is that
987 * some existing translation may chain to the interpret-only
988 * template instead of the real translation during this
989 * window. Performance, but not correctness, issue.
991 interpState->jitState = kJitTSelectAbort;
992 resetTracehead(interpState, slot);
993 } else if (slot->codeAddress) {
994 /* Nothing to do here - just return */
995 interpState->jitState = kJitTSelectAbort;
998 * Mark request. Note, we are not guaranteed exclusivity
999 * here. A window exists for another thread to be
1000 * attempting to build this same trace. Rather than
1001 * bear the cost of locking, we'll just allow that to
1002 * happen. The compiler thread, if it chooses, can
1003 * discard redundant requests.
1005 setTraceConstruction(slot, true);
1008 switch (interpState->jitState) {
1009 case kJitTSelectRequest:
1010 interpState->jitState = kJitTSelect;
1011 interpState->currTraceHead = interpState->pc;
1012 interpState->currTraceRun = 0;
1013 interpState->totalTraceLen = 0;
1014 interpState->currRunHead = interpState->pc;
1015 interpState->currRunLen = 0;
1016 interpState->trace[0].frag.startOffset =
1017 interpState->pc - interpState->method->insns;
1018 interpState->trace[0].frag.numInsts = 0;
1019 interpState->trace[0].frag.runEnd = false;
1020 interpState->trace[0].frag.hint = kJitHintNone;
1021 interpState->lastPC = 0;
1024 case kJitTSelectAbort:
1026 case kJitSingleStep:
1027 case kJitSingleStepEnd:
1030 #if defined(WITH_SELF_VERIFICATION)
1031 case kJitSelfVerification:
1035 LOGE("Unexpected JIT state: %d", interpState->jitState);
1043 * Resizes the JitTable. Must be a power of 2, and returns true on failure.
1044 * Stops all threads, and thus is a heavyweight operation.
1046 bool dvmJitResizeJitTable( unsigned int size )
1048 JitEntry *pNewTable;
1049 JitEntry *pOldTable;
1051 unsigned int oldSize;
1054 assert(gDvmJit.pJitEntryTable != NULL);
1055 assert(size && !(size & (size - 1))); /* Is power of 2? */
1057 LOGD("Jit: resizing JitTable from %d to %d", gDvmJit.jitTableSize, size);
1061 if (size <= gDvmJit.jitTableSize) {
1065 pNewTable = (JitEntry*)calloc(size, sizeof(*pNewTable));
1066 if (pNewTable == NULL) {
1069 for (i=0; i< size; i++) {
1070 pNewTable[i].u.info.chain = size; /* Initialize chain termination */
1073 /* Stop all other interpreting/jit'ng threads */
1074 dvmSuspendAllThreads(SUSPEND_FOR_TBL_RESIZE);
1076 pOldTable = gDvmJit.pJitEntryTable;
1077 oldSize = gDvmJit.jitTableSize;
1079 dvmLockMutex(&gDvmJit.tableLock);
1080 gDvmJit.pJitEntryTable = pNewTable;
1081 gDvmJit.jitTableSize = size;
1082 gDvmJit.jitTableMask = size - 1;
1083 gDvmJit.jitTableEntriesUsed = 0;
1084 dvmUnlockMutex(&gDvmJit.tableLock);
1086 for (i=0; i < oldSize; i++) {
1087 if (pOldTable[i].dPC) {
1090 p = dvmJitLookupAndAdd(pOldTable[i].dPC);
1091 p->dPC = pOldTable[i].dPC;
1093 * Compiler thread may have just updated the new entry's
1094 * code address field, so don't blindly copy null.
1096 if (pOldTable[i].codeAddress != NULL) {
1097 p->codeAddress = pOldTable[i].codeAddress;
1099 /* We need to preserve the new chain field, but copy the rest */
1100 dvmLockMutex(&gDvmJit.tableLock);
1101 chain = p->u.info.chain;
1102 p->u = pOldTable[i].u;
1103 p->u.info.chain = chain;
1104 dvmUnlockMutex(&gDvmJit.tableLock);
1110 /* Restart the world */
1111 dvmResumeAllThreads(SUSPEND_FOR_TBL_RESIZE);
1117 * Reset the JitTable to the initial clean state.
1119 void dvmJitResetTable(void)
1121 JitEntry *jitEntry = gDvmJit.pJitEntryTable;
1122 unsigned int size = gDvmJit.jitTableSize;
1125 dvmLockMutex(&gDvmJit.tableLock);
1126 memset((void *) jitEntry, 0, sizeof(JitEntry) * size);
1127 for (i=0; i< size; i++) {
1128 jitEntry[i].u.info.chain = size; /* Initialize chain termination */
1130 gDvmJit.jitTableEntriesUsed = 0;
1131 dvmUnlockMutex(&gDvmJit.tableLock);
1135 * Float/double conversion requires clamping to min and max of integer form. If
1136 * target doesn't support this normally, use these.
1138 s8 dvmJitd2l(double d)
1140 static const double kMaxLong = (double)(s8)0x7fffffffffffffffULL;
1141 static const double kMinLong = (double)(s8)0x8000000000000000ULL;
1143 return (s8)0x7fffffffffffffffULL;
1144 else if (d <= kMinLong)
1145 return (s8)0x8000000000000000ULL;
1146 else if (d != d) // NaN case
1152 s8 dvmJitf2l(float f)
1154 static const float kMaxLong = (float)(s8)0x7fffffffffffffffULL;
1155 static const float kMinLong = (float)(s8)0x8000000000000000ULL;
1157 return (s8)0x7fffffffffffffffULL;
1158 else if (f <= kMinLong)
1159 return (s8)0x8000000000000000ULL;
1160 else if (f != f) // NaN case
1167 #endif /* WITH_JIT */