src/Reactor/LLVMReactor.cpp

   1 // Copyright 2016 The SwiftShader Authors. All Rights Reserved.
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License");
   4 // you may not use this file except in compliance with the License.
   5 // You may obtain a copy of the License at
   6 //
   7 //    http://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS,
  11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 // See the License for the specific language governing permissions and
  13 // limitations under the License.
  14
  15 #include "Nucleus.hpp"
  16
  17 #include "llvm/Support/IRBuilder.h"
  18 #include "llvm/Function.h"
  19 #include "llvm/GlobalVariable.h"
  20 #include "llvm/Module.h"
  21 #include "llvm/LLVMContext.h"
  22 #include "llvm/Constants.h"
  23 #include "llvm/Intrinsics.h"
  24 #include "llvm/PassManager.h"
  25 #include "llvm/Analysis/LoopPass.h"
  26 #include "llvm/Transforms/Scalar.h"
  27 #include "llvm/Target/TargetData.h"
  28 #include "llvm/Target/TargetOptions.h"
  29 #include "llvm/Support/TargetSelect.h"
  30 #include "../lib/ExecutionEngine/JIT/JIT.h"
  31
  32 #include "LLVMRoutine.hpp"
  33 #include "LLVMRoutineManager.hpp"
  34 #include "x86.hpp"
  35 #include "CPUID.hpp"
  36 #include "Thread.hpp"
  37 #include "Memory.hpp"
  38 #include "MutexLock.hpp"
  39
  40 #include <xmmintrin.h>
  41 #include <fstream>
  42
  43 #if defined(__x86_64__) && defined(_WIN32)
  44 extern "C" void X86CompilationCallback()
  45 {
  46         assert(false);   // UNIMPLEMENTED
  47 }
  48 #endif
  49
  50 extern "C"
  51 {
  52         bool (*CodeAnalystInitialize)() = 0;
  53         void (*CodeAnalystCompleteJITLog)() = 0;
  54         bool (*CodeAnalystLogJITCode)(const void *jitCodeStartAddr, unsigned int jitCodeSize, const wchar_t *functionName) = 0;
  55 }
  56
  57 namespace llvm
  58 {
  59         extern bool JITEmitDebugInfo;
  60 }
  61
  62 namespace
  63 {
  64         sw::LLVMRoutineManager *routineManager = nullptr;
  65         llvm::ExecutionEngine *executionEngine = nullptr;
  66         llvm::IRBuilder<> *builder = nullptr;
  67         llvm::LLVMContext *context = nullptr;
  68         llvm::Module *module = nullptr;
  69         llvm::Function *function = nullptr;
  70
  71         sw::BackoffLock codegenMutex;
  72 }
  73
  74 namespace sw
  75 {
  76         using namespace llvm;
  77
  78         Optimization optimization[10] = {InstructionCombining, Disabled};
  79
  80         class Type : public llvm::Type {};
  81         class Value : public llvm::Value {};
  82         class Constant : public llvm::Constant {};
  83         class BasicBlock : public llvm::BasicBlock {};
  84
  85         inline Type *T(llvm::Type *t)
  86         {
  87                 return reinterpret_cast<Type*>(t);
  88         }
  89
  90         inline Value *V(llvm::Value *t)
  91         {
  92                 return reinterpret_cast<Value*>(t);
  93         }
  94
  95         inline std::vector<llvm::Type*> &T(std::vector<Type*> &t)
  96         {
  97                 return reinterpret_cast<std::vector<llvm::Type*>&>(t);
  98         }
  99
 100         inline Constant *C(llvm::Constant *c)
 101         {
 102                 return reinterpret_cast<Constant*>(c);
 103         }
 104
 105         inline BasicBlock *B(llvm::BasicBlock *t)
 106         {
 107                 return reinterpret_cast<BasicBlock*>(t);
 108         }
 109
 110         Nucleus::Nucleus()
 111         {
 112                 ::codegenMutex.lock();   // Reactor and LLVM are currently not thread safe
 113
 114                 InitializeNativeTarget();
 115                 JITEmitDebugInfo = false;
 116
 117                 if(!::context)
 118                 {
 119                         ::context = new LLVMContext();
 120                 }
 121
 122                 ::module = new Module("", *::context);
 123                 ::routineManager = new LLVMRoutineManager();
 124
 125                 #if defined(__x86_64__)
 126                         const char *architecture = "x86-64";
 127                 #else
 128                         const char *architecture = "x86";
 129                 #endif
 130
 131                 SmallVector<std::string, 1> MAttrs;
 132                 MAttrs.push_back(CPUID::supportsMMX()    ? "+mmx"   : "-mmx");
 133                 MAttrs.push_back(CPUID::supportsCMOV()   ? "+cmov"  : "-cmov");
 134                 MAttrs.push_back(CPUID::supportsSSE()    ? "+sse"   : "-sse");
 135                 MAttrs.push_back(CPUID::supportsSSE2()   ? "+sse2"  : "-sse2");
 136                 MAttrs.push_back(CPUID::supportsSSE3()   ? "+sse3"  : "-sse3");
 137                 MAttrs.push_back(CPUID::supportsSSSE3()  ? "+ssse3" : "-ssse3");
 138                 MAttrs.push_back(CPUID::supportsSSE4_1() ? "+sse41" : "-sse41");
 139
 140                 std::string error;
 141                 TargetMachine *targetMachine = EngineBuilder::selectTarget(::module, architecture, "", MAttrs, Reloc::Default, CodeModel::JITDefault, &error);
 142                 ::executionEngine = JIT::createJIT(::module, 0, ::routineManager, CodeGenOpt::Aggressive, true, targetMachine);
 143
 144                 if(!::builder)
 145                 {
 146                         ::builder = new IRBuilder<>(*::context);
 147
 148                         #if defined(_WIN32)
 149                                 HMODULE CodeAnalyst = LoadLibrary("CAJitNtfyLib.dll");
 150                                 if(CodeAnalyst)
 151                                 {
 152                                         CodeAnalystInitialize = (bool(*)())GetProcAddress(CodeAnalyst, "CAJIT_Initialize");
 153                                         CodeAnalystCompleteJITLog = (void(*)())GetProcAddress(CodeAnalyst, "CAJIT_CompleteJITLog");
 154                                         CodeAnalystLogJITCode = (bool(*)(const void*, unsigned int, const wchar_t*))GetProcAddress(CodeAnalyst, "CAJIT_LogJITCode");
 155
 156                                         CodeAnalystInitialize();
 157                                 }
 158                         #endif
 159                 }
 160         }
 161
 162         Nucleus::~Nucleus()
 163         {
 164                 delete ::executionEngine;
 165                 ::executionEngine = nullptr;
 166
 167                 ::routineManager = nullptr;
 168                 ::function = nullptr;
 169                 ::module = nullptr;
 170
 171                 ::codegenMutex.unlock();
 172         }
 173
 174         Routine *Nucleus::acquireRoutine(const wchar_t *name, bool runOptimizations)
 175         {
 176                 if(::builder->GetInsertBlock()->empty() || !::builder->GetInsertBlock()->back().isTerminator())
 177                 {
 178                         llvm::Type *type = ::function->getReturnType();
 179
 180                         if(type->isVoidTy())
 181                         {
 182                                 createRetVoid();
 183                         }
 184                         else
 185                         {
 186                                 createRet(V(UndefValue::get(type)));
 187                         }
 188                 }
 189
 190                 if(false)
 191                 {
 192                         std::string error;
 193                         raw_fd_ostream file("llvm-dump-unopt.txt", error);
 194                         ::module->print(file, 0);
 195                 }
 196
 197                 if(runOptimizations)
 198                 {
 199                         optimize();
 200                 }
 201
 202                 if(false)
 203                 {
 204                         std::string error;
 205                         raw_fd_ostream file("llvm-dump-opt.txt", error);
 206                         ::module->print(file, 0);
 207                 }
 208
 209                 void *entry = ::executionEngine->getPointerToFunction(::function);
 210                 LLVMRoutine *routine = ::routineManager->acquireRoutine(entry);
 211
 212                 if(CodeAnalystLogJITCode)
 213                 {
 214                         CodeAnalystLogJITCode(routine->getEntry(), routine->getCodeSize(), name);
 215                 }
 216
 217                 return routine;
 218         }
 219
 220         void Nucleus::optimize()
 221         {
 222                 static PassManager *passManager = nullptr;
 223
 224                 if(!passManager)
 225                 {
 226                         passManager = new PassManager();
 227
 228                         UnsafeFPMath = true;
 229                 //      NoInfsFPMath = true;
 230                 //      NoNaNsFPMath = true;
 231
 232                         passManager->add(new TargetData(*::executionEngine->getTargetData()));
 233                         passManager->add(createScalarReplAggregatesPass());
 234
 235                         for(int pass = 0; pass < 10 && optimization[pass] != Disabled; pass++)
 236                         {
 237                                 switch(optimization[pass])
 238                                 {
 239                                 case Disabled:                                                                 break;
 240                                 case CFGSimplification:    passManager->add(createCFGSimplificationPass());    break;
 241                                 case LICM:                 passManager->add(createLICMPass());                 break;
 242                                 case AggressiveDCE:        passManager->add(createAggressiveDCEPass());        break;
 243                                 case GVN:                  passManager->add(createGVNPass());                  break;
 244                                 case InstructionCombining: passManager->add(createInstructionCombiningPass()); break;
 245                                 case Reassociate:          passManager->add(createReassociatePass());          break;
 246                                 case DeadStoreElimination: passManager->add(createDeadStoreEliminationPass()); break;
 247                                 case SCCP:                 passManager->add(createSCCPPass());                 break;
 248                                 case ScalarReplAggregates: passManager->add(createScalarReplAggregatesPass()); break;
 249                                 default:
 250                                         assert(false);
 251                                 }
 252                         }
 253                 }
 254
 255                 passManager->run(*::module);
 256         }
 257
 258         Value *Nucleus::allocateStackVariable(Type *type, int arraySize)
 259         {
 260                 // Need to allocate it in the entry block for mem2reg to work
 261                 llvm::BasicBlock &entryBlock = ::function->getEntryBlock();
 262
 263                 Instruction *declaration;
 264
 265                 if(arraySize)
 266                 {
 267                         declaration = new AllocaInst(type, Nucleus::createConstantInt(arraySize));
 268                 }
 269                 else
 270                 {
 271                         declaration = new AllocaInst(type, (Value*)0);
 272                 }
 273
 274                 entryBlock.getInstList().push_front(declaration);
 275
 276                 return V(declaration);
 277         }
 278
 279         BasicBlock *Nucleus::createBasicBlock()
 280         {
 281                 return B(BasicBlock::Create(*::context, "", ::function));
 282         }
 283
 284         BasicBlock *Nucleus::getInsertBlock()
 285         {
 286                 return B(::builder->GetInsertBlock());
 287         }
 288
 289         void Nucleus::setInsertBlock(BasicBlock *basicBlock)
 290         {
 291         //      assert(::builder->GetInsertBlock()->back().isTerminator());
 292                 return ::builder->SetInsertPoint(basicBlock);
 293         }
 294
 295         BasicBlock *Nucleus::getPredecessor(BasicBlock *basicBlock)
 296         {
 297                 return B(*pred_begin(basicBlock));
 298         }
 299
 300         void Nucleus::createFunction(Type *ReturnType, std::vector<Type*> &Params)
 301         {
 302                 llvm::FunctionType *functionType = llvm::FunctionType::get(ReturnType, T(Params), false);
 303                 ::function = llvm::Function::Create(functionType, llvm::GlobalValue::InternalLinkage, "", ::module);
 304                 ::function->setCallingConv(llvm::CallingConv::C);
 305
 306                 ::builder->SetInsertPoint(BasicBlock::Create(*::context, "", ::function));
 307         }
 308
 309         Value *Nucleus::getArgument(unsigned int index)
 310         {
 311                 llvm::Function::arg_iterator args = ::function->arg_begin();
 312
 313                 while(index)
 314                 {
 315                         args++;
 316                         index--;
 317                 }
 318
 319                 return V(&*args);
 320         }
 321
 322         void Nucleus::createRetVoid()
 323         {
 324                 x86::emms();
 325
 326                 ::builder->CreateRetVoid();
 327         }
 328
 329         void Nucleus::createRet(Value *v)
 330         {
 331                 x86::emms();
 332
 333                 ::builder->CreateRet(v);
 334         }
 335
 336         void Nucleus::createBr(BasicBlock *dest)
 337         {
 338                 ::builder->CreateBr(dest);
 339         }
 340
 341         void Nucleus::createCondBr(Value *cond, BasicBlock *ifTrue, BasicBlock *ifFalse)
 342         {
 343                 ::builder->CreateCondBr(cond, ifTrue, ifFalse);
 344         }
 345
 346         Value *Nucleus::createAdd(Value *lhs, Value *rhs)
 347         {
 348                 return V(::builder->CreateAdd(lhs, rhs));
 349         }
 350
 351         Value *Nucleus::createSub(Value *lhs, Value *rhs)
 352         {
 353                 return V(::builder->CreateSub(lhs, rhs));
 354         }
 355
 356         Value *Nucleus::createMul(Value *lhs, Value *rhs)
 357         {
 358                 return V(::builder->CreateMul(lhs, rhs));
 359         }
 360
 361         Value *Nucleus::createUDiv(Value *lhs, Value *rhs)
 362         {
 363                 return V(::builder->CreateUDiv(lhs, rhs));
 364         }
 365
 366         Value *Nucleus::createSDiv(Value *lhs, Value *rhs)
 367         {
 368                 return V(::builder->CreateSDiv(lhs, rhs));
 369         }
 370
 371         Value *Nucleus::createFAdd(Value *lhs, Value *rhs)
 372         {
 373                 return V(::builder->CreateFAdd(lhs, rhs));
 374         }
 375
 376         Value *Nucleus::createFSub(Value *lhs, Value *rhs)
 377         {
 378                 return V(::builder->CreateFSub(lhs, rhs));
 379         }
 380
 381         Value *Nucleus::createFMul(Value *lhs, Value *rhs)
 382         {
 383                 return V(::builder->CreateFMul(lhs, rhs));
 384         }
 385
 386         Value *Nucleus::createFDiv(Value *lhs, Value *rhs)
 387         {
 388                 return V(::builder->CreateFDiv(lhs, rhs));
 389         }
 390
 391         Value *Nucleus::createURem(Value *lhs, Value *rhs)
 392         {
 393                 return V(::builder->CreateURem(lhs, rhs));
 394         }
 395
 396         Value *Nucleus::createSRem(Value *lhs, Value *rhs)
 397         {
 398                 return V(::builder->CreateSRem(lhs, rhs));
 399         }
 400
 401         Value *Nucleus::createFRem(Value *lhs, Value *rhs)
 402         {
 403                 return V(::builder->CreateFRem(lhs, rhs));
 404         }
 405
 406         Value *Nucleus::createShl(Value *lhs, Value *rhs)
 407         {
 408                 return V(::builder->CreateShl(lhs, rhs));
 409         }
 410
 411         Value *Nucleus::createLShr(Value *lhs, Value *rhs)
 412         {
 413                 return V(::builder->CreateLShr(lhs, rhs));
 414         }
 415
 416         Value *Nucleus::createAShr(Value *lhs, Value *rhs)
 417         {
 418                 return V(::builder->CreateAShr(lhs, rhs));
 419         }
 420
 421         Value *Nucleus::createAnd(Value *lhs, Value *rhs)
 422         {
 423                 return V(::builder->CreateAnd(lhs, rhs));
 424         }
 425
 426         Value *Nucleus::createOr(Value *lhs, Value *rhs)
 427         {
 428                 return V(::builder->CreateOr(lhs, rhs));
 429         }
 430
 431         Value *Nucleus::createXor(Value *lhs, Value *rhs)
 432         {
 433                 return V(::builder->CreateXor(lhs, rhs));
 434         }
 435
 436         Value *Nucleus::createAssign(Constant *constant)
 437         {
 438                 return V(constant);
 439         }
 440
 441         Value *Nucleus::createNeg(Value *v)
 442         {
 443                 return V(::builder->CreateNeg(v));
 444         }
 445
 446         Value *Nucleus::createFNeg(Value *v)
 447         {
 448                 return V(::builder->CreateFNeg(v));
 449         }
 450
 451         Value *Nucleus::createNot(Value *v)
 452         {
 453                 return V(::builder->CreateNot(v));
 454         }
 455
 456         Value *Nucleus::createLoad(Value *ptr, Type *type, bool isVolatile, unsigned int align)
 457         {
 458                 assert(ptr->getType()->getContainedType(0) == type);
 459                 return V(::builder->Insert(new LoadInst(ptr, "", isVolatile, align)));
 460         }
 461
 462         Value *Nucleus::createStore(Value *value, Value *ptr, Type *type, bool isVolatile, unsigned int align)
 463         {
 464                 assert(ptr->getType()->getContainedType(0) == type);
 465                 ::builder->Insert(new StoreInst(value, ptr, isVolatile, align));
 466                 return value;
 467         }
 468
 469         Constant *Nucleus::createStore(Constant *constant, Value *ptr, Type *type, bool isVolatile, unsigned int align)
 470         {
 471                 assert(ptr->getType()->getContainedType(0) == type);
 472                 ::builder->Insert(new StoreInst(constant, ptr, isVolatile, align));
 473                 return constant;
 474         }
 475
 476         Value *Nucleus::createGEP(Value *ptr, Type *type, Value *index)
 477         {
 478                 assert(ptr->getType()->getContainedType(0) == type);
 479                 return V(::builder->CreateGEP(ptr, index));
 480         }
 481
 482         Value *Nucleus::createAtomicAdd(Value *ptr, Value *value)
 483         {
 484                 return V(::builder->CreateAtomicRMW(AtomicRMWInst::Add, ptr, value, SequentiallyConsistent));
 485         }
 486
 487         Value *Nucleus::createTrunc(Value *v, Type *destType)
 488         {
 489                 return V(::builder->CreateTrunc(v, destType));
 490         }
 491
 492         Value *Nucleus::createZExt(Value *v, Type *destType)
 493         {
 494                 return V(::builder->CreateZExt(v, destType));
 495         }
 496
 497         Value *Nucleus::createSExt(Value *v, Type *destType)
 498         {
 499                 return V(::builder->CreateSExt(v, destType));
 500         }
 501
 502         Value *Nucleus::createFPToSI(Value *v, Type *destType)
 503         {
 504                 return V(::builder->CreateFPToSI(v, destType));
 505         }
 506
 507         Value *Nucleus::createUIToFP(Value *v, Type *destType)
 508         {
 509                 return V(::builder->CreateUIToFP(v, destType));
 510         }
 511
 512         Value *Nucleus::createSIToFP(Value *v, Type *destType)
 513         {
 514                 return V(::builder->CreateSIToFP(v, destType));
 515         }
 516
 517         Value *Nucleus::createFPTrunc(Value *v, Type *destType)
 518         {
 519                 return V(::builder->CreateFPTrunc(v, destType));
 520         }
 521
 522         Value *Nucleus::createFPExt(Value *v, Type *destType)
 523         {
 524                 return V(::builder->CreateFPExt(v, destType));
 525         }
 526
 527         Value *Nucleus::createPtrToInt(Value *v, Type *destType)
 528         {
 529                 return V(::builder->CreatePtrToInt(v, destType));
 530         }
 531
 532         Value *Nucleus::createIntToPtr(Value *v, Type *destType)
 533         {
 534                 return V(::builder->CreateIntToPtr(v, destType));
 535         }
 536
 537         Value *Nucleus::createBitCast(Value *v, Type *destType)
 538         {
 539                 return V(::builder->CreateBitCast(v, destType));
 540         }
 541
 542         Value *Nucleus::createIntCast(Value *v, Type *destType, bool isSigned)
 543         {
 544                 return V(::builder->CreateIntCast(v, destType, isSigned));
 545         }
 546
 547         Value *Nucleus::createICmpEQ(Value *lhs, Value *rhs)
 548         {
 549                 return V(::builder->CreateICmpEQ(lhs, rhs));
 550         }
 551
 552         Value *Nucleus::createICmpNE(Value *lhs, Value *rhs)
 553         {
 554                 return V(::builder->CreateICmpNE(lhs, rhs));
 555         }
 556
 557         Value *Nucleus::createICmpUGT(Value *lhs, Value *rhs)
 558         {
 559                 return V(::builder->CreateICmpUGT(lhs, rhs));
 560         }
 561
 562         Value *Nucleus::createICmpUGE(Value *lhs, Value *rhs)
 563         {
 564                 return V(::builder->CreateICmpUGE(lhs, rhs));
 565         }
 566
 567         Value *Nucleus::createICmpULT(Value *lhs, Value *rhs)
 568         {
 569                 return V(::builder->CreateICmpULT(lhs, rhs));
 570         }
 571
 572         Value *Nucleus::createICmpULE(Value *lhs, Value *rhs)
 573         {
 574                 return V(::builder->CreateICmpULE(lhs, rhs));
 575         }
 576
 577         Value *Nucleus::createICmpSGT(Value *lhs, Value *rhs)
 578         {
 579                 return V(::builder->CreateICmpSGT(lhs, rhs));
 580         }
 581
 582         Value *Nucleus::createICmpSGE(Value *lhs, Value *rhs)
 583         {
 584                 return V(::builder->CreateICmpSGE(lhs, rhs));
 585         }
 586
 587         Value *Nucleus::createICmpSLT(Value *lhs, Value *rhs)
 588         {
 589                 return V(::builder->CreateICmpSLT(lhs, rhs));
 590         }
 591
 592         Value *Nucleus::createICmpSLE(Value *lhs, Value *rhs)
 593         {
 594                 return V(::builder->CreateICmpSLE(lhs, rhs));
 595         }
 596
 597         Value *Nucleus::createFCmpOEQ(Value *lhs, Value *rhs)
 598         {
 599                 return V(::builder->CreateFCmpOEQ(lhs, rhs));
 600         }
 601
 602         Value *Nucleus::createFCmpOGT(Value *lhs, Value *rhs)
 603         {
 604                 return V(::builder->CreateFCmpOGT(lhs, rhs));
 605         }
 606
 607         Value *Nucleus::createFCmpOGE(Value *lhs, Value *rhs)
 608         {
 609                 return V(::builder->CreateFCmpOGE(lhs, rhs));
 610         }
 611
 612         Value *Nucleus::createFCmpOLT(Value *lhs, Value *rhs)
 613         {
 614                 return V(::builder->CreateFCmpOLT(lhs, rhs));
 615         }
 616
 617         Value *Nucleus::createFCmpOLE(Value *lhs, Value *rhs)
 618         {
 619                 return V(::builder->CreateFCmpOLE(lhs, rhs));
 620         }
 621
 622         Value *Nucleus::createFCmpONE(Value *lhs, Value *rhs)
 623         {
 624                 return V(::builder->CreateFCmpONE(lhs, rhs));
 625         }
 626
 627         Value *Nucleus::createFCmpORD(Value *lhs, Value *rhs)
 628         {
 629                 return V(::builder->CreateFCmpORD(lhs, rhs));
 630         }
 631
 632         Value *Nucleus::createFCmpUNO(Value *lhs, Value *rhs)
 633         {
 634                 return V(::builder->CreateFCmpUNO(lhs, rhs));
 635         }
 636
 637         Value *Nucleus::createFCmpUEQ(Value *lhs, Value *rhs)
 638         {
 639                 return V(::builder->CreateFCmpUEQ(lhs, rhs));
 640         }
 641
 642         Value *Nucleus::createFCmpUGT(Value *lhs, Value *rhs)
 643         {
 644                 return V(::builder->CreateFCmpUGT(lhs, rhs));
 645         }
 646
 647         Value *Nucleus::createFCmpUGE(Value *lhs, Value *rhs)
 648         {
 649                 return V(::builder->CreateFCmpUGE(lhs, rhs));
 650         }
 651
 652         Value *Nucleus::createFCmpULT(Value *lhs, Value *rhs)
 653         {
 654                 return V(::builder->CreateFCmpULT(lhs, rhs));
 655         }
 656
 657         Value *Nucleus::createFCmpULE(Value *lhs, Value *rhs)
 658         {
 659                 return V(::builder->CreateFCmpULE(lhs, rhs));
 660         }
 661
 662         Value *Nucleus::createFCmpUNE(Value *lhs, Value *rhs)
 663         {
 664                 return V(::builder->CreateFCmpULE(lhs, rhs));
 665         }
 666
 667         Value *Nucleus::createExtractElement(Value *vector, int index)
 668         {
 669                 return V(::builder->CreateExtractElement(vector, createConstantInt(index)));
 670         }
 671
 672         Value *Nucleus::createInsertElement(Value *vector, Value *element, int index)
 673         {
 674                 return V(::builder->CreateInsertElement(vector, element, createConstantInt(index)));
 675         }
 676
 677         Value *Nucleus::createShuffleVector(Value *V1, Value *V2, Value *mask)
 678         {
 679                 return V(::builder->CreateShuffleVector(V1, V2, mask));
 680         }
 681
 682         Value *Nucleus::createSelect(Value *C, Value *ifTrue, Value *ifFalse)
 683         {
 684                 return V(::builder->CreateSelect(C, ifTrue, ifFalse));
 685         }
 686
 687         Value *Nucleus::createSwitch(Value *v, BasicBlock *Dest, unsigned NumCases)
 688         {
 689                 return V(::builder->CreateSwitch(v, Dest, NumCases));
 690         }
 691
 692         void Nucleus::addSwitchCase(Value *Switch, int Case, BasicBlock *Branch)
 693         {
 694                 reinterpret_cast<SwitchInst*>(Switch)->addCase(llvm::ConstantInt::get(Type::getInt32Ty(*::context), Case, true), Branch);
 695         }
 696
 697         void Nucleus::createUnreachable()
 698         {
 699                 ::builder->CreateUnreachable();
 700         }
 701
 702         Value *Nucleus::createSwizzle(Value *val, unsigned char select)
 703         {
 704                 Constant *swizzle[4];
 705                 swizzle[0] = Nucleus::createConstantInt((select >> 0) & 0x03);
 706                 swizzle[1] = Nucleus::createConstantInt((select >> 2) & 0x03);
 707                 swizzle[2] = Nucleus::createConstantInt((select >> 4) & 0x03);
 708                 swizzle[3] = Nucleus::createConstantInt((select >> 6) & 0x03);
 709
 710                 Value *shuffle = Nucleus::createShuffleVector(val, V(UndefValue::get(val->getType())), V(Nucleus::createConstantVector(swizzle, 4)));
 711
 712                 return shuffle;
 713         }
 714
 715         Value *Nucleus::createMask(Value *lhs, Value *rhs, unsigned char select)
 716         {
 717                 bool mask[4] = {false, false, false, false};
 718
 719                 mask[(select >> 0) & 0x03] = true;
 720                 mask[(select >> 2) & 0x03] = true;
 721                 mask[(select >> 4) & 0x03] = true;
 722                 mask[(select >> 6) & 0x03] = true;
 723
 724                 Constant *swizzle[4];
 725                 swizzle[0] = Nucleus::createConstantInt(mask[0] ? 4 : 0);
 726                 swizzle[1] = Nucleus::createConstantInt(mask[1] ? 5 : 1);
 727                 swizzle[2] = Nucleus::createConstantInt(mask[2] ? 6 : 2);
 728                 swizzle[3] = Nucleus::createConstantInt(mask[3] ? 7 : 3);
 729
 730                 Value *shuffle = Nucleus::createShuffleVector(lhs, rhs, V(Nucleus::createConstantVector(swizzle, 4)));
 731
 732                 return shuffle;
 733         }
 734
 735         Constant *Nucleus::createConstantPointer(const void *address, Type *Ty, bool isConstant, unsigned int Align)
 736         {
 737                 const GlobalValue *existingGlobal = ::executionEngine->getGlobalValueAtAddress(const_cast<void*>(address));   // FIXME: Const
 738
 739                 if(existingGlobal)
 740                 {
 741                         return (Constant*)existingGlobal;
 742                 }
 743
 744                 llvm::GlobalValue *global = new llvm::GlobalVariable(*::module, Ty, isConstant, llvm::GlobalValue::ExternalLinkage, 0, "");
 745
 746                 global->setAlignment(Align);
 747
 748                 ::executionEngine->addGlobalMapping(global, const_cast<void*>(address));
 749
 750                 return C(global);
 751         }
 752
 753         Type *Nucleus::getPointerType(Type *ElementType)
 754         {
 755                 return T(llvm::PointerType::get(ElementType, 0));
 756         }
 757
 758         Constant *Nucleus::createNullValue(Type *Ty)
 759         {
 760                 return C(llvm::Constant::getNullValue(Ty));
 761         }
 762
 763         Constant *Nucleus::createConstantInt(int64_t i)
 764         {
 765                 return C(llvm::ConstantInt::get(Type::getInt64Ty(*::context), i, true));
 766         }
 767
 768         Constant *Nucleus::createConstantInt(int i)
 769         {
 770                 return C(llvm::ConstantInt::get(Type::getInt32Ty(*::context), i, true));
 771         }
 772
 773         Constant *Nucleus::createConstantInt(unsigned int i)
 774         {
 775                 return C(llvm::ConstantInt::get(Type::getInt32Ty(*::context), i, false));
 776         }
 777
 778         Constant *Nucleus::createConstantBool(bool b)
 779         {
 780                 return C(llvm::ConstantInt::get(Type::getInt1Ty(*::context), b));
 781         }
 782
 783         Constant *Nucleus::createConstantByte(signed char i)
 784         {
 785                 return C(llvm::ConstantInt::get(Type::getInt8Ty(*::context), i, true));
 786         }
 787
 788         Constant *Nucleus::createConstantByte(unsigned char i)
 789         {
 790                 return C(llvm::ConstantInt::get(Type::getInt8Ty(*::context), i, false));
 791         }
 792
 793         Constant *Nucleus::createConstantShort(short i)
 794         {
 795                 return C(llvm::ConstantInt::get(Type::getInt16Ty(*::context), i, true));
 796         }
 797
 798         Constant *Nucleus::createConstantShort(unsigned short i)
 799         {
 800                 return C(llvm::ConstantInt::get(Type::getInt16Ty(*::context), i, false));
 801         }
 802
 803         Constant *Nucleus::createConstantFloat(float x)
 804         {
 805                 return C(ConstantFP::get(Float::getType(), x));
 806         }
 807
 808         Constant *Nucleus::createNullPointer(Type *Ty)
 809         {
 810                 return C(llvm::ConstantPointerNull::get(llvm::PointerType::get(Ty, 0)));
 811         }
 812
 813         Constant *Nucleus::createConstantVector(Constant *const *Vals, unsigned NumVals)
 814         {
 815                 return C(llvm::ConstantVector::get(llvm::ArrayRef<llvm::Constant*>(reinterpret_cast<llvm::Constant *const*>(Vals), NumVals)));
 816         }
 817
 818         Type *Void::getType()
 819         {
 820                 return T(llvm::Type::getVoidTy(*::context));
 821         }
 822
 823         class MMX : public Variable<MMX>
 824         {
 825         public:
 826                 static Type *getType();
 827         };
 828
 829         Type *MMX::getType()
 830         {
 831                 return T(llvm::Type::getX86_MMXTy(*::context));
 832         }
 833
 834         Bool::Bool(Argument<Bool> argument)
 835         {
 836                 storeValue(argument.value);
 837         }
 838
 839         Bool::Bool()
 840         {
 841         }
 842
 843         Bool::Bool(bool x)
 844         {
 845                 storeValue(Nucleus::createConstantBool(x));
 846         }
 847
 848         Bool::Bool(RValue<Bool> rhs)
 849         {
 850                 storeValue(rhs.value);
 851         }
 852
 853         Bool::Bool(const Bool &rhs)
 854         {
 855                 Value *value = rhs.loadValue();
 856                 storeValue(value);
 857         }
 858
 859         Bool::Bool(const Reference<Bool> &rhs)
 860         {
 861                 Value *value = rhs.loadValue();
 862                 storeValue(value);
 863         }
 864
 865         RValue<Bool> Bool::operator=(RValue<Bool> rhs) const
 866         {
 867                 storeValue(rhs.value);
 868
 869                 return rhs;
 870         }
 871
 872         RValue<Bool> Bool::operator=(const Bool &rhs) const
 873         {
 874                 Value *value = rhs.loadValue();
 875                 storeValue(value);
 876
 877                 return RValue<Bool>(value);
 878         }
 879
 880         RValue<Bool> Bool::operator=(const Reference<Bool> &rhs) const
 881         {
 882                 Value *value = rhs.loadValue();
 883                 storeValue(value);
 884
 885                 return RValue<Bool>(value);
 886         }
 887
 888         RValue<Bool> operator!(RValue<Bool> val)
 889         {
 890                 return RValue<Bool>(Nucleus::createNot(val.value));
 891         }
 892
 893         RValue<Bool> operator&&(RValue<Bool> lhs, RValue<Bool> rhs)
 894         {
 895                 return RValue<Bool>(Nucleus::createAnd(lhs.value, rhs.value));
 896         }
 897
 898         RValue<Bool> operator||(RValue<Bool> lhs, RValue<Bool> rhs)
 899         {
 900                 return RValue<Bool>(Nucleus::createOr(lhs.value, rhs.value));
 901         }
 902
 903         Type *Bool::getType()
 904         {
 905                 return T(llvm::Type::getInt1Ty(*::context));
 906         }
 907
 908         Byte::Byte(Argument<Byte> argument)
 909         {
 910                 storeValue(argument.value);
 911         }
 912
 913         Byte::Byte(RValue<Int> cast)
 914         {
 915                 Value *integer = Nucleus::createTrunc(cast.value, Byte::getType());
 916
 917                 storeValue(integer);
 918         }
 919
 920         Byte::Byte(RValue<UInt> cast)
 921         {
 922                 Value *integer = Nucleus::createTrunc(cast.value, Byte::getType());
 923
 924                 storeValue(integer);
 925         }
 926
 927         Byte::Byte(RValue<UShort> cast)
 928         {
 929                 Value *integer = Nucleus::createTrunc(cast.value, Byte::getType());
 930
 931                 storeValue(integer);
 932         }
 933
 934         Byte::Byte()
 935         {
 936         }
 937
 938         Byte::Byte(int x)
 939         {
 940                 storeValue(Nucleus::createConstantByte((unsigned char)x));
 941         }
 942
 943         Byte::Byte(unsigned char x)
 944         {
 945                 storeValue(Nucleus::createConstantByte(x));
 946         }
 947
 948         Byte::Byte(RValue<Byte> rhs)
 949         {
 950                 storeValue(rhs.value);
 951         }
 952
 953         Byte::Byte(const Byte &rhs)
 954         {
 955                 Value *value = rhs.loadValue();
 956                 storeValue(value);
 957         }
 958
 959         Byte::Byte(const Reference<Byte> &rhs)
 960         {
 961                 Value *value = rhs.loadValue();
 962                 storeValue(value);
 963         }
 964
 965         RValue<Byte> Byte::operator=(RValue<Byte> rhs) const
 966         {
 967                 storeValue(rhs.value);
 968
 969                 return rhs;
 970         }
 971
 972         RValue<Byte> Byte::operator=(const Byte &rhs) const
 973         {
 974                 Value *value = rhs.loadValue();
 975                 storeValue(value);
 976
 977                 return RValue<Byte>(value);
 978         }
 979
 980         RValue<Byte> Byte::operator=(const Reference<Byte> &rhs) const
 981         {
 982                 Value *value = rhs.loadValue();
 983                 storeValue(value);
 984
 985                 return RValue<Byte>(value);
 986         }
 987
 988         RValue<Byte> operator+(RValue<Byte> lhs, RValue<Byte> rhs)
 989         {
 990                 return RValue<Byte>(Nucleus::createAdd(lhs.value, rhs.value));
 991         }
 992
 993         RValue<Byte> operator-(RValue<Byte> lhs, RValue<Byte> rhs)
 994         {
 995                 return RValue<Byte>(Nucleus::createSub(lhs.value, rhs.value));
 996         }
 997
 998         RValue<Byte> operator*(RValue<Byte> lhs, RValue<Byte> rhs)
 999         {
1000                 return RValue<Byte>(Nucleus::createMul(lhs.value, rhs.value));
1001         }
1002
1003         RValue<Byte> operator/(RValue<Byte> lhs, RValue<Byte> rhs)
1004         {
1005                 return RValue<Byte>(Nucleus::createUDiv(lhs.value, rhs.value));
1006         }
1007
1008         RValue<Byte> operator%(RValue<Byte> lhs, RValue<Byte> rhs)
1009         {
1010                 return RValue<Byte>(Nucleus::createURem(lhs.value, rhs.value));
1011         }
1012
1013         RValue<Byte> operator&(RValue<Byte> lhs, RValue<Byte> rhs)
1014         {
1015                 return RValue<Byte>(Nucleus::createAnd(lhs.value, rhs.value));
1016         }
1017
1018         RValue<Byte> operator|(RValue<Byte> lhs, RValue<Byte> rhs)
1019         {
1020                 return RValue<Byte>(Nucleus::createOr(lhs.value, rhs.value));
1021         }
1022
1023         RValue<Byte> operator^(RValue<Byte> lhs, RValue<Byte> rhs)
1024         {
1025                 return RValue<Byte>(Nucleus::createXor(lhs.value, rhs.value));
1026         }
1027
1028         RValue<Byte> operator<<(RValue<Byte> lhs, RValue<Byte> rhs)
1029         {
1030                 return RValue<Byte>(Nucleus::createShl(lhs.value, rhs.value));
1031         }
1032
1033         RValue<Byte> operator>>(RValue<Byte> lhs, RValue<Byte> rhs)
1034         {
1035                 return RValue<Byte>(Nucleus::createLShr(lhs.value, rhs.value));
1036         }
1037
1038         RValue<Byte> operator+=(const Byte &lhs, RValue<Byte> rhs)
1039         {
1040                 return lhs = lhs + rhs;
1041         }
1042
1043         RValue<Byte> operator-=(const Byte &lhs, RValue<Byte> rhs)
1044         {
1045                 return lhs = lhs - rhs;
1046         }
1047
1048         RValue<Byte> operator*=(const Byte &lhs, RValue<Byte> rhs)
1049         {
1050                 return lhs = lhs * rhs;
1051         }
1052
1053         RValue<Byte> operator/=(const Byte &lhs, RValue<Byte> rhs)
1054         {
1055                 return lhs = lhs / rhs;
1056         }
1057
1058         RValue<Byte> operator%=(const Byte &lhs, RValue<Byte> rhs)
1059         {
1060                 return lhs = lhs % rhs;
1061         }
1062
1063         RValue<Byte> operator&=(const Byte &lhs, RValue<Byte> rhs)
1064         {
1065                 return lhs = lhs & rhs;
1066         }
1067
1068         RValue<Byte> operator|=(const Byte &lhs, RValue<Byte> rhs)
1069         {
1070                 return lhs = lhs | rhs;
1071         }
1072
1073         RValue<Byte> operator^=(const Byte &lhs, RValue<Byte> rhs)
1074         {
1075                 return lhs = lhs ^ rhs;
1076         }
1077
1078         RValue<Byte> operator<<=(const Byte &lhs, RValue<Byte> rhs)
1079         {
1080                 return lhs = lhs << rhs;
1081         }
1082
1083         RValue<Byte> operator>>=(const Byte &lhs, RValue<Byte> rhs)
1084         {
1085                 return lhs = lhs >> rhs;
1086         }
1087
1088         RValue<Byte> operator+(RValue<Byte> val)
1089         {
1090                 return val;
1091         }
1092
1093         RValue<Byte> operator-(RValue<Byte> val)
1094         {
1095                 return RValue<Byte>(Nucleus::createNeg(val.value));
1096         }
1097
1098         RValue<Byte> operator~(RValue<Byte> val)
1099         {
1100                 return RValue<Byte>(Nucleus::createNot(val.value));
1101         }
1102
1103         RValue<Byte> operator++(const Byte &val, int)   // Post-increment
1104         {
1105                 RValue<Byte> res = val;
1106
1107                 Value *inc = Nucleus::createAdd(res.value, V(Nucleus::createConstantByte((unsigned char)1)));
1108                 val.storeValue(inc);
1109
1110                 return res;
1111         }
1112
1113         const Byte &operator++(const Byte &val)   // Pre-increment
1114         {
1115                 Value *inc = Nucleus::createAdd(val.loadValue(), V(Nucleus::createConstantByte((unsigned char)1)));
1116                 val.storeValue(inc);
1117
1118                 return val;
1119         }
1120
1121         RValue<Byte> operator--(const Byte &val, int)   // Post-decrement
1122         {
1123                 RValue<Byte> res = val;
1124
1125                 Value *inc = Nucleus::createSub(res.value, V(Nucleus::createConstantByte((unsigned char)1)));
1126                 val.storeValue(inc);
1127
1128                 return res;
1129         }
1130
1131         const Byte &operator--(const Byte &val)   // Pre-decrement
1132         {
1133                 Value *inc = Nucleus::createSub(val.loadValue(), V(Nucleus::createConstantByte((unsigned char)1)));
1134                 val.storeValue(inc);
1135
1136                 return val;
1137         }
1138
1139         RValue<Bool> operator<(RValue<Byte> lhs, RValue<Byte> rhs)
1140         {
1141                 return RValue<Bool>(Nucleus::createICmpULT(lhs.value, rhs.value));
1142         }
1143
1144         RValue<Bool> operator<=(RValue<Byte> lhs, RValue<Byte> rhs)
1145         {
1146                 return RValue<Bool>(Nucleus::createICmpULE(lhs.value, rhs.value));
1147         }
1148
1149         RValue<Bool> operator>(RValue<Byte> lhs, RValue<Byte> rhs)
1150         {
1151                 return RValue<Bool>(Nucleus::createICmpUGT(lhs.value, rhs.value));
1152         }
1153
1154         RValue<Bool> operator>=(RValue<Byte> lhs, RValue<Byte> rhs)
1155         {
1156                 return RValue<Bool>(Nucleus::createICmpUGE(lhs.value, rhs.value));
1157         }
1158
1159         RValue<Bool> operator!=(RValue<Byte> lhs, RValue<Byte> rhs)
1160         {
1161                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
1162         }
1163
1164         RValue<Bool> operator==(RValue<Byte> lhs, RValue<Byte> rhs)
1165         {
1166                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
1167         }
1168
1169         Type *Byte::getType()
1170         {
1171                 return T(llvm::Type::getInt8Ty(*::context));
1172         }
1173
1174         SByte::SByte(Argument<SByte> argument)
1175         {
1176                 storeValue(argument.value);
1177         }
1178
1179         SByte::SByte(RValue<Int> cast)
1180         {
1181                 Value *integer = Nucleus::createTrunc(cast.value, SByte::getType());
1182
1183                 storeValue(integer);
1184         }
1185
1186         SByte::SByte(RValue<Short> cast)
1187         {
1188                 Value *integer = Nucleus::createTrunc(cast.value, SByte::getType());
1189
1190                 storeValue(integer);
1191         }
1192
1193         SByte::SByte()
1194         {
1195         }
1196
1197         SByte::SByte(signed char x)
1198         {
1199                 storeValue(Nucleus::createConstantByte(x));
1200         }
1201
1202         SByte::SByte(RValue<SByte> rhs)
1203         {
1204                 storeValue(rhs.value);
1205         }
1206
1207         SByte::SByte(const SByte &rhs)
1208         {
1209                 Value *value = rhs.loadValue();
1210                 storeValue(value);
1211         }
1212
1213         SByte::SByte(const Reference<SByte> &rhs)
1214         {
1215                 Value *value = rhs.loadValue();
1216                 storeValue(value);
1217         }
1218
1219         RValue<SByte> SByte::operator=(RValue<SByte> rhs) const
1220         {
1221                 storeValue(rhs.value);
1222
1223                 return rhs;
1224         }
1225
1226         RValue<SByte> SByte::operator=(const SByte &rhs) const
1227         {
1228                 Value *value = rhs.loadValue();
1229                 storeValue(value);
1230
1231                 return RValue<SByte>(value);
1232         }
1233
1234         RValue<SByte> SByte::operator=(const Reference<SByte> &rhs) const
1235         {
1236                 Value *value = rhs.loadValue();
1237                 storeValue(value);
1238
1239                 return RValue<SByte>(value);
1240         }
1241
1242         RValue<SByte> operator+(RValue<SByte> lhs, RValue<SByte> rhs)
1243         {
1244                 return RValue<SByte>(Nucleus::createAdd(lhs.value, rhs.value));
1245         }
1246
1247         RValue<SByte> operator-(RValue<SByte> lhs, RValue<SByte> rhs)
1248         {
1249                 return RValue<SByte>(Nucleus::createSub(lhs.value, rhs.value));
1250         }
1251
1252         RValue<SByte> operator*(RValue<SByte> lhs, RValue<SByte> rhs)
1253         {
1254                 return RValue<SByte>(Nucleus::createMul(lhs.value, rhs.value));
1255         }
1256
1257         RValue<SByte> operator/(RValue<SByte> lhs, RValue<SByte> rhs)
1258         {
1259                 return RValue<SByte>(Nucleus::createSDiv(lhs.value, rhs.value));
1260         }
1261
1262         RValue<SByte> operator%(RValue<SByte> lhs, RValue<SByte> rhs)
1263         {
1264                 return RValue<SByte>(Nucleus::createSRem(lhs.value, rhs.value));
1265         }
1266
1267         RValue<SByte> operator&(RValue<SByte> lhs, RValue<SByte> rhs)
1268         {
1269                 return RValue<SByte>(Nucleus::createAnd(lhs.value, rhs.value));
1270         }
1271
1272         RValue<SByte> operator|(RValue<SByte> lhs, RValue<SByte> rhs)
1273         {
1274                 return RValue<SByte>(Nucleus::createOr(lhs.value, rhs.value));
1275         }
1276
1277         RValue<SByte> operator^(RValue<SByte> lhs, RValue<SByte> rhs)
1278         {
1279                 return RValue<SByte>(Nucleus::createXor(lhs.value, rhs.value));
1280         }
1281
1282         RValue<SByte> operator<<(RValue<SByte> lhs, RValue<SByte> rhs)
1283         {
1284                 return RValue<SByte>(Nucleus::createShl(lhs.value, rhs.value));
1285         }
1286
1287         RValue<SByte> operator>>(RValue<SByte> lhs, RValue<SByte> rhs)
1288         {
1289                 return RValue<SByte>(Nucleus::createAShr(lhs.value, rhs.value));
1290         }
1291
1292         RValue<SByte> operator+=(const SByte &lhs, RValue<SByte> rhs)
1293         {
1294                 return lhs = lhs + rhs;
1295         }
1296
1297         RValue<SByte> operator-=(const SByte &lhs, RValue<SByte> rhs)
1298         {
1299                 return lhs = lhs - rhs;
1300         }
1301
1302         RValue<SByte> operator*=(const SByte &lhs, RValue<SByte> rhs)
1303         {
1304                 return lhs = lhs * rhs;
1305         }
1306
1307         RValue<SByte> operator/=(const SByte &lhs, RValue<SByte> rhs)
1308         {
1309                 return lhs = lhs / rhs;
1310         }
1311
1312         RValue<SByte> operator%=(const SByte &lhs, RValue<SByte> rhs)
1313         {
1314                 return lhs = lhs % rhs;
1315         }
1316
1317         RValue<SByte> operator&=(const SByte &lhs, RValue<SByte> rhs)
1318         {
1319                 return lhs = lhs & rhs;
1320         }
1321
1322         RValue<SByte> operator|=(const SByte &lhs, RValue<SByte> rhs)
1323         {
1324                 return lhs = lhs | rhs;
1325         }
1326
1327         RValue<SByte> operator^=(const SByte &lhs, RValue<SByte> rhs)
1328         {
1329                 return lhs = lhs ^ rhs;
1330         }
1331
1332         RValue<SByte> operator<<=(const SByte &lhs, RValue<SByte> rhs)
1333         {
1334                 return lhs = lhs << rhs;
1335         }
1336
1337         RValue<SByte> operator>>=(const SByte &lhs, RValue<SByte> rhs)
1338         {
1339                 return lhs = lhs >> rhs;
1340         }
1341
1342         RValue<SByte> operator+(RValue<SByte> val)
1343         {
1344                 return val;
1345         }
1346
1347         RValue<SByte> operator-(RValue<SByte> val)
1348         {
1349                 return RValue<SByte>(Nucleus::createNeg(val.value));
1350         }
1351
1352         RValue<SByte> operator~(RValue<SByte> val)
1353         {
1354                 return RValue<SByte>(Nucleus::createNot(val.value));
1355         }
1356
1357         RValue<SByte> operator++(const SByte &val, int)   // Post-increment
1358         {
1359                 RValue<SByte> res = val;
1360
1361                 Value *inc = Nucleus::createAdd(res.value, V(Nucleus::createConstantByte((signed char)1)));
1362                 val.storeValue(inc);
1363
1364                 return res;
1365         }
1366
1367         const SByte &operator++(const SByte &val)   // Pre-increment
1368         {
1369                 Value *inc = Nucleus::createAdd(val.loadValue(), V(Nucleus::createConstantByte((signed char)1)));
1370                 val.storeValue(inc);
1371
1372                 return val;
1373         }
1374
1375         RValue<SByte> operator--(const SByte &val, int)   // Post-decrement
1376         {
1377                 RValue<SByte> res = val;
1378
1379                 Value *inc = Nucleus::createSub(res.value, V(Nucleus::createConstantByte((signed char)1)));
1380                 val.storeValue(inc);
1381
1382                 return res;
1383         }
1384
1385         const SByte &operator--(const SByte &val)   // Pre-decrement
1386         {
1387                 Value *inc = Nucleus::createSub(val.loadValue(), V(Nucleus::createConstantByte((signed char)1)));
1388                 val.storeValue(inc);
1389
1390                 return val;
1391         }
1392
1393         RValue<Bool> operator<(RValue<SByte> lhs, RValue<SByte> rhs)
1394         {
1395                 return RValue<Bool>(Nucleus::createICmpSLT(lhs.value, rhs.value));
1396         }
1397
1398         RValue<Bool> operator<=(RValue<SByte> lhs, RValue<SByte> rhs)
1399         {
1400                 return RValue<Bool>(Nucleus::createICmpSLE(lhs.value, rhs.value));
1401         }
1402
1403         RValue<Bool> operator>(RValue<SByte> lhs, RValue<SByte> rhs)
1404         {
1405                 return RValue<Bool>(Nucleus::createICmpSGT(lhs.value, rhs.value));
1406         }
1407
1408         RValue<Bool> operator>=(RValue<SByte> lhs, RValue<SByte> rhs)
1409         {
1410                 return RValue<Bool>(Nucleus::createICmpSGE(lhs.value, rhs.value));
1411         }
1412
1413         RValue<Bool> operator!=(RValue<SByte> lhs, RValue<SByte> rhs)
1414         {
1415                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
1416         }
1417
1418         RValue<Bool> operator==(RValue<SByte> lhs, RValue<SByte> rhs)
1419         {
1420                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
1421         }
1422
1423         Type *SByte::getType()
1424         {
1425                 return T(llvm::Type::getInt8Ty(*::context));
1426         }
1427
1428         Short::Short(Argument<Short> argument)
1429         {
1430                 storeValue(argument.value);
1431         }
1432
1433         Short::Short(RValue<Int> cast)
1434         {
1435                 Value *integer = Nucleus::createTrunc(cast.value, Short::getType());
1436
1437                 storeValue(integer);
1438         }
1439
1440         Short::Short()
1441         {
1442         }
1443
1444         Short::Short(short x)
1445         {
1446                 storeValue(Nucleus::createConstantShort(x));
1447         }
1448
1449         Short::Short(RValue<Short> rhs)
1450         {
1451                 storeValue(rhs.value);
1452         }
1453
1454         Short::Short(const Short &rhs)
1455         {
1456                 Value *value = rhs.loadValue();
1457                 storeValue(value);
1458         }
1459
1460         Short::Short(const Reference<Short> &rhs)
1461         {
1462                 Value *value = rhs.loadValue();
1463                 storeValue(value);
1464         }
1465
1466         RValue<Short> Short::operator=(RValue<Short> rhs) const
1467         {
1468                 storeValue(rhs.value);
1469
1470                 return rhs;
1471         }
1472
1473         RValue<Short> Short::operator=(const Short &rhs) const
1474         {
1475                 Value *value = rhs.loadValue();
1476                 storeValue(value);
1477
1478                 return RValue<Short>(value);
1479         }
1480
1481         RValue<Short> Short::operator=(const Reference<Short> &rhs) const
1482         {
1483                 Value *value = rhs.loadValue();
1484                 storeValue(value);
1485
1486                 return RValue<Short>(value);
1487         }
1488
1489         RValue<Short> operator+(RValue<Short> lhs, RValue<Short> rhs)
1490         {
1491                 return RValue<Short>(Nucleus::createAdd(lhs.value, rhs.value));
1492         }
1493
1494         RValue<Short> operator-(RValue<Short> lhs, RValue<Short> rhs)
1495         {
1496                 return RValue<Short>(Nucleus::createSub(lhs.value, rhs.value));
1497         }
1498
1499         RValue<Short> operator*(RValue<Short> lhs, RValue<Short> rhs)
1500         {
1501                 return RValue<Short>(Nucleus::createMul(lhs.value, rhs.value));
1502         }
1503
1504         RValue<Short> operator/(RValue<Short> lhs, RValue<Short> rhs)
1505         {
1506                 return RValue<Short>(Nucleus::createSDiv(lhs.value, rhs.value));
1507         }
1508
1509         RValue<Short> operator%(RValue<Short> lhs, RValue<Short> rhs)
1510         {
1511                 return RValue<Short>(Nucleus::createSRem(lhs.value, rhs.value));
1512         }
1513
1514         RValue<Short> operator&(RValue<Short> lhs, RValue<Short> rhs)
1515         {
1516                 return RValue<Short>(Nucleus::createAnd(lhs.value, rhs.value));
1517         }
1518
1519         RValue<Short> operator|(RValue<Short> lhs, RValue<Short> rhs)
1520         {
1521                 return RValue<Short>(Nucleus::createOr(lhs.value, rhs.value));
1522         }
1523
1524         RValue<Short> operator^(RValue<Short> lhs, RValue<Short> rhs)
1525         {
1526                 return RValue<Short>(Nucleus::createXor(lhs.value, rhs.value));
1527         }
1528
1529         RValue<Short> operator<<(RValue<Short> lhs, RValue<Short> rhs)
1530         {
1531                 return RValue<Short>(Nucleus::createShl(lhs.value, rhs.value));
1532         }
1533
1534         RValue<Short> operator>>(RValue<Short> lhs, RValue<Short> rhs)
1535         {
1536                 return RValue<Short>(Nucleus::createAShr(lhs.value, rhs.value));
1537         }
1538
1539         RValue<Short> operator+=(const Short &lhs, RValue<Short> rhs)
1540         {
1541                 return lhs = lhs + rhs;
1542         }
1543
1544         RValue<Short> operator-=(const Short &lhs, RValue<Short> rhs)
1545         {
1546                 return lhs = lhs - rhs;
1547         }
1548
1549         RValue<Short> operator*=(const Short &lhs, RValue<Short> rhs)
1550         {
1551                 return lhs = lhs * rhs;
1552         }
1553
1554         RValue<Short> operator/=(const Short &lhs, RValue<Short> rhs)
1555         {
1556                 return lhs = lhs / rhs;
1557         }
1558
1559         RValue<Short> operator%=(const Short &lhs, RValue<Short> rhs)
1560         {
1561                 return lhs = lhs % rhs;
1562         }
1563
1564         RValue<Short> operator&=(const Short &lhs, RValue<Short> rhs)
1565         {
1566                 return lhs = lhs & rhs;
1567         }
1568
1569         RValue<Short> operator|=(const Short &lhs, RValue<Short> rhs)
1570         {
1571                 return lhs = lhs | rhs;
1572         }
1573
1574         RValue<Short> operator^=(const Short &lhs, RValue<Short> rhs)
1575         {
1576                 return lhs = lhs ^ rhs;
1577         }
1578
1579         RValue<Short> operator<<=(const Short &lhs, RValue<Short> rhs)
1580         {
1581                 return lhs = lhs << rhs;
1582         }
1583
1584         RValue<Short> operator>>=(const Short &lhs, RValue<Short> rhs)
1585         {
1586                 return lhs = lhs >> rhs;
1587         }
1588
1589         RValue<Short> operator+(RValue<Short> val)
1590         {
1591                 return val;
1592         }
1593
1594         RValue<Short> operator-(RValue<Short> val)
1595         {
1596                 return RValue<Short>(Nucleus::createNeg(val.value));
1597         }
1598
1599         RValue<Short> operator~(RValue<Short> val)
1600         {
1601                 return RValue<Short>(Nucleus::createNot(val.value));
1602         }
1603
1604         RValue<Short> operator++(const Short &val, int)   // Post-increment
1605         {
1606                 RValue<Short> res = val;
1607
1608                 Value *inc = Nucleus::createAdd(res.value, V(Nucleus::createConstantShort((short)1)));
1609                 val.storeValue(inc);
1610
1611                 return res;
1612         }
1613
1614         const Short &operator++(const Short &val)   // Pre-increment
1615         {
1616                 Value *inc = Nucleus::createAdd(val.loadValue(), V(Nucleus::createConstantShort((short)1)));
1617                 val.storeValue(inc);
1618
1619                 return val;
1620         }
1621
1622         RValue<Short> operator--(const Short &val, int)   // Post-decrement
1623         {
1624                 RValue<Short> res = val;
1625
1626                 Value *inc = Nucleus::createSub(res.value, V(Nucleus::createConstantShort((short)1)));
1627                 val.storeValue(inc);
1628
1629                 return res;
1630         }
1631
1632         const Short &operator--(const Short &val)   // Pre-decrement
1633         {
1634                 Value *inc = Nucleus::createSub(val.loadValue(), V(Nucleus::createConstantShort((short)1)));
1635                 val.storeValue(inc);
1636
1637                 return val;
1638         }
1639
1640         RValue<Bool> operator<(RValue<Short> lhs, RValue<Short> rhs)
1641         {
1642                 return RValue<Bool>(Nucleus::createICmpSLT(lhs.value, rhs.value));
1643         }
1644
1645         RValue<Bool> operator<=(RValue<Short> lhs, RValue<Short> rhs)
1646         {
1647                 return RValue<Bool>(Nucleus::createICmpSLE(lhs.value, rhs.value));
1648         }
1649
1650         RValue<Bool> operator>(RValue<Short> lhs, RValue<Short> rhs)
1651         {
1652                 return RValue<Bool>(Nucleus::createICmpSGT(lhs.value, rhs.value));
1653         }
1654
1655         RValue<Bool> operator>=(RValue<Short> lhs, RValue<Short> rhs)
1656         {
1657                 return RValue<Bool>(Nucleus::createICmpSGE(lhs.value, rhs.value));
1658         }
1659
1660         RValue<Bool> operator!=(RValue<Short> lhs, RValue<Short> rhs)
1661         {
1662                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
1663         }
1664
1665         RValue<Bool> operator==(RValue<Short> lhs, RValue<Short> rhs)
1666         {
1667                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
1668         }
1669
1670         Type *Short::getType()
1671         {
1672                 return T(llvm::Type::getInt16Ty(*::context));
1673         }
1674
1675         UShort::UShort(Argument<UShort> argument)
1676         {
1677                 storeValue(argument.value);
1678         }
1679
1680         UShort::UShort(RValue<UInt> cast)
1681         {
1682                 Value *integer = Nucleus::createTrunc(cast.value, UShort::getType());
1683
1684                 storeValue(integer);
1685         }
1686
1687         UShort::UShort(RValue<Int> cast)
1688         {
1689                 Value *integer = Nucleus::createTrunc(cast.value, UShort::getType());
1690
1691                 storeValue(integer);
1692         }
1693
1694         UShort::UShort()
1695         {
1696         }
1697
1698         UShort::UShort(unsigned short x)
1699         {
1700                 storeValue(Nucleus::createConstantShort(x));
1701         }
1702
1703         UShort::UShort(RValue<UShort> rhs)
1704         {
1705                 storeValue(rhs.value);
1706         }
1707
1708         UShort::UShort(const UShort &rhs)
1709         {
1710                 Value *value = rhs.loadValue();
1711                 storeValue(value);
1712         }
1713
1714         UShort::UShort(const Reference<UShort> &rhs)
1715         {
1716                 Value *value = rhs.loadValue();
1717                 storeValue(value);
1718         }
1719
1720         RValue<UShort> UShort::operator=(RValue<UShort> rhs) const
1721         {
1722                 storeValue(rhs.value);
1723
1724                 return rhs;
1725         }
1726
1727         RValue<UShort> UShort::operator=(const UShort &rhs) const
1728         {
1729                 Value *value = rhs.loadValue();
1730                 storeValue(value);
1731
1732                 return RValue<UShort>(value);
1733         }
1734
1735         RValue<UShort> UShort::operator=(const Reference<UShort> &rhs) const
1736         {
1737                 Value *value = rhs.loadValue();
1738                 storeValue(value);
1739
1740                 return RValue<UShort>(value);
1741         }
1742
1743         RValue<UShort> operator+(RValue<UShort> lhs, RValue<UShort> rhs)
1744         {
1745                 return RValue<UShort>(Nucleus::createAdd(lhs.value, rhs.value));
1746         }
1747
1748         RValue<UShort> operator-(RValue<UShort> lhs, RValue<UShort> rhs)
1749         {
1750                 return RValue<UShort>(Nucleus::createSub(lhs.value, rhs.value));
1751         }
1752
1753         RValue<UShort> operator*(RValue<UShort> lhs, RValue<UShort> rhs)
1754         {
1755                 return RValue<UShort>(Nucleus::createMul(lhs.value, rhs.value));
1756         }
1757
1758         RValue<UShort> operator/(RValue<UShort> lhs, RValue<UShort> rhs)
1759         {
1760                 return RValue<UShort>(Nucleus::createUDiv(lhs.value, rhs.value));
1761         }
1762
1763         RValue<UShort> operator%(RValue<UShort> lhs, RValue<UShort> rhs)
1764         {
1765                 return RValue<UShort>(Nucleus::createURem(lhs.value, rhs.value));
1766         }
1767
1768         RValue<UShort> operator&(RValue<UShort> lhs, RValue<UShort> rhs)
1769         {
1770                 return RValue<UShort>(Nucleus::createAnd(lhs.value, rhs.value));
1771         }
1772
1773         RValue<UShort> operator|(RValue<UShort> lhs, RValue<UShort> rhs)
1774         {
1775                 return RValue<UShort>(Nucleus::createOr(lhs.value, rhs.value));
1776         }
1777
1778         RValue<UShort> operator^(RValue<UShort> lhs, RValue<UShort> rhs)
1779         {
1780                 return RValue<UShort>(Nucleus::createXor(lhs.value, rhs.value));
1781         }
1782
1783         RValue<UShort> operator<<(RValue<UShort> lhs, RValue<UShort> rhs)
1784         {
1785                 return RValue<UShort>(Nucleus::createShl(lhs.value, rhs.value));
1786         }
1787
1788         RValue<UShort> operator>>(RValue<UShort> lhs, RValue<UShort> rhs)
1789         {
1790                 return RValue<UShort>(Nucleus::createLShr(lhs.value, rhs.value));
1791         }
1792
1793         RValue<UShort> operator+=(const UShort &lhs, RValue<UShort> rhs)
1794         {
1795                 return lhs = lhs + rhs;
1796         }
1797
1798         RValue<UShort> operator-=(const UShort &lhs, RValue<UShort> rhs)
1799         {
1800                 return lhs = lhs - rhs;
1801         }
1802
1803         RValue<UShort> operator*=(const UShort &lhs, RValue<UShort> rhs)
1804         {
1805                 return lhs = lhs * rhs;
1806         }
1807
1808         RValue<UShort> operator/=(const UShort &lhs, RValue<UShort> rhs)
1809         {
1810                 return lhs = lhs / rhs;
1811         }
1812
1813         RValue<UShort> operator%=(const UShort &lhs, RValue<UShort> rhs)
1814         {
1815                 return lhs = lhs % rhs;
1816         }
1817
1818         RValue<UShort> operator&=(const UShort &lhs, RValue<UShort> rhs)
1819         {
1820                 return lhs = lhs & rhs;
1821         }
1822
1823         RValue<UShort> operator|=(const UShort &lhs, RValue<UShort> rhs)
1824         {
1825                 return lhs = lhs | rhs;
1826         }
1827
1828         RValue<UShort> operator^=(const UShort &lhs, RValue<UShort> rhs)
1829         {
1830                 return lhs = lhs ^ rhs;
1831         }
1832
1833         RValue<UShort> operator<<=(const UShort &lhs, RValue<UShort> rhs)
1834         {
1835                 return lhs = lhs << rhs;
1836         }
1837
1838         RValue<UShort> operator>>=(const UShort &lhs, RValue<UShort> rhs)
1839         {
1840                 return lhs = lhs >> rhs;
1841         }
1842
1843         RValue<UShort> operator+(RValue<UShort> val)
1844         {
1845                 return val;
1846         }
1847
1848         RValue<UShort> operator-(RValue<UShort> val)
1849         {
1850                 return RValue<UShort>(Nucleus::createNeg(val.value));
1851         }
1852
1853         RValue<UShort> operator~(RValue<UShort> val)
1854         {
1855                 return RValue<UShort>(Nucleus::createNot(val.value));
1856         }
1857
1858         RValue<UShort> operator++(const UShort &val, int)   // Post-increment
1859         {
1860                 RValue<UShort> res = val;
1861
1862                 Value *inc = Nucleus::createAdd(res.value, V(Nucleus::createConstantShort((unsigned short)1)));
1863                 val.storeValue(inc);
1864
1865                 return res;
1866         }
1867
1868         const UShort &operator++(const UShort &val)   // Pre-increment
1869         {
1870                 Value *inc = Nucleus::createAdd(val.loadValue(), V(Nucleus::createConstantShort((unsigned short)1)));
1871                 val.storeValue(inc);
1872
1873                 return val;
1874         }
1875
1876         RValue<UShort> operator--(const UShort &val, int)   // Post-decrement
1877         {
1878                 RValue<UShort> res = val;
1879
1880                 Value *inc = Nucleus::createSub(res.value, V(Nucleus::createConstantShort((unsigned short)1)));
1881                 val.storeValue(inc);
1882
1883                 return res;
1884         }
1885
1886         const UShort &operator--(const UShort &val)   // Pre-decrement
1887         {
1888                 Value *inc = Nucleus::createSub(val.loadValue(), V(Nucleus::createConstantShort((unsigned short)1)));
1889                 val.storeValue(inc);
1890
1891                 return val;
1892         }
1893
1894         RValue<Bool> operator<(RValue<UShort> lhs, RValue<UShort> rhs)
1895         {
1896                 return RValue<Bool>(Nucleus::createICmpULT(lhs.value, rhs.value));
1897         }
1898
1899         RValue<Bool> operator<=(RValue<UShort> lhs, RValue<UShort> rhs)
1900         {
1901                 return RValue<Bool>(Nucleus::createICmpULE(lhs.value, rhs.value));
1902         }
1903
1904         RValue<Bool> operator>(RValue<UShort> lhs, RValue<UShort> rhs)
1905         {
1906                 return RValue<Bool>(Nucleus::createICmpUGT(lhs.value, rhs.value));
1907         }
1908
1909         RValue<Bool> operator>=(RValue<UShort> lhs, RValue<UShort> rhs)
1910         {
1911                 return RValue<Bool>(Nucleus::createICmpUGE(lhs.value, rhs.value));
1912         }
1913
1914         RValue<Bool> operator!=(RValue<UShort> lhs, RValue<UShort> rhs)
1915         {
1916                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
1917         }
1918
1919         RValue<Bool> operator==(RValue<UShort> lhs, RValue<UShort> rhs)
1920         {
1921                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
1922         }
1923
1924         Type *UShort::getType()
1925         {
1926                 return T(llvm::Type::getInt16Ty(*::context));
1927         }
1928
1929         Type *Byte4::getType()
1930         {
1931                 #if 0
1932                         return T(VectorType::get(Byte::getType(), 4));
1933                 #else
1934                         return UInt::getType();   // FIXME: LLVM doesn't manipulate it as one 32-bit block
1935                 #endif
1936         }
1937
1938         Type *SByte4::getType()
1939         {
1940                 #if 0
1941                         return T(VectorType::get(SByte::getType(), 4));
1942                 #else
1943                         return Int::getType();   // FIXME: LLVM doesn't manipulate it as one 32-bit block
1944                 #endif
1945         }
1946
1947         Byte8::Byte8()
1948         {
1949         //      xyzw.parent = this;
1950         }
1951
1952         Byte8::Byte8(uint8_t x0, uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, uint8_t x5, uint8_t x6, uint8_t x7)
1953         {
1954         //      xyzw.parent = this;
1955
1956                 Constant *constantVector[8];
1957                 constantVector[0] = Nucleus::createConstantByte(x0);
1958                 constantVector[1] = Nucleus::createConstantByte(x1);
1959                 constantVector[2] = Nucleus::createConstantByte(x2);
1960                 constantVector[3] = Nucleus::createConstantByte(x3);
1961                 constantVector[4] = Nucleus::createConstantByte(x4);
1962                 constantVector[5] = Nucleus::createConstantByte(x5);
1963                 constantVector[6] = Nucleus::createConstantByte(x6);
1964                 constantVector[7] = Nucleus::createConstantByte(x7);
1965                 Value *vector = V(Nucleus::createConstantVector(constantVector, 8));
1966
1967                 storeValue(Nucleus::createBitCast(vector, getType()));
1968         }
1969
1970         Byte8::Byte8(int64_t x)
1971         {
1972         //      xyzw.parent = this;
1973
1974                 Constant *constantVector[8];
1975                 constantVector[0] = Nucleus::createConstantByte((unsigned char)(x >>  0));
1976                 constantVector[1] = Nucleus::createConstantByte((unsigned char)(x >>  8));
1977                 constantVector[2] = Nucleus::createConstantByte((unsigned char)(x >> 16));
1978                 constantVector[3] = Nucleus::createConstantByte((unsigned char)(x >> 24));
1979                 constantVector[4] = Nucleus::createConstantByte((unsigned char)(x >> 32));
1980                 constantVector[5] = Nucleus::createConstantByte((unsigned char)(x >> 40));
1981                 constantVector[6] = Nucleus::createConstantByte((unsigned char)(x >> 48));
1982                 constantVector[7] = Nucleus::createConstantByte((unsigned char)(x >> 56));
1983                 Value *vector = V(Nucleus::createConstantVector(constantVector, 8));
1984
1985                 storeValue(Nucleus::createBitCast(vector, getType()));
1986         }
1987
1988         Byte8::Byte8(RValue<Byte8> rhs)
1989         {
1990         //      xyzw.parent = this;
1991
1992                 storeValue(rhs.value);
1993         }
1994
1995         Byte8::Byte8(const Byte8 &rhs)
1996         {
1997         //      xyzw.parent = this;
1998
1999                 Value *value = rhs.loadValue();
2000                 storeValue(value);
2001         }
2002
2003         Byte8::Byte8(const Reference<Byte8> &rhs)
2004         {
2005         //      xyzw.parent = this;
2006
2007                 Value *value = rhs.loadValue();
2008                 storeValue(value);
2009         }
2010
2011         RValue<Byte8> Byte8::operator=(RValue<Byte8> rhs) const
2012         {
2013                 storeValue(rhs.value);
2014
2015                 return rhs;
2016         }
2017
2018         RValue<Byte8> Byte8::operator=(const Byte8 &rhs) const
2019         {
2020                 Value *value = rhs.loadValue();
2021                 storeValue(value);
2022
2023                 return RValue<Byte8>(value);
2024         }
2025
2026         RValue<Byte8> Byte8::operator=(const Reference<Byte8> &rhs) const
2027         {
2028                 Value *value = rhs.loadValue();
2029                 storeValue(value);
2030
2031                 return RValue<Byte8>(value);
2032         }
2033
2034         RValue<Byte8> operator+(RValue<Byte8> lhs, RValue<Byte8> rhs)
2035         {
2036                 if(CPUID::supportsMMX2())
2037                 {
2038                         return x86::paddb(lhs, rhs);
2039                 }
2040                 else
2041                 {
2042                         return RValue<Byte8>(Nucleus::createAdd(lhs.value, rhs.value));
2043                 }
2044         }
2045
2046         RValue<Byte8> operator-(RValue<Byte8> lhs, RValue<Byte8> rhs)
2047         {
2048                 if(CPUID::supportsMMX2())
2049                 {
2050                         return x86::psubb(lhs, rhs);
2051                 }
2052                 else
2053                 {
2054                         return RValue<Byte8>(Nucleus::createSub(lhs.value, rhs.value));
2055                 }
2056         }
2057
2058 //      RValue<Byte8> operator*(RValue<Byte8> lhs, RValue<Byte8> rhs)
2059 //      {
2060 //              return RValue<Byte8>(Nucleus::createMul(lhs.value, rhs.value));
2061 //      }
2062
2063 //      RValue<Byte8> operator/(RValue<Byte8> lhs, RValue<Byte8> rhs)
2064 //      {
2065 //              return RValue<Byte8>(Nucleus::createUDiv(lhs.value, rhs.value));
2066 //      }
2067
2068 //      RValue<Byte8> operator%(RValue<Byte8> lhs, RValue<Byte8> rhs)
2069 //      {
2070 //              return RValue<Byte8>(Nucleus::createURem(lhs.value, rhs.value));
2071 //      }
2072
2073         RValue<Byte8> operator&(RValue<Byte8> lhs, RValue<Byte8> rhs)
2074         {
2075                 if(CPUID::supportsMMX2())
2076                 {
2077                         return As<Byte8>(x86::pand(As<Short4>(lhs), As<Short4>(rhs)));
2078                 }
2079                 else
2080                 {
2081                         return RValue<Byte8>(Nucleus::createAnd(lhs.value, rhs.value));
2082                 }
2083         }
2084
2085         RValue<Byte8> operator|(RValue<Byte8> lhs, RValue<Byte8> rhs)
2086         {
2087                 if(CPUID::supportsMMX2())
2088                 {
2089                         return As<Byte8>(x86::por(As<Short4>(lhs), As<Short4>(rhs)));
2090                 }
2091                 else
2092                 {
2093                         return RValue<Byte8>(Nucleus::createOr(lhs.value, rhs.value));
2094                 }
2095         }
2096
2097         RValue<Byte8> operator^(RValue<Byte8> lhs, RValue<Byte8> rhs)
2098         {
2099                 if(CPUID::supportsMMX2())
2100                 {
2101                         return As<Byte8>(x86::pxor(As<Short4>(lhs), As<Short4>(rhs)));
2102                 }
2103                 else
2104                 {
2105                         return RValue<Byte8>(Nucleus::createXor(lhs.value, rhs.value));
2106                 }
2107         }
2108
2109 //      RValue<Byte8> operator<<(RValue<Byte8> lhs, unsigned char rhs)
2110 //      {
2111 //              return RValue<Byte8>(Nucleus::createShl(lhs.value, rhs.value));
2112 //      }
2113
2114 //      RValue<Byte8> operator>>(RValue<Byte8> lhs, unsigned char rhs)
2115 //      {
2116 //              return RValue<Byte8>(Nucleus::createLShr(lhs.value, rhs.value));
2117 //      }
2118
2119         RValue<Byte8> operator+=(const Byte8 &lhs, RValue<Byte8> rhs)
2120         {
2121                 return lhs = lhs + rhs;
2122         }
2123
2124         RValue<Byte8> operator-=(const Byte8 &lhs, RValue<Byte8> rhs)
2125         {
2126                 return lhs = lhs - rhs;
2127         }
2128
2129 //      RValue<Byte8> operator*=(const Byte8 &lhs, RValue<Byte8> rhs)
2130 //      {
2131 //              return lhs = lhs * rhs;
2132 //      }
2133
2134 //      RValue<Byte8> operator/=(const Byte8 &lhs, RValue<Byte8> rhs)
2135 //      {
2136 //              return lhs = lhs / rhs;
2137 //      }
2138
2139 //      RValue<Byte8> operator%=(const Byte8 &lhs, RValue<Byte8> rhs)
2140 //      {
2141 //              return lhs = lhs % rhs;
2142 //      }
2143
2144         RValue<Byte8> operator&=(const Byte8 &lhs, RValue<Byte8> rhs)
2145         {
2146                 return lhs = lhs & rhs;
2147         }
2148
2149         RValue<Byte8> operator|=(const Byte8 &lhs, RValue<Byte8> rhs)
2150         {
2151                 return lhs = lhs | rhs;
2152         }
2153
2154         RValue<Byte8> operator^=(const Byte8 &lhs, RValue<Byte8> rhs)
2155         {
2156                 return lhs = lhs ^ rhs;
2157         }
2158
2159 //      RValue<Byte8> operator<<=(const Byte8 &lhs, RValue<Byte8> rhs)
2160 //      {
2161 //              return lhs = lhs << rhs;
2162 //      }
2163
2164 //      RValue<Byte8> operator>>=(const Byte8 &lhs, RValue<Byte8> rhs)
2165 //      {
2166 //              return lhs = lhs >> rhs;
2167 //      }
2168
2169 //      RValue<Byte8> operator+(RValue<Byte8> val)
2170 //      {
2171 //              return val;
2172 //      }
2173
2174 //      RValue<Byte8> operator-(RValue<Byte8> val)
2175 //      {
2176 //              return RValue<Byte8>(Nucleus::createNeg(val.value));
2177 //      }
2178
2179         RValue<Byte8> operator~(RValue<Byte8> val)
2180         {
2181                 if(CPUID::supportsMMX2())
2182                 {
2183                         return val ^ Byte8(0xFFFFFFFFFFFFFFFF);
2184                 }
2185                 else
2186                 {
2187                         return RValue<Byte8>(Nucleus::createNot(val.value));
2188                 }
2189         }
2190
2191         RValue<Byte8> AddSat(RValue<Byte8> x, RValue<Byte8> y)
2192         {
2193                 return x86::paddusb(x, y);
2194         }
2195
2196         RValue<Byte8> SubSat(RValue<Byte8> x, RValue<Byte8> y)
2197         {
2198                 return x86::psubusb(x, y);
2199         }
2200
2201         RValue<Short4> Unpack(RValue<Byte4> x)
2202         {
2203                 Value *int2 = Nucleus::createInsertElement(V(UndefValue::get(VectorType::get(Int::getType(), 2))), x.value, 0);
2204                 Value *byte8 = Nucleus::createBitCast(int2, Byte8::getType());
2205
2206                 return UnpackLow(RValue<Byte8>(byte8), RValue<Byte8>(byte8));
2207         }
2208
2209         RValue<Short4> UnpackLow(RValue<Byte8> x, RValue<Byte8> y)
2210         {
2211                 if(CPUID::supportsMMX2())
2212                 {
2213                         return x86::punpcklbw(x, y);
2214                 }
2215                 else
2216                 {
2217                         Constant *shuffle[8];
2218                         shuffle[0] = Nucleus::createConstantInt(0);
2219                         shuffle[1] = Nucleus::createConstantInt(8);
2220                         shuffle[2] = Nucleus::createConstantInt(1);
2221                         shuffle[3] = Nucleus::createConstantInt(9);
2222                         shuffle[4] = Nucleus::createConstantInt(2);
2223                         shuffle[5] = Nucleus::createConstantInt(10);
2224                         shuffle[6] = Nucleus::createConstantInt(3);
2225                         shuffle[7] = Nucleus::createConstantInt(11);
2226
2227                         Value *packed = Nucleus::createShuffleVector(x.value, y.value, V(Nucleus::createConstantVector(shuffle, 8)));
2228
2229                         return RValue<Short4>(Nucleus::createBitCast(packed, Short4::getType()));
2230                 }
2231         }
2232
2233         RValue<Short4> UnpackHigh(RValue<Byte8> x, RValue<Byte8> y)
2234         {
2235                 if(CPUID::supportsMMX2())
2236                 {
2237                         return x86::punpckhbw(x, y);
2238                 }
2239                 else
2240                 {
2241                         Constant *shuffle[8];
2242                         shuffle[0] = Nucleus::createConstantInt(4);
2243                         shuffle[1] = Nucleus::createConstantInt(12);
2244                         shuffle[2] = Nucleus::createConstantInt(5);
2245                         shuffle[3] = Nucleus::createConstantInt(13);
2246                         shuffle[4] = Nucleus::createConstantInt(6);
2247                         shuffle[5] = Nucleus::createConstantInt(14);
2248                         shuffle[6] = Nucleus::createConstantInt(7);
2249                         shuffle[7] = Nucleus::createConstantInt(15);
2250
2251                         Value *packed = Nucleus::createShuffleVector(x.value, y.value, V(Nucleus::createConstantVector(shuffle, 8)));
2252
2253                         return RValue<Short4>(Nucleus::createBitCast(packed, Short4::getType()));
2254                 }
2255         }
2256
2257         RValue<Int> SignMask(RValue<Byte8> x)
2258         {
2259                 return x86::pmovmskb(x);
2260         }
2261
2262 //      RValue<Byte8> CmpGT(RValue<Byte8> x, RValue<Byte8> y)
2263 //      {
2264 //              return x86::pcmpgtb(x, y);   // FIXME: Signedness
2265 //      }
2266
2267         RValue<Byte8> CmpEQ(RValue<Byte8> x, RValue<Byte8> y)
2268         {
2269                 return x86::pcmpeqb(x, y);
2270         }
2271
2272         Type *Byte8::getType()
2273         {
2274                 if(CPUID::supportsMMX2())
2275                 {
2276                         return MMX::getType();
2277                 }
2278                 else
2279                 {
2280                         return T(VectorType::get(Byte::getType(), 8));
2281                 }
2282         }
2283
2284         SByte8::SByte8()
2285         {
2286         //      xyzw.parent = this;
2287         }
2288
2289         SByte8::SByte8(uint8_t x0, uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, uint8_t x5, uint8_t x6, uint8_t x7)
2290         {
2291         //      xyzw.parent = this;
2292
2293                 Constant *constantVector[8];
2294                 constantVector[0] = Nucleus::createConstantByte(x0);
2295                 constantVector[1] = Nucleus::createConstantByte(x1);
2296                 constantVector[2] = Nucleus::createConstantByte(x2);
2297                 constantVector[3] = Nucleus::createConstantByte(x3);
2298                 constantVector[4] = Nucleus::createConstantByte(x4);
2299                 constantVector[5] = Nucleus::createConstantByte(x5);
2300                 constantVector[6] = Nucleus::createConstantByte(x6);
2301                 constantVector[7] = Nucleus::createConstantByte(x7);
2302                 Value *vector = V(Nucleus::createConstantVector(constantVector, 8));
2303
2304                 storeValue(Nucleus::createBitCast(vector, getType()));
2305         }
2306
2307         SByte8::SByte8(int64_t x)
2308         {
2309         //      xyzw.parent = this;
2310
2311                 Constant *constantVector[8];
2312                 constantVector[0] = Nucleus::createConstantByte((unsigned char)(x >>  0));
2313                 constantVector[1] = Nucleus::createConstantByte((unsigned char)(x >>  8));
2314                 constantVector[2] = Nucleus::createConstantByte((unsigned char)(x >> 16));
2315                 constantVector[3] = Nucleus::createConstantByte((unsigned char)(x >> 24));
2316                 constantVector[4] = Nucleus::createConstantByte((unsigned char)(x >> 32));
2317                 constantVector[5] = Nucleus::createConstantByte((unsigned char)(x >> 40));
2318                 constantVector[6] = Nucleus::createConstantByte((unsigned char)(x >> 48));
2319                 constantVector[7] = Nucleus::createConstantByte((unsigned char)(x >> 56));
2320                 Value *vector = V(Nucleus::createConstantVector(constantVector, 8));
2321
2322                 storeValue(Nucleus::createBitCast(vector, getType()));
2323         }
2324
2325         SByte8::SByte8(RValue<SByte8> rhs)
2326         {
2327         //      xyzw.parent = this;
2328
2329                 storeValue(rhs.value);
2330         }
2331
2332         SByte8::SByte8(const SByte8 &rhs)
2333         {
2334         //      xyzw.parent = this;
2335
2336                 Value *value = rhs.loadValue();
2337                 storeValue(value);
2338         }
2339
2340         SByte8::SByte8(const Reference<SByte8> &rhs)
2341         {
2342         //      xyzw.parent = this;
2343
2344                 Value *value = rhs.loadValue();
2345                 storeValue(value);
2346         }
2347
2348         RValue<SByte8> SByte8::operator=(RValue<SByte8> rhs) const
2349         {
2350                 storeValue(rhs.value);
2351
2352                 return rhs;
2353         }
2354
2355         RValue<SByte8> SByte8::operator=(const SByte8 &rhs) const
2356         {
2357                 Value *value = rhs.loadValue();
2358                 storeValue(value);
2359
2360                 return RValue<SByte8>(value);
2361         }
2362
2363         RValue<SByte8> SByte8::operator=(const Reference<SByte8> &rhs) const
2364         {
2365                 Value *value = rhs.loadValue();
2366                 storeValue(value);
2367
2368                 return RValue<SByte8>(value);
2369         }
2370
2371         RValue<SByte8> operator+(RValue<SByte8> lhs, RValue<SByte8> rhs)
2372         {
2373                 if(CPUID::supportsMMX2())
2374                 {
2375                         return As<SByte8>(x86::paddb(As<Byte8>(lhs), As<Byte8>(rhs)));
2376                 }
2377                 else
2378                 {
2379                         return RValue<SByte8>(Nucleus::createAdd(lhs.value, rhs.value));
2380                 }
2381         }
2382
2383         RValue<SByte8> operator-(RValue<SByte8> lhs, RValue<SByte8> rhs)
2384         {
2385                 if(CPUID::supportsMMX2())
2386                 {
2387                         return As<SByte8>(x86::psubb(As<Byte8>(lhs), As<Byte8>(rhs)));
2388                 }
2389                 else
2390                 {
2391                         return RValue<SByte8>(Nucleus::createSub(lhs.value, rhs.value));
2392                 }
2393         }
2394
2395 //      RValue<SByte8> operator*(RValue<SByte8> lhs, RValue<SByte8> rhs)
2396 //      {
2397 //              return RValue<SByte8>(Nucleus::createMul(lhs.value, rhs.value));
2398 //      }
2399
2400 //      RValue<SByte8> operator/(RValue<SByte8> lhs, RValue<SByte8> rhs)
2401 //      {
2402 //              return RValue<SByte8>(Nucleus::createSDiv(lhs.value, rhs.value));
2403 //      }
2404
2405 //      RValue<SByte8> operator%(RValue<SByte8> lhs, RValue<SByte8> rhs)
2406 //      {
2407 //              return RValue<SByte8>(Nucleus::createSRem(lhs.value, rhs.value));
2408 //      }
2409
2410         RValue<SByte8> operator&(RValue<SByte8> lhs, RValue<SByte8> rhs)
2411         {
2412                 return RValue<SByte8>(Nucleus::createAnd(lhs.value, rhs.value));
2413         }
2414
2415         RValue<SByte8> operator|(RValue<SByte8> lhs, RValue<SByte8> rhs)
2416         {
2417                 return RValue<SByte8>(Nucleus::createOr(lhs.value, rhs.value));
2418         }
2419
2420         RValue<SByte8> operator^(RValue<SByte8> lhs, RValue<SByte8> rhs)
2421         {
2422                 return RValue<SByte8>(Nucleus::createXor(lhs.value, rhs.value));
2423         }
2424
2425 //      RValue<SByte8> operator<<(RValue<SByte8> lhs, unsigned char rhs)
2426 //      {
2427 //              return RValue<SByte8>(Nucleus::createShl(lhs.value, rhs.value));
2428 //      }
2429
2430 //      RValue<SByte8> operator>>(RValue<SByte8> lhs, unsigned char rhs)
2431 //      {
2432 //              return RValue<SByte8>(Nucleus::createAShr(lhs.value, rhs.value));
2433 //      }
2434
2435         RValue<SByte8> operator+=(const SByte8 &lhs, RValue<SByte8> rhs)
2436         {
2437                 return lhs = lhs + rhs;
2438         }
2439
2440         RValue<SByte8> operator-=(const SByte8 &lhs, RValue<SByte8> rhs)
2441         {
2442                 return lhs = lhs - rhs;
2443         }
2444
2445 //      RValue<SByte8> operator*=(const SByte8 &lhs, RValue<SByte8> rhs)
2446 //      {
2447 //              return lhs = lhs * rhs;
2448 //      }
2449
2450 //      RValue<SByte8> operator/=(const SByte8 &lhs, RValue<SByte8> rhs)
2451 //      {
2452 //              return lhs = lhs / rhs;
2453 //      }
2454
2455 //      RValue<SByte8> operator%=(const SByte8 &lhs, RValue<SByte8> rhs)
2456 //      {
2457 //              return lhs = lhs % rhs;
2458 //      }
2459
2460         RValue<SByte8> operator&=(const SByte8 &lhs, RValue<SByte8> rhs)
2461         {
2462                 return lhs = lhs & rhs;
2463         }
2464
2465         RValue<SByte8> operator|=(const SByte8 &lhs, RValue<SByte8> rhs)
2466         {
2467                 return lhs = lhs | rhs;
2468         }
2469
2470         RValue<SByte8> operator^=(const SByte8 &lhs, RValue<SByte8> rhs)
2471         {
2472                 return lhs = lhs ^ rhs;
2473         }
2474
2475 //      RValue<SByte8> operator<<=(const SByte8 &lhs, RValue<SByte8> rhs)
2476 //      {
2477 //              return lhs = lhs << rhs;
2478 //      }
2479
2480 //      RValue<SByte8> operator>>=(const SByte8 &lhs, RValue<SByte8> rhs)
2481 //      {
2482 //              return lhs = lhs >> rhs;
2483 //      }
2484
2485 //      RValue<SByte8> operator+(RValue<SByte8> val)
2486 //      {
2487 //              return val;
2488 //      }
2489
2490 //      RValue<SByte8> operator-(RValue<SByte8> val)
2491 //      {
2492 //              return RValue<SByte8>(Nucleus::createNeg(val.value));
2493 //      }
2494
2495         RValue<SByte8> operator~(RValue<SByte8> val)
2496         {
2497                 if(CPUID::supportsMMX2())
2498                 {
2499                         return val ^ SByte8(0xFFFFFFFFFFFFFFFF);
2500                 }
2501                 else
2502                 {
2503                         return RValue<SByte8>(Nucleus::createNot(val.value));
2504                 }
2505         }
2506
2507         RValue<SByte8> AddSat(RValue<SByte8> x, RValue<SByte8> y)
2508         {
2509                 return x86::paddsb(x, y);
2510         }
2511
2512         RValue<SByte8> SubSat(RValue<SByte8> x, RValue<SByte8> y)
2513         {
2514                 return x86::psubsb(x, y);
2515         }
2516
2517         RValue<Short4> UnpackLow(RValue<SByte8> x, RValue<SByte8> y)
2518         {
2519                 if(CPUID::supportsMMX2())
2520                 {
2521                         return As<Short4>(x86::punpcklbw(As<Byte8>(x), As<Byte8>(y)));
2522                 }
2523                 else
2524                 {
2525                         Constant *shuffle[8];
2526                         shuffle[0] = Nucleus::createConstantInt(0);
2527                         shuffle[1] = Nucleus::createConstantInt(8);
2528                         shuffle[2] = Nucleus::createConstantInt(1);
2529                         shuffle[3] = Nucleus::createConstantInt(9);
2530                         shuffle[4] = Nucleus::createConstantInt(2);
2531                         shuffle[5] = Nucleus::createConstantInt(10);
2532                         shuffle[6] = Nucleus::createConstantInt(3);
2533                         shuffle[7] = Nucleus::createConstantInt(11);
2534
2535                         Value *packed = Nucleus::createShuffleVector(x.value, y.value, V(Nucleus::createConstantVector(shuffle, 8)));
2536
2537                         return RValue<Short4>(Nucleus::createBitCast(packed, Short4::getType()));
2538                 }
2539         }
2540
2541         RValue<Short4> UnpackHigh(RValue<SByte8> x, RValue<SByte8> y)
2542         {
2543                 if(CPUID::supportsMMX2())
2544                 {
2545                         return As<Short4>(x86::punpckhbw(As<Byte8>(x), As<Byte8>(y)));
2546                 }
2547                 else
2548                 {
2549                         Constant *shuffle[8];
2550                         shuffle[0] = Nucleus::createConstantInt(4);
2551                         shuffle[1] = Nucleus::createConstantInt(12);
2552                         shuffle[2] = Nucleus::createConstantInt(5);
2553                         shuffle[3] = Nucleus::createConstantInt(13);
2554                         shuffle[4] = Nucleus::createConstantInt(6);
2555                         shuffle[5] = Nucleus::createConstantInt(14);
2556                         shuffle[6] = Nucleus::createConstantInt(7);
2557                         shuffle[7] = Nucleus::createConstantInt(15);
2558
2559                         Value *packed = Nucleus::createShuffleVector(x.value, y.value, V(Nucleus::createConstantVector(shuffle, 8)));
2560
2561                         return RValue<Short4>(Nucleus::createBitCast(packed, Short4::getType()));
2562                 }
2563         }
2564
2565         RValue<Int> SignMask(RValue<SByte8> x)
2566         {
2567                 return x86::pmovmskb(As<Byte8>(x));
2568         }
2569
2570         RValue<Byte8> CmpGT(RValue<SByte8> x, RValue<SByte8> y)
2571         {
2572                 return x86::pcmpgtb(x, y);
2573         }
2574
2575         RValue<Byte8> CmpEQ(RValue<SByte8> x, RValue<SByte8> y)
2576         {
2577                 return x86::pcmpeqb(As<Byte8>(x), As<Byte8>(y));
2578         }
2579
2580         Type *SByte8::getType()
2581         {
2582                 if(CPUID::supportsMMX2())
2583                 {
2584                         return MMX::getType();
2585                 }
2586                 else
2587                 {
2588                         return T(VectorType::get(SByte::getType(), 8));
2589                 }
2590         }
2591
2592         Byte16::Byte16(RValue<Byte16> rhs)
2593         {
2594         //      xyzw.parent = this;
2595
2596                 storeValue(rhs.value);
2597         }
2598
2599         Byte16::Byte16(const Byte16 &rhs)
2600         {
2601         //      xyzw.parent = this;
2602
2603                 Value *value = rhs.loadValue();
2604                 storeValue(value);
2605         }
2606
2607         Byte16::Byte16(const Reference<Byte16> &rhs)
2608         {
2609         //      xyzw.parent = this;
2610
2611                 Value *value = rhs.loadValue();
2612                 storeValue(value);
2613         }
2614
2615         RValue<Byte16> Byte16::operator=(RValue<Byte16> rhs) const
2616         {
2617                 storeValue(rhs.value);
2618
2619                 return rhs;
2620         }
2621
2622         RValue<Byte16> Byte16::operator=(const Byte16 &rhs) const
2623         {
2624                 Value *value = rhs.loadValue();
2625                 storeValue(value);
2626
2627                 return RValue<Byte16>(value);
2628         }
2629
2630         RValue<Byte16> Byte16::operator=(const Reference<Byte16> &rhs) const
2631         {
2632                 Value *value = rhs.loadValue();
2633                 storeValue(value);
2634
2635                 return RValue<Byte16>(value);
2636         }
2637
2638         Type *Byte16::getType()
2639         {
2640                 return T(VectorType::get(Byte::getType(), 16));
2641         }
2642
2643         Type *SByte16::getType()
2644         {
2645                 return T( VectorType::get(SByte::getType(), 16));
2646         }
2647
2648         Short4::Short4(RValue<Int> cast)
2649         {
2650                 Value *extend = Nucleus::createZExt(cast.value, Long::getType());
2651                 Value *swizzle = Swizzle(RValue<Short4>(extend), 0x00).value;
2652
2653                 storeValue(swizzle);
2654         }
2655
2656         Short4::Short4(RValue<Int4> cast)
2657         {
2658                 Value *short8 = Nucleus::createBitCast(cast.value, Short8::getType());
2659
2660                 #if 0   // FIXME: Check codegen (pshuflw phshufhw pshufd)
2661                         Constant *pack[8];
2662                         pack[0] = Nucleus::createConstantInt(0);
2663                         pack[1] = Nucleus::createConstantInt(2);
2664                         pack[2] = Nucleus::createConstantInt(4);
2665                         pack[3] = Nucleus::createConstantInt(6);
2666
2667                         Value *short4 = Nucleus::createShuffleVector(short8, short8, Nucleus::createConstantVector(pack, 4));
2668                 #else
2669                         Value *packed;
2670
2671                         // FIXME: Use Swizzle<Short8>
2672                         if(!CPUID::supportsSSSE3())
2673                         {
2674                                 Constant *pshuflw[8];
2675                                 pshuflw[0] = Nucleus::createConstantInt(0);
2676                                 pshuflw[1] = Nucleus::createConstantInt(2);
2677                                 pshuflw[2] = Nucleus::createConstantInt(0);
2678                                 pshuflw[3] = Nucleus::createConstantInt(2);
2679                                 pshuflw[4] = Nucleus::createConstantInt(4);
2680                                 pshuflw[5] = Nucleus::createConstantInt(5);
2681                                 pshuflw[6] = Nucleus::createConstantInt(6);
2682                                 pshuflw[7] = Nucleus::createConstantInt(7);
2683
2684                                 Constant *pshufhw[8];
2685                                 pshufhw[0] = Nucleus::createConstantInt(0);
2686                                 pshufhw[1] = Nucleus::createConstantInt(1);
2687                                 pshufhw[2] = Nucleus::createConstantInt(2);
2688                                 pshufhw[3] = Nucleus::createConstantInt(3);
2689                                 pshufhw[4] = Nucleus::createConstantInt(4);
2690                                 pshufhw[5] = Nucleus::createConstantInt(6);
2691                                 pshufhw[6] = Nucleus::createConstantInt(4);
2692                                 pshufhw[7] = Nucleus::createConstantInt(6);
2693
2694                                 Value *shuffle1 = Nucleus::createShuffleVector(short8, V(UndefValue::get(Short8::getType())), V(Nucleus::createConstantVector(pshuflw, 8)));
2695                                 Value *shuffle2 = Nucleus::createShuffleVector(shuffle1, V(UndefValue::get(Short8::getType())), V(Nucleus::createConstantVector(pshufhw, 8)));
2696                                 Value *int4 = Nucleus::createBitCast(shuffle2, Int4::getType());
2697                                 packed = Nucleus::createSwizzle(int4, 0x88);
2698                         }
2699                         else
2700                         {
2701                                 Constant *pshufb[16];
2702                                 pshufb[0] = Nucleus::createConstantInt(0);
2703                                 pshufb[1] = Nucleus::createConstantInt(1);
2704                                 pshufb[2] = Nucleus::createConstantInt(4);
2705                                 pshufb[3] = Nucleus::createConstantInt(5);
2706                                 pshufb[4] = Nucleus::createConstantInt(8);
2707                                 pshufb[5] = Nucleus::createConstantInt(9);
2708                                 pshufb[6] = Nucleus::createConstantInt(12);
2709                                 pshufb[7] = Nucleus::createConstantInt(13);
2710                                 pshufb[8] = Nucleus::createConstantInt(0);
2711                                 pshufb[9] = Nucleus::createConstantInt(1);
2712                                 pshufb[10] = Nucleus::createConstantInt(4);
2713                                 pshufb[11] = Nucleus::createConstantInt(5);
2714                                 pshufb[12] = Nucleus::createConstantInt(8);
2715                                 pshufb[13] = Nucleus::createConstantInt(9);
2716                                 pshufb[14] = Nucleus::createConstantInt(12);
2717                                 pshufb[15] = Nucleus::createConstantInt(13);
2718
2719                                 Value *byte16 = Nucleus::createBitCast(cast.value, Byte16::getType());
2720                                 packed = Nucleus::createShuffleVector(byte16, V(UndefValue::get(Byte16::getType())), V(Nucleus::createConstantVector(pshufb, 16)));
2721                         }
2722
2723                         #if 0   // FIXME: No optimal instruction selection
2724                                 Value *qword2 = Nucleus::createBitCast(packed, Long2::getType());
2725                                 Value *element = Nucleus::createExtractElement(qword2, 0);
2726                                 Value *short4 = Nucleus::createBitCast(element, Short4::getType());
2727                         #else   // FIXME: Requires SSE
2728                                 Value *int2 = RValue<Int2>(Int2(RValue<Int4>(packed))).value;
2729                                 Value *short4 = Nucleus::createBitCast(int2, Short4::getType());
2730                         #endif
2731                 #endif
2732
2733                 storeValue(short4);
2734         }
2735
2736 //      Short4::Short4(RValue<Float> cast)
2737 //      {
2738 //      }
2739
2740         Short4::Short4(RValue<Float4> cast)
2741         {
2742                 Int4 v4i32 = Int4(cast);
2743                 v4i32 = As<Int4>(x86::packssdw(v4i32, v4i32));
2744
2745                 storeValue(As<Short4>(Int2(v4i32)).value);
2746         }
2747
2748         Short4::Short4()
2749         {
2750         //      xyzw.parent = this;
2751         }
2752
2753         Short4::Short4(short xyzw)
2754         {
2755                 //      xyzw.parent = this;
2756
2757                 Constant *constantVector[4];
2758                 constantVector[0] = Nucleus::createConstantShort(xyzw);
2759                 constantVector[1] = Nucleus::createConstantShort(xyzw);
2760                 constantVector[2] = Nucleus::createConstantShort(xyzw);
2761                 constantVector[3] = Nucleus::createConstantShort(xyzw);
2762                 Value *vector = V(Nucleus::createConstantVector(constantVector, 4));
2763
2764                 storeValue(Nucleus::createBitCast(vector, getType()));
2765         }
2766
2767         Short4::Short4(short x, short y, short z, short w)
2768         {
2769         //      xyzw.parent = this;
2770
2771                 Constant *constantVector[4];
2772                 constantVector[0] = Nucleus::createConstantShort(x);
2773                 constantVector[1] = Nucleus::createConstantShort(y);
2774                 constantVector[2] = Nucleus::createConstantShort(z);
2775                 constantVector[3] = Nucleus::createConstantShort(w);
2776                 Value *vector = V(Nucleus::createConstantVector(constantVector, 4));
2777
2778                 storeValue(Nucleus::createBitCast(vector, getType()));
2779         }
2780
2781         Short4::Short4(RValue<Short4> rhs)
2782         {
2783         //      xyzw.parent = this;
2784
2785                 storeValue(rhs.value);
2786         }
2787
2788         Short4::Short4(const Short4 &rhs)
2789         {
2790         //      xyzw.parent = this;
2791
2792                 Value *value = rhs.loadValue();
2793                 storeValue(value);
2794         }
2795
2796         Short4::Short4(const Reference<Short4> &rhs)
2797         {
2798         //      xyzw.parent = this;
2799
2800                 Value *value = rhs.loadValue();
2801                 storeValue(value);
2802         }
2803
2804         Short4::Short4(RValue<UShort4> rhs)
2805         {
2806         //      xyzw.parent = this;
2807
2808                 storeValue(rhs.value);
2809         }
2810
2811         Short4::Short4(const UShort4 &rhs)
2812         {
2813         //      xyzw.parent = this;
2814
2815                 storeValue(rhs.loadValue());
2816         }
2817
2818         Short4::Short4(const Reference<UShort4> &rhs)
2819         {
2820         //      xyzw.parent = this;
2821
2822                 storeValue(rhs.loadValue());
2823         }
2824
2825         RValue<Short4> Short4::operator=(RValue<Short4> rhs) const
2826         {
2827                 storeValue(rhs.value);
2828
2829                 return rhs;
2830         }
2831
2832         RValue<Short4> Short4::operator=(const Short4 &rhs) const
2833         {
2834                 Value *value = rhs.loadValue();
2835                 storeValue(value);
2836
2837                 return RValue<Short4>(value);
2838         }
2839
2840         RValue<Short4> Short4::operator=(const Reference<Short4> &rhs) const
2841         {
2842                 Value *value = rhs.loadValue();
2843                 storeValue(value);
2844
2845                 return RValue<Short4>(value);
2846         }
2847
2848         RValue<Short4> Short4::operator=(RValue<UShort4> rhs) const
2849         {
2850                 storeValue(rhs.value);
2851
2852                 return RValue<Short4>(rhs);
2853         }
2854
2855         RValue<Short4> Short4::operator=(const UShort4 &rhs) const
2856         {
2857                 Value *value = rhs.loadValue();
2858                 storeValue(value);
2859
2860                 return RValue<Short4>(value);
2861         }
2862
2863         RValue<Short4> Short4::operator=(const Reference<UShort4> &rhs) const
2864         {
2865                 Value *value = rhs.loadValue();
2866                 storeValue(value);
2867
2868                 return RValue<Short4>(value);
2869         }
2870
2871         RValue<Short4> operator+(RValue<Short4> lhs, RValue<Short4> rhs)
2872         {
2873                 if(CPUID::supportsMMX2())
2874                 {
2875                         return x86::paddw(lhs, rhs);
2876                 }
2877                 else
2878                 {
2879                         return RValue<Short4>(Nucleus::createAdd(lhs.value, rhs.value));
2880                 }
2881         }
2882
2883         RValue<Short4> operator-(RValue<Short4> lhs, RValue<Short4> rhs)
2884         {
2885                 if(CPUID::supportsMMX2())
2886                 {
2887                         return x86::psubw(lhs, rhs);
2888                 }
2889                 else
2890                 {
2891                         return RValue<Short4>(Nucleus::createSub(lhs.value, rhs.value));
2892                 }
2893         }
2894
2895         RValue<Short4> operator*(RValue<Short4> lhs, RValue<Short4> rhs)
2896         {
2897                 if(CPUID::supportsMMX2())
2898                 {
2899                         return x86::pmullw(lhs, rhs);
2900                 }
2901                 else
2902                 {
2903                         return RValue<Short4>(Nucleus::createMul(lhs.value, rhs.value));
2904                 }
2905         }
2906
2907 //      RValue<Short4> operator/(RValue<Short4> lhs, RValue<Short4> rhs)
2908 //      {
2909 //              return RValue<Short4>(Nucleus::createSDiv(lhs.value, rhs.value));
2910 //      }
2911
2912 //      RValue<Short4> operator%(RValue<Short4> lhs, RValue<Short4> rhs)
2913 //      {
2914 //              return RValue<Short4>(Nucleus::createSRem(lhs.value, rhs.value));
2915 //      }
2916
2917         RValue<Short4> operator&(RValue<Short4> lhs, RValue<Short4> rhs)
2918         {
2919                 if(CPUID::supportsMMX2())
2920                 {
2921                         return x86::pand(lhs, rhs);
2922                 }
2923                 else
2924                 {
2925                         return RValue<Short4>(Nucleus::createAnd(lhs.value, rhs.value));
2926                 }
2927         }
2928
2929         RValue<Short4> operator|(RValue<Short4> lhs, RValue<Short4> rhs)
2930         {
2931                 if(CPUID::supportsMMX2())
2932                 {
2933                         return x86::por(lhs, rhs);
2934                 }
2935                 else
2936                 {
2937                         return RValue<Short4>(Nucleus::createOr(lhs.value, rhs.value));
2938                 }
2939         }
2940
2941         RValue<Short4> operator^(RValue<Short4> lhs, RValue<Short4> rhs)
2942         {
2943                 if(CPUID::supportsMMX2())
2944                 {
2945                         return x86::pxor(lhs, rhs);
2946                 }
2947                 else
2948                 {
2949                         return RValue<Short4>(Nucleus::createXor(lhs.value, rhs.value));
2950                 }
2951         }
2952
2953         RValue<Short4> operator<<(RValue<Short4> lhs, unsigned char rhs)
2954         {
2955         //      return RValue<Short4>(Nucleus::createShl(lhs.value, rhs.value));
2956
2957                 return x86::psllw(lhs, rhs);
2958         }
2959
2960         RValue<Short4> operator>>(RValue<Short4> lhs, unsigned char rhs)
2961         {
2962         //      return RValue<Short4>(Nucleus::createAShr(lhs.value, rhs.value));
2963
2964                 return x86::psraw(lhs, rhs);
2965         }
2966
2967         RValue<Short4> operator<<(RValue<Short4> lhs, RValue<Long1> rhs)
2968         {
2969         //      return RValue<Short4>(Nucleus::createShl(lhs.value, rhs.value));
2970
2971                 return x86::psllw(lhs, rhs);
2972         }
2973
2974         RValue<Short4> operator>>(RValue<Short4> lhs, RValue<Long1> rhs)
2975         {
2976         //      return RValue<Short4>(Nucleus::createAShr(lhs.value, rhs.value));
2977
2978                 return x86::psraw(lhs, rhs);
2979         }
2980
2981         RValue<Short4> operator+=(const Short4 &lhs, RValue<Short4> rhs)
2982         {
2983                 return lhs = lhs + rhs;
2984         }
2985
2986         RValue<Short4> operator-=(const Short4 &lhs, RValue<Short4> rhs)
2987         {
2988                 return lhs = lhs - rhs;
2989         }
2990
2991         RValue<Short4> operator*=(const Short4 &lhs, RValue<Short4> rhs)
2992         {
2993                 return lhs = lhs * rhs;
2994         }
2995
2996 //      RValue<Short4> operator/=(const Short4 &lhs, RValue<Short4> rhs)
2997 //      {
2998 //              return lhs = lhs / rhs;
2999 //      }
3000
3001 //      RValue<Short4> operator%=(const Short4 &lhs, RValue<Short4> rhs)
3002 //      {
3003 //              return lhs = lhs % rhs;
3004 //      }
3005
3006         RValue<Short4> operator&=(const Short4 &lhs, RValue<Short4> rhs)
3007         {
3008                 return lhs = lhs & rhs;
3009         }
3010
3011         RValue<Short4> operator|=(const Short4 &lhs, RValue<Short4> rhs)
3012         {
3013                 return lhs = lhs | rhs;
3014         }
3015
3016         RValue<Short4> operator^=(const Short4 &lhs, RValue<Short4> rhs)
3017         {
3018                 return lhs = lhs ^ rhs;
3019         }
3020
3021         RValue<Short4> operator<<=(const Short4 &lhs, unsigned char rhs)
3022         {
3023                 return lhs = lhs << rhs;
3024         }
3025
3026         RValue<Short4> operator>>=(const Short4 &lhs, unsigned char rhs)
3027         {
3028                 return lhs = lhs >> rhs;
3029         }
3030
3031         RValue<Short4> operator<<=(const Short4 &lhs, RValue<Long1> rhs)
3032         {
3033                 return lhs = lhs << rhs;
3034         }
3035
3036         RValue<Short4> operator>>=(const Short4 &lhs, RValue<Long1> rhs)
3037         {
3038                 return lhs = lhs >> rhs;
3039         }
3040
3041 //      RValue<Short4> operator+(RValue<Short4> val)
3042 //      {
3043 //              return val;
3044 //      }
3045
3046         RValue<Short4> operator-(RValue<Short4> val)
3047         {
3048                 if(CPUID::supportsMMX2())
3049                 {
3050                         return Short4(0, 0, 0, 0) - val;
3051                 }
3052                 else
3053                 {
3054                         return RValue<Short4>(Nucleus::createNeg(val.value));
3055                 }
3056         }
3057
3058         RValue<Short4> operator~(RValue<Short4> val)
3059         {
3060                 if(CPUID::supportsMMX2())
3061                 {
3062                         return val ^ Short4(0xFFFFu, 0xFFFFu, 0xFFFFu, 0xFFFFu);
3063                 }
3064                 else
3065                 {
3066                         return RValue<Short4>(Nucleus::createNot(val.value));
3067                 }
3068         }
3069
3070         RValue<Short4> RoundShort4(RValue<Float4> cast)
3071         {
3072                 RValue<Int4> v4i32 = x86::cvtps2dq(cast);
3073                 RValue<Short8> v8i16 = x86::packssdw(v4i32, v4i32);
3074
3075                 return As<Short4>(Int2(As<Int4>(v8i16)));
3076         }
3077
3078         RValue<Short4> Max(RValue<Short4> x, RValue<Short4> y)
3079         {
3080                 return x86::pmaxsw(x, y);
3081         }
3082
3083         RValue<Short4> Min(RValue<Short4> x, RValue<Short4> y)
3084         {
3085                 return x86::pminsw(x, y);
3086         }
3087
3088         RValue<Short4> AddSat(RValue<Short4> x, RValue<Short4> y)
3089         {
3090                 return x86::paddsw(x, y);
3091         }
3092
3093         RValue<Short4> SubSat(RValue<Short4> x, RValue<Short4> y)
3094         {
3095                 return x86::psubsw(x, y);
3096         }
3097
3098         RValue<Short4> MulHigh(RValue<Short4> x, RValue<Short4> y)
3099         {
3100                 return x86::pmulhw(x, y);
3101         }
3102
3103         RValue<Int2> MulAdd(RValue<Short4> x, RValue<Short4> y)
3104         {
3105                 return x86::pmaddwd(x, y);
3106         }
3107
3108         RValue<SByte8> Pack(RValue<Short4> x, RValue<Short4> y)
3109         {
3110                 return x86::packsswb(x, y);
3111         }
3112
3113         RValue<Int2> UnpackLow(RValue<Short4> x, RValue<Short4> y)
3114         {
3115                 if(CPUID::supportsMMX2())
3116                 {
3117                         return x86::punpcklwd(x, y);
3118                 }
3119                 else
3120                 {
3121                         Constant *shuffle[4];
3122                         shuffle[0] = Nucleus::createConstantInt(0);
3123                         shuffle[1] = Nucleus::createConstantInt(4);
3124                         shuffle[2] = Nucleus::createConstantInt(1);
3125                         shuffle[3] = Nucleus::createConstantInt(5);
3126
3127                         Value *packed = Nucleus::createShuffleVector(x.value, y.value, V(Nucleus::createConstantVector(shuffle, 4)));
3128
3129                         return RValue<Int2>(Nucleus::createBitCast(packed, Int2::getType()));
3130                 }
3131         }
3132
3133         RValue<Int2> UnpackHigh(RValue<Short4> x, RValue<Short4> y)
3134         {
3135                 if(CPUID::supportsMMX2())
3136                 {
3137                         return x86::punpckhwd(x, y);
3138                 }
3139                 else
3140                 {
3141                         Constant *shuffle[4];
3142                         shuffle[0] = Nucleus::createConstantInt(2);
3143                         shuffle[1] = Nucleus::createConstantInt(6);
3144                         shuffle[2] = Nucleus::createConstantInt(3);
3145                         shuffle[3] = Nucleus::createConstantInt(7);
3146
3147                         Value *packed = Nucleus::createShuffleVector(x.value, y.value, V(Nucleus::createConstantVector(shuffle, 4)));
3148
3149                         return RValue<Int2>(Nucleus::createBitCast(packed, Int2::getType()));
3150                 }
3151         }
3152
3153         RValue<Short4> Swizzle(RValue<Short4> x, unsigned char select)
3154         {
3155                 if(CPUID::supportsMMX2())
3156                 {
3157                         return x86::pshufw(x, select);
3158                 }
3159                 else
3160                 {
3161                         return RValue<Short4>(Nucleus::createSwizzle(x.value, select));
3162                 }
3163         }
3164
3165         RValue<Short4> Insert(RValue<Short4> val, RValue<Short> element, int i)
3166         {
3167                 if(CPUID::supportsMMX2())
3168                 {
3169                         return x86::pinsrw(val, Int(element), i);
3170                 }
3171                 else
3172                 {
3173                         return RValue<Short4>(Nucleus::createInsertElement(val.value, element.value, i));
3174                 }
3175         }
3176
3177         RValue<Short> Extract(RValue<Short4> val, int i)
3178         {
3179                 if(CPUID::supportsMMX2())
3180                 {
3181                         return Short(x86::pextrw(val, i));
3182                 }
3183                 else
3184                 {
3185                         return RValue<Short>(Nucleus::createExtractElement(val.value, i));
3186                 }
3187         }
3188
3189         RValue<Short4> CmpGT(RValue<Short4> x, RValue<Short4> y)
3190         {
3191                 return x86::pcmpgtw(x, y);
3192         }
3193
3194         RValue<Short4> CmpEQ(RValue<Short4> x, RValue<Short4> y)
3195         {
3196                 return x86::pcmpeqw(x, y);
3197         }
3198
3199         Type *Short4::getType()
3200         {
3201                 if(CPUID::supportsMMX2())
3202                 {
3203                         return MMX::getType();
3204                 }
3205                 else
3206                 {
3207                         return T(VectorType::get(Short::getType(), 4));
3208                 }
3209         }
3210
3211         UShort4::UShort4(RValue<Int4> cast)
3212         {
3213                 *this = Short4(cast);
3214         }
3215
3216         UShort4::UShort4(RValue<Float4> cast, bool saturate)
3217         {
3218                 Float4 sat;
3219
3220                 if(saturate)
3221                 {
3222                         if(CPUID::supportsSSE4_1())
3223                         {
3224                                 sat = Min(cast, Float4(0xFFFF));   // packusdw takes care of 0x0000 saturation
3225                         }
3226                         else
3227                         {
3228                                 sat = Max(Min(cast, Float4(0xFFFF)), Float4(0x0000));
3229                         }
3230                 }
3231                 else
3232                 {
3233                         sat = cast;
3234                 }
3235
3236                 Int4 int4(sat);
3237
3238                 if(!saturate || !CPUID::supportsSSE4_1())
3239                 {
3240                         *this = Short4(Int4(int4));
3241                 }
3242                 else
3243                 {
3244                         *this = As<Short4>(Int2(As<Int4>(x86::packusdw(As<UInt4>(int4), As<UInt4>(int4)))));
3245                 }
3246         }
3247
3248         UShort4::UShort4()
3249         {
3250         //      xyzw.parent = this;
3251         }
3252
3253         UShort4::UShort4(unsigned short xyzw)
3254         {
3255                 //      xyzw.parent = this;
3256
3257                 Constant *constantVector[4];
3258                 constantVector[0] = Nucleus::createConstantShort(xyzw);
3259                 constantVector[1] = Nucleus::createConstantShort(xyzw);
3260                 constantVector[2] = Nucleus::createConstantShort(xyzw);
3261                 constantVector[3] = Nucleus::createConstantShort(xyzw);
3262                 Value *vector = V(Nucleus::createConstantVector(constantVector, 4));
3263
3264                 storeValue(Nucleus::createBitCast(vector, getType()));
3265         }
3266
3267         UShort4::UShort4(unsigned short x, unsigned short y, unsigned short z, unsigned short w)
3268         {
3269         //      xyzw.parent = this;
3270
3271                 Constant *constantVector[4];
3272                 constantVector[0] = Nucleus::createConstantShort(x);
3273                 constantVector[1] = Nucleus::createConstantShort(y);
3274                 constantVector[2] = Nucleus::createConstantShort(z);
3275                 constantVector[3] = Nucleus::createConstantShort(w);
3276                 Value *vector = V(Nucleus::createConstantVector(constantVector, 4));
3277
3278                 storeValue(Nucleus::createBitCast(vector, getType()));
3279         }
3280
3281         UShort4::UShort4(RValue<UShort4> rhs)
3282         {
3283         //      xyzw.parent = this;
3284
3285                 storeValue(rhs.value);
3286         }
3287
3288         UShort4::UShort4(const UShort4 &rhs)
3289         {
3290         //      xyzw.parent = this;
3291
3292                 Value *value = rhs.loadValue();
3293                 storeValue(value);
3294         }
3295
3296         UShort4::UShort4(const Reference<UShort4> &rhs)
3297         {
3298         //      xyzw.parent = this;
3299
3300                 Value *value = rhs.loadValue();
3301                 storeValue(value);
3302         }
3303
3304         UShort4::UShort4(RValue<Short4> rhs)
3305         {
3306         //      xyzw.parent = this;
3307
3308                 storeValue(rhs.value);
3309         }
3310
3311         UShort4::UShort4(const Short4 &rhs)
3312         {
3313         //      xyzw.parent = this;
3314
3315                 Value *value = rhs.loadValue();
3316                 storeValue(value);
3317         }
3318
3319         UShort4::UShort4(const Reference<Short4> &rhs)
3320         {
3321         //      xyzw.parent = this;
3322
3323                 Value *value = rhs.loadValue();
3324                 storeValue(value);
3325         }
3326
3327         RValue<UShort4> UShort4::operator=(RValue<UShort4> rhs) const
3328         {
3329                 storeValue(rhs.value);
3330
3331                 return rhs;
3332         }
3333
3334         RValue<UShort4> UShort4::operator=(const UShort4 &rhs) const
3335         {
3336                 Value *value = rhs.loadValue();
3337                 storeValue(value);
3338
3339                 return RValue<UShort4>(value);
3340         }
3341
3342         RValue<UShort4> UShort4::operator=(const Reference<UShort4> &rhs) const
3343         {
3344                 Value *value = rhs.loadValue();
3345                 storeValue(value);
3346
3347                 return RValue<UShort4>(value);
3348         }
3349
3350         RValue<UShort4> UShort4::operator=(RValue<Short4> rhs) const
3351         {
3352                 storeValue(rhs.value);
3353
3354                 return RValue<UShort4>(rhs);
3355         }
3356
3357         RValue<UShort4> UShort4::operator=(const Short4 &rhs) const
3358         {
3359                 Value *value = rhs.loadValue();
3360                 storeValue(value);
3361
3362                 return RValue<UShort4>(value);
3363         }
3364
3365         RValue<UShort4> UShort4::operator=(const Reference<Short4> &rhs) const
3366         {
3367                 Value *value = rhs.loadValue();
3368                 storeValue(value);
3369
3370                 return RValue<UShort4>(value);
3371         }
3372
3373         RValue<UShort4> operator+(RValue<UShort4> lhs, RValue<UShort4> rhs)
3374         {
3375                 if(CPUID::supportsMMX2())
3376                 {
3377                         return As<UShort4>(x86::paddw(As<Short4>(lhs), As<Short4>(rhs)));
3378                 }
3379                 else
3380                 {
3381                         return RValue<UShort4>(Nucleus::createAdd(lhs.value, rhs.value));
3382                 }
3383         }
3384
3385         RValue<UShort4> operator-(RValue<UShort4> lhs, RValue<UShort4> rhs)
3386         {
3387                 if(CPUID::supportsMMX2())
3388                 {
3389                         return As<UShort4>(x86::psubw(As<Short4>(lhs), As<Short4>(rhs)));
3390                 }
3391                 else
3392                 {
3393                         return RValue<UShort4>(Nucleus::createSub(lhs.value, rhs.value));
3394                 }
3395         }
3396
3397         RValue<UShort4> operator*(RValue<UShort4> lhs, RValue<UShort4> rhs)
3398         {
3399                 if(CPUID::supportsMMX2())
3400                 {
3401                         return As<UShort4>(x86::pmullw(As<Short4>(lhs), As<Short4>(rhs)));
3402                 }
3403                 else
3404                 {
3405                         return RValue<UShort4>(Nucleus::createMul(lhs.value, rhs.value));
3406                 }
3407         }
3408
3409         RValue<UShort4> operator<<(RValue<UShort4> lhs, unsigned char rhs)
3410         {
3411         //      return RValue<Short4>(Nucleus::createShl(lhs.value, rhs.value));
3412
3413                 return As<UShort4>(x86::psllw(As<Short4>(lhs), rhs));
3414         }
3415
3416         RValue<UShort4> operator>>(RValue<UShort4> lhs, unsigned char rhs)
3417         {
3418         //      return RValue<Short4>(Nucleus::createLShr(lhs.value, rhs.value));
3419
3420                 return x86::psrlw(lhs, rhs);
3421         }
3422
3423         RValue<UShort4> operator<<(RValue<UShort4> lhs, RValue<Long1> rhs)
3424         {
3425         //      return RValue<Short4>(Nucleus::createShl(lhs.value, rhs.value));
3426
3427                 return As<UShort4>(x86::psllw(As<Short4>(lhs), rhs));
3428         }
3429
3430         RValue<UShort4> operator>>(RValue<UShort4> lhs, RValue<Long1> rhs)
3431         {
3432         //      return RValue<Short4>(Nucleus::createLShr(lhs.value, rhs.value));
3433
3434                 return x86::psrlw(lhs, rhs);
3435         }
3436
3437         RValue<UShort4> operator<<=(const UShort4 &lhs, unsigned char rhs)
3438         {
3439                 return lhs = lhs << rhs;
3440         }
3441
3442         RValue<UShort4> operator>>=(const UShort4 &lhs, unsigned char rhs)
3443         {
3444                 return lhs = lhs >> rhs;
3445         }
3446
3447         RValue<UShort4> operator<<=(const UShort4 &lhs, RValue<Long1> rhs)
3448         {
3449                 return lhs = lhs << rhs;
3450         }
3451
3452         RValue<UShort4> operator>>=(const UShort4 &lhs, RValue<Long1> rhs)
3453         {
3454                 return lhs = lhs >> rhs;
3455         }
3456
3457         RValue<UShort4> operator~(RValue<UShort4> val)
3458         {
3459                 if(CPUID::supportsMMX2())
3460                 {
3461                         return As<UShort4>(As<Short4>(val) ^ Short4(0xFFFFu, 0xFFFFu, 0xFFFFu, 0xFFFFu));
3462                 }
3463                 else
3464                 {
3465                         return RValue<UShort4>(Nucleus::createNot(val.value));
3466                 }
3467         }
3468
3469         RValue<UShort4> Max(RValue<UShort4> x, RValue<UShort4> y)
3470         {
3471                 return RValue<UShort4>(Max(As<Short4>(x) - Short4(0x8000u, 0x8000u, 0x8000u, 0x8000u), As<Short4>(y) - Short4(0x8000u, 0x8000u, 0x8000u, 0x8000u)) + Short4(0x8000u, 0x8000u, 0x8000u, 0x8000u));
3472         }
3473
3474         RValue<UShort4> Min(RValue<UShort4> x, RValue<UShort4> y)
3475         {
3476                 return RValue<UShort4>(Min(As<Short4>(x) - Short4(0x8000u, 0x8000u, 0x8000u, 0x8000u), As<Short4>(y) - Short4(0x8000u, 0x8000u, 0x8000u, 0x8000u)) + Short4(0x8000u, 0x8000u, 0x8000u, 0x8000u));
3477         }
3478
3479         RValue<UShort4> AddSat(RValue<UShort4> x, RValue<UShort4> y)
3480         {
3481                 return x86::paddusw(x, y);
3482         }
3483
3484         RValue<UShort4> SubSat(RValue<UShort4> x, RValue<UShort4> y)
3485         {
3486                 return x86::psubusw(x, y);
3487         }
3488
3489         RValue<UShort4> MulHigh(RValue<UShort4> x, RValue<UShort4> y)
3490         {
3491                 return x86::pmulhuw(x, y);
3492         }
3493
3494         RValue<UShort4> Average(RValue<UShort4> x, RValue<UShort4> y)
3495         {
3496                 return x86::pavgw(x, y);
3497         }
3498
3499         RValue<Byte8> Pack(RValue<UShort4> x, RValue<UShort4> y)
3500         {
3501                 return x86::packuswb(x, y);
3502         }
3503
3504         Type *UShort4::getType()
3505         {
3506                 if(CPUID::supportsMMX2())
3507                 {
3508                         return MMX::getType();
3509                 }
3510                 else
3511                 {
3512                         return T(VectorType::get(UShort::getType(), 4));
3513                 }
3514         }
3515
3516         Short8::Short8(short c0, short c1, short c2, short c3, short c4, short c5, short c6, short c7)
3517         {
3518         //      xyzw.parent = this;
3519
3520                 Constant *constantVector[8];
3521                 constantVector[0] = Nucleus::createConstantShort(c0);
3522                 constantVector[1] = Nucleus::createConstantShort(c1);
3523                 constantVector[2] = Nucleus::createConstantShort(c2);
3524                 constantVector[3] = Nucleus::createConstantShort(c3);
3525                 constantVector[4] = Nucleus::createConstantShort(c4);
3526                 constantVector[5] = Nucleus::createConstantShort(c5);
3527                 constantVector[6] = Nucleus::createConstantShort(c6);
3528                 constantVector[7] = Nucleus::createConstantShort(c7);
3529
3530                 storeValue(Nucleus::createConstantVector(constantVector, 8));
3531         }
3532
3533         Short8::Short8(RValue<Short8> rhs)
3534         {
3535         //      xyzw.parent = this;
3536
3537                 storeValue(rhs.value);
3538         }
3539
3540         Short8::Short8(const Reference<Short8> &rhs)
3541         {
3542         //      xyzw.parent = this;
3543
3544                 Value *value = rhs.loadValue();
3545                 storeValue(value);
3546         }
3547
3548         Short8::Short8(RValue<Short4> lo, RValue<Short4> hi)
3549         {
3550                 Value *loLong = Nucleus::createBitCast(lo.value, Long::getType());
3551                 Value *hiLong = Nucleus::createBitCast(hi.value, Long::getType());
3552
3553                 Value *long2 = V(UndefValue::get(Long2::getType()));
3554                 long2 = Nucleus::createInsertElement(long2, loLong, 0);
3555                 long2 = Nucleus::createInsertElement(long2, hiLong, 1);
3556                 Value *short8 = Nucleus::createBitCast(long2, Short8::getType());
3557
3558                 storeValue(short8);
3559         }
3560
3561         RValue<Short8> operator+(RValue<Short8> lhs, RValue<Short8> rhs)
3562         {
3563                 return RValue<Short8>(Nucleus::createAdd(lhs.value, rhs.value));
3564         }
3565
3566         RValue<Short8> operator&(RValue<Short8> lhs, RValue<Short8> rhs)
3567         {
3568                 return RValue<Short8>(Nucleus::createAnd(lhs.value, rhs.value));
3569         }
3570
3571         RValue<Short8> operator<<(RValue<Short8> lhs, unsigned char rhs)
3572         {
3573                 return x86::psllw(lhs, rhs);   // FIXME: Fallback required
3574         }
3575
3576         RValue<Short8> operator>>(RValue<Short8> lhs, unsigned char rhs)
3577         {
3578                 return x86::psraw(lhs, rhs);   // FIXME: Fallback required
3579         }
3580
3581         RValue<Int4> MulAdd(RValue<Short8> x, RValue<Short8> y)
3582         {
3583                 return x86::pmaddwd(x, y);   // FIXME: Fallback required
3584         }
3585
3586         RValue<Int4> Abs(RValue<Int4> x)
3587         {
3588                 if(CPUID::supportsSSSE3())
3589                 {
3590                         return x86::pabsd(x);
3591                 }
3592                 else
3593                 {
3594                         Int4 mask = (x >> 31);
3595                         return (mask ^ x) - mask;
3596                 }
3597         }
3598
3599         RValue<Short8> MulHigh(RValue<Short8> x, RValue<Short8> y)
3600         {
3601                 return x86::pmulhw(x, y);   // FIXME: Fallback required
3602         }
3603
3604         Type *Short8::getType()
3605         {
3606                 return T(VectorType::get(Short::getType(), 8));
3607         }
3608
3609         UShort8::UShort8(unsigned short c0, unsigned short c1, unsigned short c2, unsigned short c3, unsigned short c4, unsigned short c5, unsigned short c6, unsigned short c7)
3610         {
3611         //      xyzw.parent = this;
3612
3613                 Constant *constantVector[8];
3614                 constantVector[0] = Nucleus::createConstantShort(c0);
3615                 constantVector[1] = Nucleus::createConstantShort(c1);
3616                 constantVector[2] = Nucleus::createConstantShort(c2);
3617                 constantVector[3] = Nucleus::createConstantShort(c3);
3618                 constantVector[4] = Nucleus::createConstantShort(c4);
3619                 constantVector[5] = Nucleus::createConstantShort(c5);
3620                 constantVector[6] = Nucleus::createConstantShort(c6);
3621                 constantVector[7] = Nucleus::createConstantShort(c7);
3622
3623                 storeValue(Nucleus::createConstantVector(constantVector, 8));
3624         }
3625
3626         UShort8::UShort8(RValue<UShort8> rhs)
3627         {
3628         //      xyzw.parent = this;
3629
3630                 storeValue(rhs.value);
3631         }
3632
3633         UShort8::UShort8(const Reference<UShort8> &rhs)
3634         {
3635         //      xyzw.parent = this;
3636
3637                 Value *value = rhs.loadValue();
3638                 storeValue(value);
3639         }
3640
3641         UShort8::UShort8(RValue<UShort4> lo, RValue<UShort4> hi)
3642         {
3643                 Value *loLong = Nucleus::createBitCast(lo.value, Long::getType());
3644                 Value *hiLong = Nucleus::createBitCast(hi.value, Long::getType());
3645
3646                 Value *long2 = V(UndefValue::get(Long2::getType()));
3647                 long2 = Nucleus::createInsertElement(long2, loLong, 0);
3648                 long2 = Nucleus::createInsertElement(long2, hiLong, 1);
3649                 Value *short8 = Nucleus::createBitCast(long2, Short8::getType());
3650
3651                 storeValue(short8);
3652         }
3653
3654         RValue<UShort8> UShort8::operator=(RValue<UShort8> rhs) const
3655         {
3656                 storeValue(rhs.value);
3657
3658                 return rhs;
3659         }
3660
3661         RValue<UShort8> UShort8::operator=(const UShort8 &rhs) const
3662         {
3663                 Value *value = rhs.loadValue();
3664                 storeValue(value);
3665
3666                 return RValue<UShort8>(value);
3667         }
3668
3669         RValue<UShort8> UShort8::operator=(const Reference<UShort8> &rhs) const
3670         {
3671                 Value *value = rhs.loadValue();
3672                 storeValue(value);
3673
3674                 return RValue<UShort8>(value);
3675         }
3676
3677         RValue<UShort8> operator&(RValue<UShort8> lhs, RValue<UShort8> rhs)
3678         {
3679                 return RValue<UShort8>(Nucleus::createAnd(lhs.value, rhs.value));
3680         }
3681
3682         RValue<UShort8> operator<<(RValue<UShort8> lhs, unsigned char rhs)
3683         {
3684                 return As<UShort8>(x86::psllw(As<Short8>(lhs), rhs));   // FIXME: Fallback required
3685         }
3686
3687         RValue<UShort8> operator>>(RValue<UShort8> lhs, unsigned char rhs)
3688         {
3689                 return x86::psrlw(lhs, rhs);   // FIXME: Fallback required
3690         }
3691
3692         RValue<UShort8> operator+(RValue<UShort8> lhs, RValue<UShort8> rhs)
3693         {
3694                 return RValue<UShort8>(Nucleus::createAdd(lhs.value, rhs.value));
3695         }
3696
3697         RValue<UShort8> operator*(RValue<UShort8> lhs, RValue<UShort8> rhs)
3698         {
3699                 return RValue<UShort8>(Nucleus::createMul(lhs.value, rhs.value));
3700         }
3701
3702         RValue<UShort8> operator+=(const UShort8 &lhs, RValue<UShort8> rhs)
3703         {
3704                 return lhs = lhs + rhs;
3705         }
3706
3707         RValue<UShort8> operator~(RValue<UShort8> val)
3708         {
3709                 return RValue<UShort8>(Nucleus::createNot(val.value));
3710         }
3711
3712         RValue<UShort8> Swizzle(RValue<UShort8> x, char select0, char select1, char select2, char select3, char select4, char select5, char select6, char select7)
3713         {
3714                 Constant *pshufb[16];
3715                 pshufb[0] = Nucleus::createConstantInt(select0 + 0);
3716                 pshufb[1] = Nucleus::createConstantInt(select0 + 1);
3717                 pshufb[2] = Nucleus::createConstantInt(select1 + 0);
3718                 pshufb[3] = Nucleus::createConstantInt(select1 + 1);
3719                 pshufb[4] = Nucleus::createConstantInt(select2 + 0);
3720                 pshufb[5] = Nucleus::createConstantInt(select2 + 1);
3721                 pshufb[6] = Nucleus::createConstantInt(select3 + 0);
3722                 pshufb[7] = Nucleus::createConstantInt(select3 + 1);
3723                 pshufb[8] = Nucleus::createConstantInt(select4 + 0);
3724                 pshufb[9] = Nucleus::createConstantInt(select4 + 1);
3725                 pshufb[10] = Nucleus::createConstantInt(select5 + 0);
3726                 pshufb[11] = Nucleus::createConstantInt(select5 + 1);
3727                 pshufb[12] = Nucleus::createConstantInt(select6 + 0);
3728                 pshufb[13] = Nucleus::createConstantInt(select6 + 1);
3729                 pshufb[14] = Nucleus::createConstantInt(select7 + 0);
3730                 pshufb[15] = Nucleus::createConstantInt(select7 + 1);
3731
3732                 Value *byte16 = Nucleus::createBitCast(x.value, Byte16::getType());
3733                 Value *shuffle = Nucleus::createShuffleVector(byte16, V(UndefValue::get(Byte16::getType())), V(Nucleus::createConstantVector(pshufb, 16)));
3734                 Value *short8 = Nucleus::createBitCast(shuffle, UShort8::getType());
3735
3736                 return RValue<UShort8>(short8);
3737         }
3738
3739         RValue<UShort8> MulHigh(RValue<UShort8> x, RValue<UShort8> y)
3740         {
3741                 return x86::pmulhuw(x, y);   // FIXME: Fallback required
3742         }
3743
3744         // FIXME: Implement as Shuffle(x, y, Select(i0, ..., i16)) and Shuffle(x, y, SELECT_PACK_REPEAT(element))
3745 //      RValue<UShort8> PackRepeat(RValue<Byte16> x, RValue<Byte16> y, int element)
3746 //      {
3747 //              Constant *pshufb[16];
3748 //              pshufb[0] = Nucleus::createConstantInt(element + 0);
3749 //              pshufb[1] = Nucleus::createConstantInt(element + 0);
3750 //              pshufb[2] = Nucleus::createConstantInt(element + 4);
3751 //              pshufb[3] = Nucleus::createConstantInt(element + 4);
3752 //              pshufb[4] = Nucleus::createConstantInt(element + 8);
3753 //              pshufb[5] = Nucleus::createConstantInt(element + 8);
3754 //              pshufb[6] = Nucleus::createConstantInt(element + 12);
3755 //              pshufb[7] = Nucleus::createConstantInt(element + 12);
3756 //              pshufb[8] = Nucleus::createConstantInt(element + 16);
3757 //              pshufb[9] = Nucleus::createConstantInt(element + 16);
3758 //              pshufb[10] = Nucleus::createConstantInt(element + 20);
3759 //              pshufb[11] = Nucleus::createConstantInt(element + 20);
3760 //              pshufb[12] = Nucleus::createConstantInt(element + 24);
3761 //              pshufb[13] = Nucleus::createConstantInt(element + 24);
3762 //              pshufb[14] = Nucleus::createConstantInt(element + 28);
3763 //              pshufb[15] = Nucleus::createConstantInt(element + 28);
3764 //
3765 //              Value *shuffle = Nucleus::createShuffleVector(x.value, y.value, Nucleus::createConstantVector(pshufb, 16));
3766 //              Value *short8 = Nucleus::createBitCast(shuffle, UShort8::getType());
3767 //
3768 //              return RValue<UShort8>(short8);
3769 //      }
3770
3771         Type *UShort8::getType()
3772         {
3773                 return T(VectorType::get(UShort::getType(), 8));
3774         }
3775
3776         Int::Int(Argument<Int> argument)
3777         {
3778                 storeValue(argument.value);
3779         }
3780
3781         Int::Int(RValue<Byte> cast)
3782         {
3783                 Value *integer = Nucleus::createZExt(cast.value, Int::getType());
3784
3785                 storeValue(integer);
3786         }
3787
3788         Int::Int(RValue<SByte> cast)
3789         {
3790                 Value *integer = Nucleus::createSExt(cast.value, Int::getType());
3791
3792                 storeValue(integer);
3793         }
3794
3795         Int::Int(RValue<Short> cast)
3796         {
3797                 Value *integer = Nucleus::createSExt(cast.value, Int::getType());
3798
3799                 storeValue(integer);
3800         }
3801
3802         Int::Int(RValue<UShort> cast)
3803         {
3804                 Value *integer = Nucleus::createZExt(cast.value, Int::getType());
3805
3806                 storeValue(integer);
3807         }
3808
3809         Int::Int(RValue<Int2> cast)
3810         {
3811                 *this = Extract(cast, 0);
3812         }
3813
3814         Int::Int(RValue<Long> cast)
3815         {
3816                 Value *integer = Nucleus::createTrunc(cast.value, Int::getType());
3817
3818                 storeValue(integer);
3819         }
3820
3821         Int::Int(RValue<Float> cast)
3822         {
3823                 Value *integer = Nucleus::createFPToSI(cast.value, Int::getType());
3824
3825                 storeValue(integer);
3826         }
3827
3828         Int::Int()
3829         {
3830         }
3831
3832         Int::Int(int x)
3833         {
3834                 storeValue(Nucleus::createConstantInt(x));
3835         }
3836
3837         Int::Int(RValue<Int> rhs)
3838         {
3839                 storeValue(rhs.value);
3840         }
3841
3842         Int::Int(RValue<UInt> rhs)
3843         {
3844                 storeValue(rhs.value);
3845         }
3846
3847         Int::Int(const Int &rhs)
3848         {
3849                 Value *value = rhs.loadValue();
3850                 storeValue(value);
3851         }
3852
3853         Int::Int(const Reference<Int> &rhs)
3854         {
3855                 Value *value = rhs.loadValue();
3856                 storeValue(value);
3857         }
3858
3859         Int::Int(const UInt &rhs)
3860         {
3861                 Value *value = rhs.loadValue();
3862                 storeValue(value);
3863         }
3864
3865         Int::Int(const Reference<UInt> &rhs)
3866         {
3867                 Value *value = rhs.loadValue();
3868                 storeValue(value);
3869         }
3870
3871         RValue<Int> Int::operator=(int rhs) const
3872         {
3873                 return RValue<Int>(storeValue(Nucleus::createConstantInt(rhs)));
3874         }
3875
3876         RValue<Int> Int::operator=(RValue<Int> rhs) const
3877         {
3878                 storeValue(rhs.value);
3879
3880                 return rhs;
3881         }
3882
3883         RValue<Int> Int::operator=(RValue<UInt> rhs) const
3884         {
3885                 storeValue(rhs.value);
3886
3887                 return RValue<Int>(rhs);
3888         }
3889
3890         RValue<Int> Int::operator=(const Int &rhs) const
3891         {
3892                 Value *value = rhs.loadValue();
3893                 storeValue(value);
3894
3895                 return RValue<Int>(value);
3896         }
3897
3898         RValue<Int> Int::operator=(const Reference<Int> &rhs) const
3899         {
3900                 Value *value = rhs.loadValue();
3901                 storeValue(value);
3902
3903                 return RValue<Int>(value);
3904         }
3905
3906         RValue<Int> Int::operator=(const UInt &rhs) const
3907         {
3908                 Value *value = rhs.loadValue();
3909                 storeValue(value);
3910
3911                 return RValue<Int>(value);
3912         }
3913
3914         RValue<Int> Int::operator=(const Reference<UInt> &rhs) const
3915         {
3916                 Value *value = rhs.loadValue();
3917                 storeValue(value);
3918
3919                 return RValue<Int>(value);
3920         }
3921
3922         RValue<Int> operator+(RValue<Int> lhs, RValue<Int> rhs)
3923         {
3924                 return RValue<Int>(Nucleus::createAdd(lhs.value, rhs.value));
3925         }
3926
3927         RValue<Int> operator-(RValue<Int> lhs, RValue<Int> rhs)
3928         {
3929                 return RValue<Int>(Nucleus::createSub(lhs.value, rhs.value));
3930         }
3931
3932         RValue<Int> operator*(RValue<Int> lhs, RValue<Int> rhs)
3933         {
3934                 return RValue<Int>(Nucleus::createMul(lhs.value, rhs.value));
3935         }
3936
3937         RValue<Int> operator/(RValue<Int> lhs, RValue<Int> rhs)
3938         {
3939                 return RValue<Int>(Nucleus::createSDiv(lhs.value, rhs.value));
3940         }
3941
3942         RValue<Int> operator%(RValue<Int> lhs, RValue<Int> rhs)
3943         {
3944                 return RValue<Int>(Nucleus::createSRem(lhs.value, rhs.value));
3945         }
3946
3947         RValue<Int> operator&(RValue<Int> lhs, RValue<Int> rhs)
3948         {
3949                 return RValue<Int>(Nucleus::createAnd(lhs.value, rhs.value));
3950         }
3951
3952         RValue<Int> operator|(RValue<Int> lhs, RValue<Int> rhs)
3953         {
3954                 return RValue<Int>(Nucleus::createOr(lhs.value, rhs.value));
3955         }
3956
3957         RValue<Int> operator^(RValue<Int> lhs, RValue<Int> rhs)
3958         {
3959                 return RValue<Int>(Nucleus::createXor(lhs.value, rhs.value));
3960         }
3961
3962         RValue<Int> operator<<(RValue<Int> lhs, RValue<Int> rhs)
3963         {
3964                 return RValue<Int>(Nucleus::createShl(lhs.value, rhs.value));
3965         }
3966
3967         RValue<Int> operator>>(RValue<Int> lhs, RValue<Int> rhs)
3968         {
3969                 return RValue<Int>(Nucleus::createAShr(lhs.value, rhs.value));
3970         }
3971
3972         RValue<Int> operator+=(const Int &lhs, RValue<Int> rhs)
3973         {
3974                 return lhs = lhs + rhs;
3975         }
3976
3977         RValue<Int> operator-=(const Int &lhs, RValue<Int> rhs)
3978         {
3979                 return lhs = lhs - rhs;
3980         }
3981
3982         RValue<Int> operator*=(const Int &lhs, RValue<Int> rhs)
3983         {
3984                 return lhs = lhs * rhs;
3985         }
3986
3987         RValue<Int> operator/=(const Int &lhs, RValue<Int> rhs)
3988         {
3989                 return lhs = lhs / rhs;
3990         }
3991
3992         RValue<Int> operator%=(const Int &lhs, RValue<Int> rhs)
3993         {
3994                 return lhs = lhs % rhs;
3995         }
3996
3997         RValue<Int> operator&=(const Int &lhs, RValue<Int> rhs)
3998         {
3999                 return lhs = lhs & rhs;
4000         }
4001
4002         RValue<Int> operator|=(const Int &lhs, RValue<Int> rhs)
4003         {
4004                 return lhs = lhs | rhs;
4005         }
4006
4007         RValue<Int> operator^=(const Int &lhs, RValue<Int> rhs)
4008         {
4009                 return lhs = lhs ^ rhs;
4010         }
4011
4012         RValue<Int> operator<<=(const Int &lhs, RValue<Int> rhs)
4013         {
4014                 return lhs = lhs << rhs;
4015         }
4016
4017         RValue<Int> operator>>=(const Int &lhs, RValue<Int> rhs)
4018         {
4019                 return lhs = lhs >> rhs;
4020         }
4021
4022         RValue<Int> operator+(RValue<Int> val)
4023         {
4024                 return val;
4025         }
4026
4027         RValue<Int> operator-(RValue<Int> val)
4028         {
4029                 return RValue<Int>(Nucleus::createNeg(val.value));
4030         }
4031
4032         RValue<Int> operator~(RValue<Int> val)
4033         {
4034                 return RValue<Int>(Nucleus::createNot(val.value));
4035         }
4036
4037         RValue<Int> operator++(const Int &val, int)   // Post-increment
4038         {
4039                 RValue<Int> res = val;
4040
4041                 Value *inc = Nucleus::createAdd(res.value, V(Nucleus::createConstantInt(1)));
4042                 val.storeValue(inc);
4043
4044                 return res;
4045         }
4046
4047         const Int &operator++(const Int &val)   // Pre-increment
4048         {
4049                 Value *inc = Nucleus::createAdd(val.loadValue(), V(Nucleus::createConstantInt(1)));
4050                 val.storeValue(inc);
4051
4052                 return val;
4053         }
4054
4055         RValue<Int> operator--(const Int &val, int)   // Post-decrement
4056         {
4057                 RValue<Int> res = val;
4058
4059                 Value *inc = Nucleus::createSub(res.value, V(Nucleus::createConstantInt(1)));
4060                 val.storeValue(inc);
4061
4062                 return res;
4063         }
4064
4065         const Int &operator--(const Int &val)   // Pre-decrement
4066         {
4067                 Value *inc = Nucleus::createSub(val.loadValue(), V(Nucleus::createConstantInt(1)));
4068                 val.storeValue(inc);
4069
4070                 return val;
4071         }
4072
4073         RValue<Bool> operator<(RValue<Int> lhs, RValue<Int> rhs)
4074         {
4075                 return RValue<Bool>(Nucleus::createICmpSLT(lhs.value, rhs.value));
4076         }
4077
4078         RValue<Bool> operator<=(RValue<Int> lhs, RValue<Int> rhs)
4079         {
4080                 return RValue<Bool>(Nucleus::createICmpSLE(lhs.value, rhs.value));
4081         }
4082
4083         RValue<Bool> operator>(RValue<Int> lhs, RValue<Int> rhs)
4084         {
4085                 return RValue<Bool>(Nucleus::createICmpSGT(lhs.value, rhs.value));
4086         }
4087
4088         RValue<Bool> operator>=(RValue<Int> lhs, RValue<Int> rhs)
4089         {
4090                 return RValue<Bool>(Nucleus::createICmpSGE(lhs.value, rhs.value));
4091         }
4092
4093         RValue<Bool> operator!=(RValue<Int> lhs, RValue<Int> rhs)
4094         {
4095                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
4096         }
4097
4098         RValue<Bool> operator==(RValue<Int> lhs, RValue<Int> rhs)
4099         {
4100                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
4101         }
4102
4103         RValue<Int> Max(RValue<Int> x, RValue<Int> y)
4104         {
4105                 return IfThenElse(x > y, x, y);
4106         }
4107
4108         RValue<Int> Min(RValue<Int> x, RValue<Int> y)
4109         {
4110                 return IfThenElse(x < y, x, y);
4111         }
4112
4113         RValue<Int> Clamp(RValue<Int> x, RValue<Int> min, RValue<Int> max)
4114         {
4115                 return Min(Max(x, min), max);
4116         }
4117
4118         RValue<Int> RoundInt(RValue<Float> cast)
4119         {
4120                 return x86::cvtss2si(cast);
4121
4122         //      return IfThenElse(val > 0.0f, Int(val + 0.5f), Int(val - 0.5f));
4123         }
4124
4125         Type *Int::getType()
4126         {
4127                 return T(llvm::Type::getInt32Ty(*::context));
4128         }
4129
4130         Long::Long(RValue<Int> cast)
4131         {
4132                 Value *integer = Nucleus::createSExt(cast.value, Long::getType());
4133
4134                 storeValue(integer);
4135         }
4136
4137         Long::Long(RValue<UInt> cast)
4138         {
4139                 Value *integer = Nucleus::createZExt(cast.value, Long::getType());
4140
4141                 storeValue(integer);
4142         }
4143
4144         Long::Long()
4145         {
4146         }
4147
4148         Long::Long(RValue<Long> rhs)
4149         {
4150                 storeValue(rhs.value);
4151         }
4152
4153         RValue<Long> Long::operator=(int64_t rhs) const
4154         {
4155                 return RValue<Long>(storeValue(Nucleus::createConstantInt(rhs)));
4156         }
4157
4158         RValue<Long> Long::operator=(RValue<Long> rhs) const
4159         {
4160                 storeValue(rhs.value);
4161
4162                 return rhs;
4163         }
4164
4165         RValue<Long> Long::operator=(const Long &rhs) const
4166         {
4167                 Value *value = rhs.loadValue();
4168                 storeValue(value);
4169
4170                 return RValue<Long>(value);
4171         }
4172
4173         RValue<Long> Long::operator=(const Reference<Long> &rhs) const
4174         {
4175                 Value *value = rhs.loadValue();
4176                 storeValue(value);
4177
4178                 return RValue<Long>(value);
4179         }
4180
4181         RValue<Long> operator+(RValue<Long> lhs, RValue<Long> rhs)
4182         {
4183                 return RValue<Long>(Nucleus::createAdd(lhs.value, rhs.value));
4184         }
4185
4186         RValue<Long> operator-(RValue<Long> lhs, RValue<Long> rhs)
4187         {
4188                 return RValue<Long>(Nucleus::createSub(lhs.value, rhs.value));
4189         }
4190
4191         RValue<Long> operator+=(const Long &lhs, RValue<Long> rhs)
4192         {
4193                 return lhs = lhs + rhs;
4194         }
4195
4196         RValue<Long> operator-=(const Long &lhs, RValue<Long> rhs)
4197         {
4198                 return lhs = lhs - rhs;
4199         }
4200
4201         RValue<Long> AddAtomic(RValue<Pointer<Long> > x, RValue<Long> y)
4202         {
4203                 return RValue<Long>(Nucleus::createAtomicAdd(x.value, y.value));
4204         }
4205
4206         Type *Long::getType()
4207         {
4208                 return T(llvm::Type::getInt64Ty(*::context));
4209         }
4210
4211         Long1::Long1(const RValue<UInt> cast)
4212         {
4213                 Value *undefCast = Nucleus::createInsertElement(V(UndefValue::get(VectorType::get(Int::getType(), 2))), cast.value, 0);
4214                 Value *zeroCast = Nucleus::createInsertElement(undefCast, V(Nucleus::createConstantInt(0)), 1);
4215
4216                 storeValue(Nucleus::createBitCast(zeroCast, Long1::getType()));
4217         }
4218
4219         Long1::Long1(RValue<Long1> rhs)
4220         {
4221                 storeValue(rhs.value);
4222         }
4223
4224         Type *Long1::getType()
4225         {
4226                 if(CPUID::supportsMMX2())
4227                 {
4228                         return MMX::getType();
4229                 }
4230                 else
4231                 {
4232                         return T(VectorType::get(Long::getType(), 1));
4233                 }
4234         }
4235
4236         RValue<Long2> UnpackHigh(RValue<Long2> x, RValue<Long2> y)
4237         {
4238                 Constant *shuffle[2];
4239                 shuffle[0] = Nucleus::createConstantInt(1);
4240                 shuffle[1] = Nucleus::createConstantInt(3);
4241
4242                 Value *packed = Nucleus::createShuffleVector(x.value, y.value, V(Nucleus::createConstantVector(shuffle, 2)));
4243
4244                 return RValue<Long2>(packed);
4245         }
4246
4247         Type *Long2::getType()
4248         {
4249                 return T(VectorType::get(Long::getType(), 2));
4250         }
4251
4252         UInt::UInt(Argument<UInt> argument)
4253         {
4254                 storeValue(argument.value);
4255         }
4256
4257         UInt::UInt(RValue<UShort> cast)
4258         {
4259                 Value *integer = Nucleus::createZExt(cast.value, UInt::getType());
4260
4261                 storeValue(integer);
4262         }
4263
4264         UInt::UInt(RValue<Long> cast)
4265         {
4266                 Value *integer = Nucleus::createTrunc(cast.value, UInt::getType());
4267
4268                 storeValue(integer);
4269         }
4270
4271         UInt::UInt(RValue<Float> cast)
4272         {
4273                 // Note: createFPToUI is broken, must perform conversion using createFPtoSI
4274                 // Value *integer = Nucleus::createFPToUI(cast.value, UInt::getType());
4275
4276                 // Smallest positive value representable in UInt, but not in Int
4277                 const unsigned int ustart = 0x80000000u;
4278                 const float ustartf = float(ustart);
4279
4280                 // If the value is negative, store 0, otherwise store the result of the conversion
4281                 storeValue((~(As<Int>(cast) >> 31) &
4282                 // Check if the value can be represented as an Int
4283                         IfThenElse(cast >= ustartf,
4284                 // If the value is too large, subtract ustart and re-add it after conversion.
4285                                 As<Int>(As<UInt>(Int(cast - Float(ustartf))) + UInt(ustart)),
4286                 // Otherwise, just convert normally
4287                                 Int(cast))).value);
4288         }
4289
4290         UInt::UInt()
4291         {
4292         }
4293
4294         UInt::UInt(int x)
4295         {
4296                 storeValue(Nucleus::createConstantInt(x));
4297         }
4298
4299         UInt::UInt(unsigned int x)
4300         {
4301                 storeValue(Nucleus::createConstantInt(x));
4302         }
4303
4304         UInt::UInt(RValue<UInt> rhs)
4305         {
4306                 storeValue(rhs.value);
4307         }
4308
4309         UInt::UInt(RValue<Int> rhs)
4310         {
4311                 storeValue(rhs.value);
4312         }
4313
4314         UInt::UInt(const UInt &rhs)
4315         {
4316                 Value *value = rhs.loadValue();
4317                 storeValue(value);
4318         }
4319
4320         UInt::UInt(const Reference<UInt> &rhs)
4321         {
4322                 Value *value = rhs.loadValue();
4323                 storeValue(value);
4324         }
4325
4326         UInt::UInt(const Int &rhs)
4327         {
4328                 Value *value = rhs.loadValue();
4329                 storeValue(value);
4330         }
4331
4332         UInt::UInt(const Reference<Int> &rhs)
4333         {
4334                 Value *value = rhs.loadValue();
4335                 storeValue(value);
4336         }
4337
4338         RValue<UInt> UInt::operator=(unsigned int rhs) const
4339         {
4340                 return RValue<UInt>(storeValue(Nucleus::createConstantInt(rhs)));
4341         }
4342
4343         RValue<UInt> UInt::operator=(RValue<UInt> rhs) const
4344         {
4345                 storeValue(rhs.value);
4346
4347                 return rhs;
4348         }
4349
4350         RValue<UInt> UInt::operator=(RValue<Int> rhs) const
4351         {
4352                 storeValue(rhs.value);
4353
4354                 return RValue<UInt>(rhs);
4355         }
4356
4357         RValue<UInt> UInt::operator=(const UInt &rhs) const
4358         {
4359                 Value *value = rhs.loadValue();
4360                 storeValue(value);
4361
4362                 return RValue<UInt>(value);
4363         }
4364
4365         RValue<UInt> UInt::operator=(const Reference<UInt> &rhs) const
4366         {
4367                 Value *value = rhs.loadValue();
4368                 storeValue(value);
4369
4370                 return RValue<UInt>(value);
4371         }
4372
4373         RValue<UInt> UInt::operator=(const Int &rhs) const
4374         {
4375                 Value *value = rhs.loadValue();
4376                 storeValue(value);
4377
4378                 return RValue<UInt>(value);
4379         }
4380
4381         RValue<UInt> UInt::operator=(const Reference<Int> &rhs) const
4382         {
4383                 Value *value = rhs.loadValue();
4384                 storeValue(value);
4385
4386                 return RValue<UInt>(value);
4387         }
4388
4389         RValue<UInt> operator+(RValue<UInt> lhs, RValue<UInt> rhs)
4390         {
4391                 return RValue<UInt>(Nucleus::createAdd(lhs.value, rhs.value));
4392         }
4393
4394         RValue<UInt> operator-(RValue<UInt> lhs, RValue<UInt> rhs)
4395         {
4396                 return RValue<UInt>(Nucleus::createSub(lhs.value, rhs.value));
4397         }
4398
4399         RValue<UInt> operator*(RValue<UInt> lhs, RValue<UInt> rhs)
4400         {
4401                 return RValue<UInt>(Nucleus::createMul(lhs.value, rhs.value));
4402         }
4403
4404         RValue<UInt> operator/(RValue<UInt> lhs, RValue<UInt> rhs)
4405         {
4406                 return RValue<UInt>(Nucleus::createUDiv(lhs.value, rhs.value));
4407         }
4408
4409         RValue<UInt> operator%(RValue<UInt> lhs, RValue<UInt> rhs)
4410         {
4411                 return RValue<UInt>(Nucleus::createURem(lhs.value, rhs.value));
4412         }
4413
4414         RValue<UInt> operator&(RValue<UInt> lhs, RValue<UInt> rhs)
4415         {
4416                 return RValue<UInt>(Nucleus::createAnd(lhs.value, rhs.value));
4417         }
4418
4419         RValue<UInt> operator|(RValue<UInt> lhs, RValue<UInt> rhs)
4420         {
4421                 return RValue<UInt>(Nucleus::createOr(lhs.value, rhs.value));
4422         }
4423
4424         RValue<UInt> operator^(RValue<UInt> lhs, RValue<UInt> rhs)
4425         {
4426                 return RValue<UInt>(Nucleus::createXor(lhs.value, rhs.value));
4427         }
4428
4429         RValue<UInt> operator<<(RValue<UInt> lhs, RValue<UInt> rhs)
4430         {
4431                 return RValue<UInt>(Nucleus::createShl(lhs.value, rhs.value));
4432         }
4433
4434         RValue<UInt> operator>>(RValue<UInt> lhs, RValue<UInt> rhs)
4435         {
4436                 return RValue<UInt>(Nucleus::createLShr(lhs.value, rhs.value));
4437         }
4438
4439         RValue<UInt> operator+=(const UInt &lhs, RValue<UInt> rhs)
4440         {
4441                 return lhs = lhs + rhs;
4442         }
4443
4444         RValue<UInt> operator-=(const UInt &lhs, RValue<UInt> rhs)
4445         {
4446                 return lhs = lhs - rhs;
4447         }
4448
4449         RValue<UInt> operator*=(const UInt &lhs, RValue<UInt> rhs)
4450         {
4451                 return lhs = lhs * rhs;
4452         }
4453
4454         RValue<UInt> operator/=(const UInt &lhs, RValue<UInt> rhs)
4455         {
4456                 return lhs = lhs / rhs;
4457         }
4458
4459         RValue<UInt> operator%=(const UInt &lhs, RValue<UInt> rhs)
4460         {
4461                 return lhs = lhs % rhs;
4462         }
4463
4464         RValue<UInt> operator&=(const UInt &lhs, RValue<UInt> rhs)
4465         {
4466                 return lhs = lhs & rhs;
4467         }
4468
4469         RValue<UInt> operator|=(const UInt &lhs, RValue<UInt> rhs)
4470         {
4471                 return lhs = lhs | rhs;
4472         }
4473
4474         RValue<UInt> operator^=(const UInt &lhs, RValue<UInt> rhs)
4475         {
4476                 return lhs = lhs ^ rhs;
4477         }
4478
4479         RValue<UInt> operator<<=(const UInt &lhs, RValue<UInt> rhs)
4480         {
4481                 return lhs = lhs << rhs;
4482         }
4483
4484         RValue<UInt> operator>>=(const UInt &lhs, RValue<UInt> rhs)
4485         {
4486                 return lhs = lhs >> rhs;
4487         }
4488
4489         RValue<UInt> operator+(RValue<UInt> val)
4490         {
4491                 return val;
4492         }
4493
4494         RValue<UInt> operator-(RValue<UInt> val)
4495         {
4496                 return RValue<UInt>(Nucleus::createNeg(val.value));
4497         }
4498
4499         RValue<UInt> operator~(RValue<UInt> val)
4500         {
4501                 return RValue<UInt>(Nucleus::createNot(val.value));
4502         }
4503
4504         RValue<UInt> operator++(const UInt &val, int)   // Post-increment
4505         {
4506                 RValue<UInt> res = val;
4507
4508                 Value *inc = Nucleus::createAdd(res.value, V(Nucleus::createConstantInt(1)));
4509                 val.storeValue(inc);
4510
4511                 return res;
4512         }
4513
4514         const UInt &operator++(const UInt &val)   // Pre-increment
4515         {
4516                 Value *inc = Nucleus::createAdd(val.loadValue(), V(Nucleus::createConstantInt(1)));
4517                 val.storeValue(inc);
4518
4519                 return val;
4520         }
4521
4522         RValue<UInt> operator--(const UInt &val, int)   // Post-decrement
4523         {
4524                 RValue<UInt> res = val;
4525
4526                 Value *inc = Nucleus::createSub(res.value, V(Nucleus::createConstantInt(1)));
4527                 val.storeValue(inc);
4528
4529                 return res;
4530         }
4531
4532         const UInt &operator--(const UInt &val)   // Pre-decrement
4533         {
4534                 Value *inc = Nucleus::createSub(val.loadValue(), V(Nucleus::createConstantInt(1)));
4535                 val.storeValue(inc);
4536
4537                 return val;
4538         }
4539
4540         RValue<UInt> Max(RValue<UInt> x, RValue<UInt> y)
4541         {
4542                 return IfThenElse(x > y, x, y);
4543         }
4544
4545         RValue<UInt> Min(RValue<UInt> x, RValue<UInt> y)
4546         {
4547                 return IfThenElse(x < y, x, y);
4548         }
4549
4550         RValue<UInt> Clamp(RValue<UInt> x, RValue<UInt> min, RValue<UInt> max)
4551         {
4552                 return Min(Max(x, min), max);
4553         }
4554
4555         RValue<Bool> operator<(RValue<UInt> lhs, RValue<UInt> rhs)
4556         {
4557                 return RValue<Bool>(Nucleus::createICmpULT(lhs.value, rhs.value));
4558         }
4559
4560         RValue<Bool> operator<=(RValue<UInt> lhs, RValue<UInt> rhs)
4561         {
4562                 return RValue<Bool>(Nucleus::createICmpULE(lhs.value, rhs.value));
4563         }
4564
4565         RValue<Bool> operator>(RValue<UInt> lhs, RValue<UInt> rhs)
4566         {
4567                 return RValue<Bool>(Nucleus::createICmpUGT(lhs.value, rhs.value));
4568         }
4569
4570         RValue<Bool> operator>=(RValue<UInt> lhs, RValue<UInt> rhs)
4571         {
4572                 return RValue<Bool>(Nucleus::createICmpUGE(lhs.value, rhs.value));
4573         }
4574
4575         RValue<Bool> operator!=(RValue<UInt> lhs, RValue<UInt> rhs)
4576         {
4577                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
4578         }
4579
4580         RValue<Bool> operator==(RValue<UInt> lhs, RValue<UInt> rhs)
4581         {
4582                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
4583         }
4584
4585 //      RValue<UInt> RoundUInt(RValue<Float> cast)
4586 //      {
4587 //              return x86::cvtss2si(val);   // FIXME: Unsigned
4588 //
4589 //      //      return IfThenElse(val > 0.0f, Int(val + 0.5f), Int(val - 0.5f));
4590 //      }
4591
4592         Type *UInt::getType()
4593         {
4594                 return T(llvm::Type::getInt32Ty(*::context));
4595         }
4596
4597 //      Int2::Int2(RValue<Int> cast)
4598 //      {
4599 //              Value *extend = Nucleus::createZExt(cast.value, Long::getType());
4600 //              Value *vector = Nucleus::createBitCast(extend, Int2::getType());
4601 //
4602 //              Constant *shuffle[2];
4603 //              shuffle[0] = Nucleus::createConstantInt(0);
4604 //              shuffle[1] = Nucleus::createConstantInt(0);
4605 //
4606 //              Value *replicate = Nucleus::createShuffleVector(vector, UndefValue::get(Int2::getType()), Nucleus::createConstantVector(shuffle, 2));
4607 //
4608 //              storeValue(replicate);
4609 //      }
4610
4611         Int2::Int2(RValue<Int4> cast)
4612         {
4613                 Value *long2 = Nucleus::createBitCast(cast.value, Long2::getType());
4614                 Value *element = Nucleus::createExtractElement(long2, 0);
4615                 Value *int2 = Nucleus::createBitCast(element, Int2::getType());
4616
4617                 storeValue(int2);
4618         }
4619
4620         Int2::Int2()
4621         {
4622         //      xy.parent = this;
4623         }
4624
4625         Int2::Int2(int x, int y)
4626         {
4627         //      xy.parent = this;
4628
4629                 Constant *constantVector[2];
4630                 constantVector[0] = Nucleus::createConstantInt(x);
4631                 constantVector[1] = Nucleus::createConstantInt(y);
4632                 Value *vector = V(Nucleus::createConstantVector(constantVector, 2));
4633
4634                 storeValue(Nucleus::createBitCast(vector, getType()));
4635         }
4636
4637         Int2::Int2(RValue<Int2> rhs)
4638         {
4639         //      xy.parent = this;
4640
4641                 storeValue(rhs.value);
4642         }
4643
4644         Int2::Int2(const Int2 &rhs)
4645         {
4646         //      xy.parent = this;
4647
4648                 Value *value = rhs.loadValue();
4649                 storeValue(value);
4650         }
4651
4652         Int2::Int2(const Reference<Int2> &rhs)
4653         {
4654         //      xy.parent = this;
4655
4656                 Value *value = rhs.loadValue();
4657                 storeValue(value);
4658         }
4659
4660         Int2::Int2(RValue<Int> lo, RValue<Int> hi)
4661         {
4662                 if(CPUID::supportsMMX2())
4663                 {
4664                         // movd mm0, lo
4665                         // movd mm1, hi
4666                         // punpckldq mm0, mm1
4667                         storeValue(As<Int2>(UnpackLow(As<Int2>(Long1(RValue<UInt>(lo))), As<Int2>(Long1(RValue<UInt>(hi))))).value);
4668                 }
4669                 else
4670                 {
4671                         Constant *shuffle[2];
4672                         shuffle[0] = Nucleus::createConstantInt(0);
4673                         shuffle[1] = Nucleus::createConstantInt(1);
4674
4675                         Value *packed = Nucleus::createShuffleVector(Nucleus::createBitCast(lo.value, T(VectorType::get(Int::getType(), 1))), Nucleus::createBitCast(hi.value, T(VectorType::get(Int::getType(), 1))), V(Nucleus::createConstantVector(shuffle, 2)));
4676
4677                         storeValue(Nucleus::createBitCast(packed, Int2::getType()));
4678                 }
4679         }
4680
4681         RValue<Int2> Int2::operator=(RValue<Int2> rhs) const
4682         {
4683                 storeValue(rhs.value);
4684
4685                 return rhs;
4686         }
4687
4688         RValue<Int2> Int2::operator=(const Int2 &rhs) const
4689         {
4690                 Value *value = rhs.loadValue();
4691                 storeValue(value);
4692
4693                 return RValue<Int2>(value);
4694         }
4695
4696         RValue<Int2> Int2::operator=(const Reference<Int2> &rhs) const
4697         {
4698                 Value *value = rhs.loadValue();
4699                 storeValue(value);
4700
4701                 return RValue<Int2>(value);
4702         }
4703
4704         RValue<Int2> operator+(RValue<Int2> lhs, RValue<Int2> rhs)
4705         {
4706                 if(CPUID::supportsMMX2())
4707                 {
4708                         return x86::paddd(lhs, rhs);
4709                 }
4710                 else
4711                 {
4712                         return RValue<Int2>(Nucleus::createAdd(lhs.value, rhs.value));
4713                 }
4714         }
4715
4716         RValue<Int2> operator-(RValue<Int2> lhs, RValue<Int2> rhs)
4717         {
4718                 if(CPUID::supportsMMX2())
4719                 {
4720                         return x86::psubd(lhs, rhs);
4721                 }
4722                 else
4723                 {
4724                         return RValue<Int2>(Nucleus::createSub(lhs.value, rhs.value));
4725                 }
4726         }
4727
4728 //      RValue<Int2> operator*(RValue<Int2> lhs, RValue<Int2> rhs)
4729 //      {
4730 //              return RValue<Int2>(Nucleus::createMul(lhs.value, rhs.value));
4731 //      }
4732
4733 //      RValue<Int2> operator/(RValue<Int2> lhs, RValue<Int2> rhs)
4734 //      {
4735 //              return RValue<Int2>(Nucleus::createSDiv(lhs.value, rhs.value));
4736 //      }
4737
4738 //      RValue<Int2> operator%(RValue<Int2> lhs, RValue<Int2> rhs)
4739 //      {
4740 //              return RValue<Int2>(Nucleus::createSRem(lhs.value, rhs.value));
4741 //      }
4742
4743         RValue<Int2> operator&(RValue<Int2> lhs, RValue<Int2> rhs)
4744         {
4745                 if(CPUID::supportsMMX2())
4746                 {
4747                         return As<Int2>(x86::pand(As<Short4>(lhs), As<Short4>(rhs)));
4748                 }
4749                 else
4750                 {
4751                         return RValue<Int2>(Nucleus::createAnd(lhs.value, rhs.value));
4752                 }
4753         }
4754
4755         RValue<Int2> operator|(RValue<Int2> lhs, RValue<Int2> rhs)
4756         {
4757                 if(CPUID::supportsMMX2())
4758                 {
4759                         return As<Int2>(x86::por(As<Short4>(lhs), As<Short4>(rhs)));
4760                 }
4761                 else
4762                 {
4763                         return RValue<Int2>(Nucleus::createOr(lhs.value, rhs.value));
4764                 }
4765         }
4766
4767         RValue<Int2> operator^(RValue<Int2> lhs, RValue<Int2> rhs)
4768         {
4769                 if(CPUID::supportsMMX2())
4770                 {
4771                         return As<Int2>(x86::pxor(As<Short4>(lhs), As<Short4>(rhs)));
4772                 }
4773                 else
4774                 {
4775                         return RValue<Int2>(Nucleus::createXor(lhs.value, rhs.value));
4776                 }
4777         }
4778
4779         RValue<Int2> operator<<(RValue<Int2> lhs, unsigned char rhs)
4780         {
4781         //      return RValue<Int2>(Nucleus::createShl(lhs.value, rhs.value));
4782
4783                 return x86::pslld(lhs, rhs);
4784         }
4785
4786         RValue<Int2> operator>>(RValue<Int2> lhs, unsigned char rhs)
4787         {
4788         //      return RValue<Int2>(Nucleus::createAShr(lhs.value, rhs.value));
4789
4790                 return x86::psrad(lhs, rhs);
4791         }
4792
4793         RValue<Int2> operator<<(RValue<Int2> lhs, RValue<Long1> rhs)
4794         {
4795         //      return RValue<Int2>(Nucleus::createShl(lhs.value, rhs.value));
4796
4797                 return x86::pslld(lhs, rhs);
4798         }
4799
4800         RValue<Int2> operator>>(RValue<Int2> lhs, RValue<Long1> rhs)
4801         {
4802         //      return RValue<Int2>(Nucleus::createAShr(lhs.value, rhs.value));
4803
4804                 return x86::psrad(lhs, rhs);
4805         }
4806
4807         RValue<Int2> operator+=(const Int2 &lhs, RValue<Int2> rhs)
4808         {
4809                 return lhs = lhs + rhs;
4810         }
4811
4812         RValue<Int2> operator-=(const Int2 &lhs, RValue<Int2> rhs)
4813         {
4814                 return lhs = lhs - rhs;
4815         }
4816
4817 //      RValue<Int2> operator*=(const Int2 &lhs, RValue<Int2> rhs)
4818 //      {
4819 //              return lhs = lhs * rhs;
4820 //      }
4821
4822 //      RValue<Int2> operator/=(const Int2 &lhs, RValue<Int2> rhs)
4823 //      {
4824 //              return lhs = lhs / rhs;
4825 //      }
4826
4827 //      RValue<Int2> operator%=(const Int2 &lhs, RValue<Int2> rhs)
4828 //      {
4829 //              return lhs = lhs % rhs;
4830 //      }
4831
4832         RValue<Int2> operator&=(const Int2 &lhs, RValue<Int2> rhs)
4833         {
4834                 return lhs = lhs & rhs;
4835         }
4836
4837         RValue<Int2> operator|=(const Int2 &lhs, RValue<Int2> rhs)
4838         {
4839                 return lhs = lhs | rhs;
4840         }
4841
4842         RValue<Int2> operator^=(const Int2 &lhs, RValue<Int2> rhs)
4843         {
4844                 return lhs = lhs ^ rhs;
4845         }
4846
4847         RValue<Int2> operator<<=(const Int2 &lhs, unsigned char rhs)
4848         {
4849                 return lhs = lhs << rhs;
4850         }
4851
4852         RValue<Int2> operator>>=(const Int2 &lhs, unsigned char rhs)
4853         {
4854                 return lhs = lhs >> rhs;
4855         }
4856
4857         RValue<Int2> operator<<=(const Int2 &lhs, RValue<Long1> rhs)
4858         {
4859                 return lhs = lhs << rhs;
4860         }
4861
4862         RValue<Int2> operator>>=(const Int2 &lhs, RValue<Long1> rhs)
4863         {
4864                 return lhs = lhs >> rhs;
4865         }
4866
4867 //      RValue<Int2> operator+(RValue<Int2> val)
4868 //      {
4869 //              return val;
4870 //      }
4871
4872 //      RValue<Int2> operator-(RValue<Int2> val)
4873 //      {
4874 //              return RValue<Int2>(Nucleus::createNeg(val.value));
4875 //      }
4876
4877         RValue<Int2> operator~(RValue<Int2> val)
4878         {
4879                 if(CPUID::supportsMMX2())
4880                 {
4881                         return val ^ Int2(0xFFFFFFFF, 0xFFFFFFFF);
4882                 }
4883                 else
4884                 {
4885                         return RValue<Int2>(Nucleus::createNot(val.value));
4886                 }
4887         }
4888
4889         RValue<Long1> UnpackLow(RValue<Int2> x, RValue<Int2> y)
4890         {
4891                 if(CPUID::supportsMMX2())
4892                 {
4893                         return x86::punpckldq(x, y);
4894                 }
4895                 else
4896                 {
4897                         Constant *shuffle[2];
4898                         shuffle[0] = Nucleus::createConstantInt(0);
4899                         shuffle[1] = Nucleus::createConstantInt(2);
4900
4901                         Value *packed = Nucleus::createShuffleVector(x.value, y.value, V(Nucleus::createConstantVector(shuffle, 2)));
4902
4903                         return RValue<Long1>(Nucleus::createBitCast(packed, Long1::getType()));
4904                 }
4905         }
4906
4907         RValue<Long1> UnpackHigh(RValue<Int2> x, RValue<Int2> y)
4908         {
4909                 if(CPUID::supportsMMX2())
4910                 {
4911                         return x86::punpckhdq(x, y);
4912                 }
4913                 else
4914                 {
4915                         Constant *shuffle[2];
4916                         shuffle[0] = Nucleus::createConstantInt(1);
4917                         shuffle[1] = Nucleus::createConstantInt(3);
4918
4919                         Value *packed = Nucleus::createShuffleVector(x.value, y.value, V(Nucleus::createConstantVector(shuffle, 2)));
4920
4921                         return RValue<Long1>(Nucleus::createBitCast(packed, Long1::getType()));
4922                 }
4923         }
4924
4925         RValue<Int> Extract(RValue<Int2> val, int i)
4926         {
4927                 if(false)   // FIXME: LLVM does not generate optimal code
4928                 {
4929                         return RValue<Int>(Nucleus::createExtractElement(val.value, i));
4930                 }
4931                 else
4932                 {
4933                         if(i == 0)
4934                         {
4935                                 return RValue<Int>(Nucleus::createExtractElement(Nucleus::createBitCast(val.value, T(VectorType::get(Int::getType(), 2))), 0));
4936                         }
4937                         else
4938                         {
4939                                 Int2 val2 = As<Int2>(UnpackHigh(val, val));
4940
4941                                 return Extract(val2, 0);
4942                         }
4943                 }
4944         }
4945
4946         RValue<Int2> Insert(RValue<Int2> val, RValue<Int> element, int i)
4947         {
4948                 return RValue<Int2>(Nucleus::createBitCast(Nucleus::createInsertElement(Nucleus::createBitCast(val.value, T(VectorType::get(Int::getType(), 2))), element.value, i), Int2::getType()));
4949         }
4950
4951         Type *Int2::getType()
4952         {
4953                 if(CPUID::supportsMMX2())
4954                 {
4955                         return MMX::getType();
4956                 }
4957                 else
4958                 {
4959                         return T(VectorType::get(Int::getType(), 2));
4960                 }
4961         }
4962
4963         UInt2::UInt2()
4964         {
4965         //      xy.parent = this;
4966         }
4967
4968         UInt2::UInt2(unsigned int x, unsigned int y)
4969         {
4970         //      xy.parent = this;
4971
4972                 Constant *constantVector[2];
4973                 constantVector[0] = Nucleus::createConstantInt(x);
4974                 constantVector[1] = Nucleus::createConstantInt(y);
4975                 Value *vector = V(Nucleus::createConstantVector(constantVector, 2));
4976
4977                 storeValue(Nucleus::createBitCast(vector, getType()));
4978         }
4979
4980         UInt2::UInt2(RValue<UInt2> rhs)
4981         {
4982         //      xy.parent = this;
4983
4984                 storeValue(rhs.value);
4985         }
4986
4987         UInt2::UInt2(const UInt2 &rhs)
4988         {
4989         //      xy.parent = this;
4990
4991                 Value *value = rhs.loadValue();
4992                 storeValue(value);
4993         }
4994
4995         UInt2::UInt2(const Reference<UInt2> &rhs)
4996         {
4997         //      xy.parent = this;
4998
4999                 Value *value = rhs.loadValue();
5000                 storeValue(value);
5001         }
5002
5003         RValue<UInt2> UInt2::operator=(RValue<UInt2> rhs) const
5004         {
5005                 storeValue(rhs.value);
5006
5007                 return rhs;
5008         }
5009
5010         RValue<UInt2> UInt2::operator=(const UInt2 &rhs) const
5011         {
5012                 Value *value = rhs.loadValue();
5013                 storeValue(value);
5014
5015                 return RValue<UInt2>(value);
5016         }
5017
5018         RValue<UInt2> UInt2::operator=(const Reference<UInt2> &rhs) const
5019         {
5020                 Value *value = rhs.loadValue();
5021                 storeValue(value);
5022
5023                 return RValue<UInt2>(value);
5024         }
5025
5026         RValue<UInt2> operator+(RValue<UInt2> lhs, RValue<UInt2> rhs)
5027         {
5028                 if(CPUID::supportsMMX2())
5029                 {
5030                         return As<UInt2>(x86::paddd(As<Int2>(lhs), As<Int2>(rhs)));
5031                 }
5032                 else
5033                 {
5034                         return RValue<UInt2>(Nucleus::createAdd(lhs.value, rhs.value));
5035                 }
5036         }
5037
5038         RValue<UInt2> operator-(RValue<UInt2> lhs, RValue<UInt2> rhs)
5039         {
5040                 if(CPUID::supportsMMX2())
5041                 {
5042                         return As<UInt2>(x86::psubd(As<Int2>(lhs), As<Int2>(rhs)));
5043                 }
5044                 else
5045                 {
5046                         return RValue<UInt2>(Nucleus::createSub(lhs.value, rhs.value));
5047                 }
5048         }
5049
5050 //      RValue<UInt2> operator*(RValue<UInt2> lhs, RValue<UInt2> rhs)
5051 //      {
5052 //              return RValue<UInt2>(Nucleus::createMul(lhs.value, rhs.value));
5053 //      }
5054
5055 //      RValue<UInt2> operator/(RValue<UInt2> lhs, RValue<UInt2> rhs)
5056 //      {
5057 //              return RValue<UInt2>(Nucleus::createUDiv(lhs.value, rhs.value));
5058 //      }
5059
5060 //      RValue<UInt2> operator%(RValue<UInt2> lhs, RValue<UInt2> rhs)
5061 //      {
5062 //              return RValue<UInt2>(Nucleus::createURem(lhs.value, rhs.value));
5063 //      }
5064
5065         RValue<UInt2> operator&(RValue<UInt2> lhs, RValue<UInt2> rhs)
5066         {
5067                 if(CPUID::supportsMMX2())
5068                 {
5069                         return As<UInt2>(x86::pand(As<Short4>(lhs), As<Short4>(rhs)));
5070                 }
5071                 else
5072                 {
5073                         return RValue<UInt2>(Nucleus::createAnd(lhs.value, rhs.value));
5074                 }
5075         }
5076
5077         RValue<UInt2> operator|(RValue<UInt2> lhs, RValue<UInt2> rhs)
5078         {
5079                 if(CPUID::supportsMMX2())
5080                 {
5081                         return As<UInt2>(x86::por(As<Short4>(lhs), As<Short4>(rhs)));
5082                 }
5083                 else
5084                 {
5085                         return RValue<UInt2>(Nucleus::createOr(lhs.value, rhs.value));
5086                 }
5087         }
5088
5089         RValue<UInt2> operator^(RValue<UInt2> lhs, RValue<UInt2> rhs)
5090         {
5091                 if(CPUID::supportsMMX2())
5092                 {
5093                         return As<UInt2>(x86::pxor(As<Short4>(lhs), As<Short4>(rhs)));
5094                 }
5095                 else
5096                 {
5097                         return RValue<UInt2>(Nucleus::createXor(lhs.value, rhs.value));
5098                 }
5099         }
5100
5101         RValue<UInt2> operator<<(RValue<UInt2> lhs, unsigned char rhs)
5102         {
5103         //      return RValue<UInt2>(Nucleus::createShl(lhs.value, rhs.value));
5104
5105                 return As<UInt2>(x86::pslld(As<Int2>(lhs), rhs));
5106         }
5107
5108         RValue<UInt2> operator>>(RValue<UInt2> lhs, unsigned char rhs)
5109         {
5110         //      return RValue<UInt2>(Nucleus::createLShr(lhs.value, rhs.value));
5111
5112                 return x86::psrld(lhs, rhs);
5113         }
5114
5115         RValue<UInt2> operator<<(RValue<UInt2> lhs, RValue<Long1> rhs)
5116         {
5117         //      return RValue<UInt2>(Nucleus::createShl(lhs.value, rhs.value));
5118
5119                 return As<UInt2>(x86::pslld(As<Int2>(lhs), rhs));
5120         }
5121
5122         RValue<UInt2> operator>>(RValue<UInt2> lhs, RValue<Long1> rhs)
5123         {
5124         //      return RValue<UInt2>(Nucleus::createLShr(lhs.value, rhs.value));
5125
5126                 return x86::psrld(lhs, rhs);
5127         }
5128
5129         RValue<UInt2> operator+=(const UInt2 &lhs, RValue<UInt2> rhs)
5130         {
5131                 return lhs = lhs + rhs;
5132         }
5133
5134         RValue<UInt2> operator-=(const UInt2 &lhs, RValue<UInt2> rhs)
5135         {
5136                 return lhs = lhs - rhs;
5137         }
5138
5139 //      RValue<UInt2> operator*=(const UInt2 &lhs, RValue<UInt2> rhs)
5140 //      {
5141 //              return lhs = lhs * rhs;
5142 //      }
5143
5144 //      RValue<UInt2> operator/=(const UInt2 &lhs, RValue<UInt2> rhs)
5145 //      {
5146 //              return lhs = lhs / rhs;
5147 //      }
5148
5149 //      RValue<UInt2> operator%=(const UInt2 &lhs, RValue<UInt2> rhs)
5150 //      {
5151 //              return lhs = lhs % rhs;
5152 //      }
5153
5154         RValue<UInt2> operator&=(const UInt2 &lhs, RValue<UInt2> rhs)
5155         {
5156                 return lhs = lhs & rhs;
5157         }
5158
5159         RValue<UInt2> operator|=(const UInt2 &lhs, RValue<UInt2> rhs)
5160         {
5161                 return lhs = lhs | rhs;
5162         }
5163
5164         RValue<UInt2> operator^=(const UInt2 &lhs, RValue<UInt2> rhs)
5165         {
5166                 return lhs = lhs ^ rhs;
5167         }
5168
5169         RValue<UInt2> operator<<=(const UInt2 &lhs, unsigned char rhs)
5170         {
5171                 return lhs = lhs << rhs;
5172         }
5173
5174         RValue<UInt2> operator>>=(const UInt2 &lhs, unsigned char rhs)
5175         {
5176                 return lhs = lhs >> rhs;
5177         }
5178
5179         RValue<UInt2> operator<<=(const UInt2 &lhs, RValue<Long1> rhs)
5180         {
5181                 return lhs = lhs << rhs;
5182         }
5183
5184         RValue<UInt2> operator>>=(const UInt2 &lhs, RValue<Long1> rhs)
5185         {
5186                 return lhs = lhs >> rhs;
5187         }
5188
5189 //      RValue<UInt2> operator+(RValue<UInt2> val)
5190 //      {
5191 //              return val;
5192 //      }
5193
5194 //      RValue<UInt2> operator-(RValue<UInt2> val)
5195 //      {
5196 //              return RValue<UInt2>(Nucleus::createNeg(val.value));
5197 //      }
5198
5199         RValue<UInt2> operator~(RValue<UInt2> val)
5200         {
5201                 if(CPUID::supportsMMX2())
5202                 {
5203                         return val ^ UInt2(0xFFFFFFFF, 0xFFFFFFFF);
5204                 }
5205                 else
5206                 {
5207                         return RValue<UInt2>(Nucleus::createNot(val.value));
5208                 }
5209         }
5210
5211         Type *UInt2::getType()
5212         {
5213                 if(CPUID::supportsMMX2())
5214                 {
5215                         return MMX::getType();
5216                 }
5217                 else
5218                 {
5219                         return T(VectorType::get(UInt::getType(), 2));
5220                 }
5221         }
5222
5223         Int4::Int4(RValue<Byte4> cast)
5224         {
5225                 Value *x = Nucleus::createBitCast(cast.value, Int::getType());
5226                 Value *a = Nucleus::createInsertElement(V(UndefValue::get(Int4::getType())), x, 0);
5227
5228                 Value *e;
5229
5230                 if (CPUID::supportsSSE4_1())
5231                 {
5232                         e = x86::pmovzxbd(RValue<Int4>(a)).value;
5233                 }
5234                 else
5235                 {
5236                         Constant *swizzle[16];
5237                         swizzle[0] = Nucleus::createConstantInt(0);
5238                         swizzle[1] = Nucleus::createConstantInt(16);
5239                         swizzle[2] = Nucleus::createConstantInt(1);
5240                         swizzle[3] = Nucleus::createConstantInt(17);
5241                         swizzle[4] = Nucleus::createConstantInt(2);
5242                         swizzle[5] = Nucleus::createConstantInt(18);
5243                         swizzle[6] = Nucleus::createConstantInt(3);
5244                         swizzle[7] = Nucleus::createConstantInt(19);
5245                         swizzle[8] = Nucleus::createConstantInt(4);
5246                         swizzle[9] = Nucleus::createConstantInt(20);
5247                         swizzle[10] = Nucleus::createConstantInt(5);
5248                         swizzle[11] = Nucleus::createConstantInt(21);
5249                         swizzle[12] = Nucleus::createConstantInt(6);
5250                         swizzle[13] = Nucleus::createConstantInt(22);
5251                         swizzle[14] = Nucleus::createConstantInt(7);
5252                         swizzle[15] = Nucleus::createConstantInt(23);
5253
5254                         Value *b = Nucleus::createBitCast(a, Byte16::getType());
5255                         Value *c = Nucleus::createShuffleVector(b, V(Nucleus::createNullValue(Byte16::getType())), V(Nucleus::createConstantVector(swizzle, 16)));
5256
5257                         Constant *swizzle2[8];
5258                         swizzle2[0] = Nucleus::createConstantInt(0);
5259                         swizzle2[1] = Nucleus::createConstantInt(8);
5260                         swizzle2[2] = Nucleus::createConstantInt(1);
5261                         swizzle2[3] = Nucleus::createConstantInt(9);
5262                         swizzle2[4] = Nucleus::createConstantInt(2);
5263                         swizzle2[5] = Nucleus::createConstantInt(10);
5264                         swizzle2[6] = Nucleus::createConstantInt(3);
5265                         swizzle2[7] = Nucleus::createConstantInt(11);
5266
5267                         Value *d = Nucleus::createBitCast(c, Short8::getType());
5268                         e = Nucleus::createShuffleVector(d, V(Nucleus::createNullValue(Short8::getType())), V(Nucleus::createConstantVector(swizzle2, 8)));
5269                 }
5270
5271                 Value *f = Nucleus::createBitCast(e, Int4::getType());
5272                 storeValue(f);
5273         }
5274
5275         Int4::Int4(RValue<SByte4> cast)
5276         {
5277                 Value *x = Nucleus::createBitCast(cast.value, Int::getType());
5278                 Value *a = Nucleus::createInsertElement(V(UndefValue::get(Int4::getType())), x, 0);
5279
5280                 Value *g;
5281
5282                 if (CPUID::supportsSSE4_1())
5283                 {
5284                         g = x86::pmovsxbd(RValue<Int4>(a)).value;
5285                 }
5286                 else
5287                 {
5288                         Constant *swizzle[16];
5289                         swizzle[0] = Nucleus::createConstantInt(0);
5290                         swizzle[1] = Nucleus::createConstantInt(0);
5291                         swizzle[2] = Nucleus::createConstantInt(1);
5292                         swizzle[3] = Nucleus::createConstantInt(1);
5293                         swizzle[4] = Nucleus::createConstantInt(2);
5294                         swizzle[5] = Nucleus::createConstantInt(2);
5295                         swizzle[6] = Nucleus::createConstantInt(3);
5296                         swizzle[7] = Nucleus::createConstantInt(3);
5297                         swizzle[8] = Nucleus::createConstantInt(4);
5298                         swizzle[9] = Nucleus::createConstantInt(4);
5299                         swizzle[10] = Nucleus::createConstantInt(5);
5300                         swizzle[11] = Nucleus::createConstantInt(5);
5301                         swizzle[12] = Nucleus::createConstantInt(6);
5302                         swizzle[13] = Nucleus::createConstantInt(6);
5303                         swizzle[14] = Nucleus::createConstantInt(7);
5304                         swizzle[15] = Nucleus::createConstantInt(7);
5305
5306                         Value *b = Nucleus::createBitCast(a, Byte16::getType());
5307                         Value *c = Nucleus::createShuffleVector(b, b, V(Nucleus::createConstantVector(swizzle, 16)));
5308
5309                         Constant *swizzle2[8];
5310                         swizzle2[0] = Nucleus::createConstantInt(0);
5311                         swizzle2[1] = Nucleus::createConstantInt(0);
5312                         swizzle2[2] = Nucleus::createConstantInt(1);
5313                         swizzle2[3] = Nucleus::createConstantInt(1);
5314                         swizzle2[4] = Nucleus::createConstantInt(2);
5315                         swizzle2[5] = Nucleus::createConstantInt(2);
5316                         swizzle2[6] = Nucleus::createConstantInt(3);
5317                         swizzle2[7] = Nucleus::createConstantInt(3);
5318
5319                         Value *d = Nucleus::createBitCast(c, Short8::getType());
5320                         Value *e = Nucleus::createShuffleVector(d, d, V(Nucleus::createConstantVector(swizzle2, 8)));
5321
5322                         Value *f = Nucleus::createBitCast(e, Int4::getType());
5323                         //      g = Nucleus::createAShr(f, Nucleus::createConstantInt(24));
5324                         g = x86::psrad(RValue<Int4>(f), 24).value;
5325                 }
5326
5327                 storeValue(g);
5328         }
5329
5330         Int4::Int4(RValue<Float4> cast)
5331         {
5332         //      xyzw.parent = this;
5333
5334                 Value *xyzw = Nucleus::createFPToSI(cast.value, Int4::getType());
5335
5336                 storeValue(xyzw);
5337         }
5338
5339         Int4::Int4(RValue<Short4> cast)
5340         {
5341                 Value *long2 = V(UndefValue::get(Long2::getType()));
5342                 Value *element = Nucleus::createBitCast(cast.value, Long::getType());
5343                 long2 = Nucleus::createInsertElement(long2, element, 0);
5344                 RValue<Int4> vector = RValue<Int4>(Nucleus::createBitCast(long2, Int4::getType()));
5345
5346                 if(CPUID::supportsSSE4_1())
5347                 {
5348                         storeValue(x86::pmovsxwd(vector).value);
5349                 }
5350                 else
5351                 {
5352                         Value *b = Nucleus::createBitCast(vector.value, Short8::getType());
5353
5354                         Constant *swizzle[8];
5355                         swizzle[0] = Nucleus::createConstantInt(0);
5356                         swizzle[1] = Nucleus::createConstantInt(0);
5357                         swizzle[2] = Nucleus::createConstantInt(1);
5358                         swizzle[3] = Nucleus::createConstantInt(1);
5359                         swizzle[4] = Nucleus::createConstantInt(2);
5360                         swizzle[5] = Nucleus::createConstantInt(2);
5361                         swizzle[6] = Nucleus::createConstantInt(3);
5362                         swizzle[7] = Nucleus::createConstantInt(3);
5363
5364                         Value *c = Nucleus::createShuffleVector(b, b, V(Nucleus::createConstantVector(swizzle, 8)));
5365                         Value *d = Nucleus::createBitCast(c, Int4::getType());
5366                         storeValue(d);
5367
5368                         // Each Short is packed into each Int in the (Short | Short) format.
5369                         // Shifting by 16 will retrieve the original Short value.
5370                         // Shitfing an Int will propagate the sign bit, which will work
5371                         // for both positive and negative values of a Short.
5372                         *this >>= 16;
5373                 }
5374         }
5375
5376         Int4::Int4(RValue<UShort4> cast)
5377         {
5378                 Value *long2 = V(UndefValue::get(Long2::getType()));
5379                 Value *element = Nucleus::createBitCast(cast.value, Long::getType());
5380                 long2 = Nucleus::createInsertElement(long2, element, 0);
5381                 RValue<Int4> vector = RValue<Int4>(Nucleus::createBitCast(long2, Int4::getType()));
5382
5383                 if(CPUID::supportsSSE4_1())
5384                 {
5385                         storeValue(x86::pmovzxwd(RValue<Int4>(vector)).value);
5386                 }
5387                 else
5388                 {
5389                         Value *b = Nucleus::createBitCast(vector.value, Short8::getType());
5390
5391                         Constant *swizzle[8];
5392                         swizzle[0] = Nucleus::createConstantInt(0);
5393                         swizzle[1] = Nucleus::createConstantInt(8);
5394                         swizzle[2] = Nucleus::createConstantInt(1);
5395                         swizzle[3] = Nucleus::createConstantInt(9);
5396                         swizzle[4] = Nucleus::createConstantInt(2);
5397                         swizzle[5] = Nucleus::createConstantInt(10);
5398                         swizzle[6] = Nucleus::createConstantInt(3);
5399                         swizzle[7] = Nucleus::createConstantInt(11);
5400
5401                         Value *c = Nucleus::createShuffleVector(b, V(Nucleus::createNullValue(Short8::getType())), V(Nucleus::createConstantVector(swizzle, 8)));
5402                         Value *d = Nucleus::createBitCast(c, Int4::getType());
5403                         storeValue(d);
5404                 }
5405         }
5406
5407         Int4::Int4()
5408         {
5409         //      xyzw.parent = this;
5410         }
5411
5412         Int4::Int4(int xyzw)
5413         {
5414                 constant(xyzw, xyzw, xyzw, xyzw);
5415         }
5416
5417         Int4::Int4(int x, int yzw)
5418         {
5419                 constant(x, yzw, yzw, yzw);
5420         }
5421
5422         Int4::Int4(int x, int y, int zw)
5423         {
5424                 constant(x, y, zw, zw);
5425         }
5426
5427         Int4::Int4(int x, int y, int z, int w)
5428         {
5429                 constant(x, y, z, w);
5430         }
5431
5432         void Int4::constant(int x, int y, int z, int w)
5433         {
5434         //      xyzw.parent = this;
5435
5436                 Constant *constantVector[4];
5437                 constantVector[0] = Nucleus::createConstantInt(x);
5438                 constantVector[1] = Nucleus::createConstantInt(y);
5439                 constantVector[2] = Nucleus::createConstantInt(z);
5440                 constantVector[3] = Nucleus::createConstantInt(w);
5441
5442                 storeValue(Nucleus::createConstantVector(constantVector, 4));
5443         }
5444
5445         Int4::Int4(RValue<Int4> rhs)
5446         {
5447         //      xyzw.parent = this;
5448
5449                 storeValue(rhs.value);
5450         }
5451
5452         Int4::Int4(const Int4 &rhs)
5453         {
5454         //      xyzw.parent = this;
5455
5456                 Value *value = rhs.loadValue();
5457                 storeValue(value);
5458         }
5459
5460         Int4::Int4(const Reference<Int4> &rhs)
5461         {
5462         //      xyzw.parent = this;
5463
5464                 Value *value = rhs.loadValue();
5465                 storeValue(value);
5466         }
5467
5468         Int4::Int4(RValue<UInt4> rhs)
5469         {
5470         //      xyzw.parent = this;
5471
5472                 storeValue(rhs.value);
5473         }
5474
5475         Int4::Int4(const UInt4 &rhs)
5476         {
5477         //      xyzw.parent = this;
5478
5479                 Value *value = rhs.loadValue();
5480                 storeValue(value);
5481         }
5482
5483         Int4::Int4(const Reference<UInt4> &rhs)
5484         {
5485         //      xyzw.parent = this;
5486
5487                 Value *value = rhs.loadValue();
5488                 storeValue(value);
5489         }
5490
5491         Int4::Int4(RValue<Int2> lo, RValue<Int2> hi)
5492         {
5493         //      xyzw.parent = this;
5494
5495                 Value *loLong = Nucleus::createBitCast(lo.value, Long::getType());
5496                 Value *hiLong = Nucleus::createBitCast(hi.value, Long::getType());
5497
5498                 Value *long2 = V(UndefValue::get(Long2::getType()));
5499                 long2 = Nucleus::createInsertElement(long2, loLong, 0);
5500                 long2 = Nucleus::createInsertElement(long2, hiLong, 1);
5501                 Value *int4 = Nucleus::createBitCast(long2, Int4::getType());
5502
5503                 storeValue(int4);
5504         }
5505
5506         Int4::Int4(RValue<Int> rhs)
5507         {
5508         //      xyzw.parent = this;
5509
5510                 Value *vector = loadValue();
5511                 Value *insert = Nucleus::createInsertElement(vector, rhs.value, 0);
5512
5513                 Constant *swizzle[4];
5514                 swizzle[0] = Nucleus::createConstantInt(0);
5515                 swizzle[1] = Nucleus::createConstantInt(0);
5516                 swizzle[2] = Nucleus::createConstantInt(0);
5517                 swizzle[3] = Nucleus::createConstantInt(0);
5518
5519                 Value *replicate = Nucleus::createShuffleVector(insert, V(UndefValue::get(Int4::getType())), V(Nucleus::createConstantVector(swizzle, 4)));
5520
5521                 storeValue(replicate);
5522         }
5523
5524         Int4::Int4(const Int &rhs)
5525         {
5526         //      xyzw.parent = this;
5527
5528                 *this = RValue<Int>(rhs.loadValue());
5529         }
5530
5531         Int4::Int4(const Reference<Int> &rhs)
5532         {
5533         //      xyzw.parent = this;
5534
5535                 *this = RValue<Int>(rhs.loadValue());
5536         }
5537
5538         RValue<Int4> Int4::operator=(RValue<Int4> rhs) const
5539         {
5540                 storeValue(rhs.value);
5541
5542                 return rhs;
5543         }
5544
5545         RValue<Int4> Int4::operator=(const Int4 &rhs) const
5546         {
5547                 Value *value = rhs.loadValue();
5548                 storeValue(value);
5549
5550                 return RValue<Int4>(value);
5551         }
5552
5553         RValue<Int4> Int4::operator=(const Reference<Int4> &rhs) const
5554         {
5555                 Value *value = rhs.loadValue();
5556                 storeValue(value);
5557
5558                 return RValue<Int4>(value);
5559         }
5560
5561         RValue<Int4> operator+(RValue<Int4> lhs, RValue<Int4> rhs)
5562         {
5563                 return RValue<Int4>(Nucleus::createAdd(lhs.value, rhs.value));
5564         }
5565
5566         RValue<Int4> operator-(RValue<Int4> lhs, RValue<Int4> rhs)
5567         {
5568                 return RValue<Int4>(Nucleus::createSub(lhs.value, rhs.value));
5569         }
5570
5571         RValue<Int4> operator*(RValue<Int4> lhs, RValue<Int4> rhs)
5572         {
5573                 return RValue<Int4>(Nucleus::createMul(lhs.value, rhs.value));
5574         }
5575
5576         RValue<Int4> operator/(RValue<Int4> lhs, RValue<Int4> rhs)
5577         {
5578                 return RValue<Int4>(Nucleus::createSDiv(lhs.value, rhs.value));
5579         }
5580
5581         RValue<Int4> operator%(RValue<Int4> lhs, RValue<Int4> rhs)
5582         {
5583                 return RValue<Int4>(Nucleus::createSRem(lhs.value, rhs.value));
5584         }
5585
5586         RValue<Int4> operator&(RValue<Int4> lhs, RValue<Int4> rhs)
5587         {
5588                 return RValue<Int4>(Nucleus::createAnd(lhs.value, rhs.value));
5589         }
5590
5591         RValue<Int4> operator|(RValue<Int4> lhs, RValue<Int4> rhs)
5592         {
5593                 return RValue<Int4>(Nucleus::createOr(lhs.value, rhs.value));
5594         }
5595
5596         RValue<Int4> operator^(RValue<Int4> lhs, RValue<Int4> rhs)
5597         {
5598                 return RValue<Int4>(Nucleus::createXor(lhs.value, rhs.value));
5599         }
5600
5601         RValue<Int4> operator<<(RValue<Int4> lhs, unsigned char rhs)
5602         {
5603                 return x86::pslld(lhs, rhs);
5604         }
5605
5606         RValue<Int4> operator>>(RValue<Int4> lhs, unsigned char rhs)
5607         {
5608                 return x86::psrad(lhs, rhs);
5609         }
5610
5611         RValue<Int4> operator<<(RValue<Int4> lhs, RValue<Int4> rhs)
5612         {
5613                 return RValue<Int4>(Nucleus::createShl(lhs.value, rhs.value));
5614         }
5615
5616         RValue<Int4> operator>>(RValue<Int4> lhs, RValue<Int4> rhs)
5617         {
5618                 return RValue<Int4>(Nucleus::createAShr(lhs.value, rhs.value));
5619         }
5620
5621         RValue<Int4> operator+=(const Int4 &lhs, RValue<Int4> rhs)
5622         {
5623                 return lhs = lhs + rhs;
5624         }
5625
5626         RValue<Int4> operator-=(const Int4 &lhs, RValue<Int4> rhs)
5627         {
5628                 return lhs = lhs - rhs;
5629         }
5630
5631         RValue<Int4> operator*=(const Int4 &lhs, RValue<Int4> rhs)
5632         {
5633                 return lhs = lhs * rhs;
5634         }
5635
5636 //      RValue<Int4> operator/=(const Int4 &lhs, RValue<Int4> rhs)
5637 //      {
5638 //              return lhs = lhs / rhs;
5639 //      }
5640
5641 //      RValue<Int4> operator%=(const Int4 &lhs, RValue<Int4> rhs)
5642 //      {
5643 //              return lhs = lhs % rhs;
5644 //      }
5645
5646         RValue<Int4> operator&=(const Int4 &lhs, RValue<Int4> rhs)
5647         {
5648                 return lhs = lhs & rhs;
5649         }
5650
5651         RValue<Int4> operator|=(const Int4 &lhs, RValue<Int4> rhs)
5652         {
5653                 return lhs = lhs | rhs;
5654         }
5655
5656         RValue<Int4> operator^=(const Int4 &lhs, RValue<Int4> rhs)
5657         {
5658                 return lhs = lhs ^ rhs;
5659         }
5660
5661         RValue<Int4> operator<<=(const Int4 &lhs, unsigned char rhs)
5662         {
5663                 return lhs = lhs << rhs;
5664         }
5665
5666         RValue<Int4> operator>>=(const Int4 &lhs, unsigned char rhs)
5667         {
5668                 return lhs = lhs >> rhs;
5669         }
5670
5671         RValue<Int4> operator+(RValue<Int4> val)
5672         {
5673                 return val;
5674         }
5675
5676         RValue<Int4> operator-(RValue<Int4> val)
5677         {
5678                 return RValue<Int4>(Nucleus::createNeg(val.value));
5679         }
5680
5681         RValue<Int4> operator~(RValue<Int4> val)
5682         {
5683                 return RValue<Int4>(Nucleus::createNot(val.value));
5684         }
5685
5686         RValue<Int4> CmpEQ(RValue<Int4> x, RValue<Int4> y)
5687         {
5688                 // FIXME: An LLVM bug causes SExt(ICmpCC()) to produce 0 or 1 instead of 0 or ~0
5689                 //        Restore the following line when LLVM is updated to a version where this issue is fixed.
5690                 // return RValue<Int4>(Nucleus::createSExt(Nucleus::createICmpEQ(x.value, y.value), Int4::getType()));
5691                 return RValue<Int4>(Nucleus::createSExt(Nucleus::createICmpNE(x.value, y.value), Int4::getType())) ^ Int4(0xFFFFFFFF);
5692         }
5693
5694         RValue<Int4> CmpLT(RValue<Int4> x, RValue<Int4> y)
5695         {
5696                 return RValue<Int4>(Nucleus::createSExt(Nucleus::createICmpSLT(x.value, y.value), Int4::getType()));
5697         }
5698
5699         RValue<Int4> CmpLE(RValue<Int4> x, RValue<Int4> y)
5700         {
5701                 // FIXME: An LLVM bug causes SExt(ICmpCC()) to produce 0 or 1 instead of 0 or ~0
5702                 //        Restore the following line when LLVM is updated to a version where this issue is fixed.
5703                 // return RValue<Int4>(Nucleus::createSExt(Nucleus::createICmpSLE(x.value, y.value), Int4::getType()));
5704                 return RValue<Int4>(Nucleus::createSExt(Nucleus::createICmpSGT(x.value, y.value), Int4::getType())) ^ Int4(0xFFFFFFFF);
5705         }
5706
5707         RValue<Int4> CmpNEQ(RValue<Int4> x, RValue<Int4> y)
5708         {
5709                 return RValue<Int4>(Nucleus::createSExt(Nucleus::createICmpNE(x.value, y.value), Int4::getType()));
5710         }
5711
5712         RValue<Int4> CmpNLT(RValue<Int4> x, RValue<Int4> y)
5713         {
5714                 // FIXME: An LLVM bug causes SExt(ICmpCC()) to produce 0 or 1 instead of 0 or ~0
5715                 //        Restore the following line when LLVM is updated to a version where this issue is fixed.
5716                 // return RValue<Int4>(Nucleus::createSExt(Nucleus::createICmpSGE(x.value, y.value), Int4::getType()));
5717                 return RValue<Int4>(Nucleus::createSExt(Nucleus::createICmpSLT(x.value, y.value), Int4::getType())) ^ Int4(0xFFFFFFFF);
5718         }
5719
5720         RValue<Int4> CmpNLE(RValue<Int4> x, RValue<Int4> y)
5721         {
5722                 return RValue<Int4>(Nucleus::createSExt(Nucleus::createICmpSGT(x.value, y.value), Int4::getType()));
5723         }
5724
5725         RValue<Int4> Max(RValue<Int4> x, RValue<Int4> y)
5726         {
5727                 if(CPUID::supportsSSE4_1())
5728                 {
5729                         return x86::pmaxsd(x, y);
5730                 }
5731                 else
5732                 {
5733                         RValue<Int4> greater = CmpNLE(x, y);
5734                         return x & greater | y & ~greater;
5735                 }
5736         }
5737
5738         RValue<Int4> Min(RValue<Int4> x, RValue<Int4> y)
5739         {
5740                 if(CPUID::supportsSSE4_1())
5741                 {
5742                         return x86::pminsd(x, y);
5743                 }
5744                 else
5745                 {
5746                         RValue<Int4> less = CmpLT(x, y);
5747                         return x & less | y & ~less;
5748                 }
5749         }
5750
5751         RValue<Int4> RoundInt(RValue<Float4> cast)
5752         {
5753                 return x86::cvtps2dq(cast);
5754         }
5755
5756         RValue<Short8> Pack(RValue<Int4> x, RValue<Int4> y)
5757         {
5758                 return x86::packssdw(x, y);
5759         }
5760
5761         RValue<Int> Extract(RValue<Int4> x, int i)
5762         {
5763                 return RValue<Int>(Nucleus::createExtractElement(x.value, i));
5764         }
5765
5766         RValue<Int4> Insert(RValue<Int4> x, RValue<Int> element, int i)
5767         {
5768                 return RValue<Int4>(Nucleus::createInsertElement(x.value, element.value, i));
5769         }
5770
5771         RValue<Int> SignMask(RValue<Int4> x)
5772         {
5773                 return x86::movmskps(As<Float4>(x));
5774         }
5775
5776         RValue<Int4> Swizzle(RValue<Int4> x, unsigned char select)
5777         {
5778                 return RValue<Int4>(Nucleus::createSwizzle(x.value, select));
5779         }
5780
5781         Type *Int4::getType()
5782         {
5783                 return T(VectorType::get(Int::getType(), 4));
5784         }
5785
5786         UInt4::UInt4(RValue<Float4> cast)
5787         {
5788         //      xyzw.parent = this;
5789
5790                 // Note: createFPToUI is broken, must perform conversion using createFPtoSI
5791                 // Value *xyzw = Nucleus::createFPToUI(cast.value, UInt4::getType());
5792
5793                 // Smallest positive value representable in UInt, but not in Int
5794                 const unsigned int ustart = 0x80000000u;
5795                 const float ustartf = float(ustart);
5796
5797                 // Check if the value can be represented as an Int
5798                 Int4 uiValue = CmpNLT(cast, Float4(ustartf));
5799                 // If the value is too large, subtract ustart and re-add it after conversion.
5800                 uiValue = (uiValue & As<Int4>(As<UInt4>(Int4(cast - Float4(ustartf))) + UInt4(ustart))) |
5801                 // Otherwise, just convert normally
5802                           (~uiValue & Int4(cast));
5803                 // If the value is negative, store 0, otherwise store the result of the conversion
5804                 storeValue((~(As<Int4>(cast) >> 31) & uiValue).value);
5805         }
5806
5807         UInt4::UInt4()
5808         {
5809         //      xyzw.parent = this;
5810         }
5811
5812         UInt4::UInt4(int xyzw)
5813         {
5814                 constant(xyzw, xyzw, xyzw, xyzw);
5815         }
5816
5817         UInt4::UInt4(int x, int yzw)
5818         {
5819                 constant(x, yzw, yzw, yzw);
5820         }
5821
5822         UInt4::UInt4(int x, int y, int zw)
5823         {
5824                 constant(x, y, zw, zw);
5825         }
5826
5827         UInt4::UInt4(int x, int y, int z, int w)
5828         {
5829                 constant(x, y, z, w);
5830         }
5831
5832         void UInt4::constant(int x, int y, int z, int w)
5833         {
5834         //      xyzw.parent = this;
5835
5836                 Constant *constantVector[4];
5837                 constantVector[0] = Nucleus::createConstantInt(x);
5838                 constantVector[1] = Nucleus::createConstantInt(y);
5839                 constantVector[2] = Nucleus::createConstantInt(z);
5840                 constantVector[3] = Nucleus::createConstantInt(w);
5841
5842                 storeValue(Nucleus::createConstantVector(constantVector, 4));
5843         }
5844
5845         UInt4::UInt4(RValue<UInt4> rhs)
5846         {
5847         //      xyzw.parent = this;
5848
5849                 storeValue(rhs.value);
5850         }
5851
5852         UInt4::UInt4(const UInt4 &rhs)
5853         {
5854         //      xyzw.parent = this;
5855
5856                 Value *value = rhs.loadValue();
5857                 storeValue(value);
5858         }
5859
5860         UInt4::UInt4(const Reference<UInt4> &rhs)
5861         {
5862         //      xyzw.parent = this;
5863
5864                 Value *value = rhs.loadValue();
5865                 storeValue(value);
5866         }
5867
5868         UInt4::UInt4(RValue<Int4> rhs)
5869         {
5870         //      xyzw.parent = this;
5871
5872                 storeValue(rhs.value);
5873         }
5874
5875         UInt4::UInt4(const Int4 &rhs)
5876         {
5877         //      xyzw.parent = this;
5878
5879                 Value *value = rhs.loadValue();
5880                 storeValue(value);
5881         }
5882
5883         UInt4::UInt4(const Reference<Int4> &rhs)
5884         {
5885         //      xyzw.parent = this;
5886
5887                 Value *value = rhs.loadValue();
5888                 storeValue(value);
5889         }
5890
5891         UInt4::UInt4(RValue<UInt2> lo, RValue<UInt2> hi)
5892         {
5893                 Value *loLong = Nucleus::createBitCast(lo.value, Long::getType());
5894                 Value *hiLong = Nucleus::createBitCast(hi.value, Long::getType());
5895
5896                 Value *long2 = V(UndefValue::get(Long2::getType()));
5897                 long2 = Nucleus::createInsertElement(long2, loLong, 0);
5898                 long2 = Nucleus::createInsertElement(long2, hiLong, 1);
5899                 Value *uint4 = Nucleus::createBitCast(long2, Int4::getType());
5900
5901                 storeValue(uint4);
5902         }
5903
5904         RValue<UInt4> UInt4::operator=(RValue<UInt4> rhs) const
5905         {
5906                 storeValue(rhs.value);
5907
5908                 return rhs;
5909         }
5910
5911         RValue<UInt4> UInt4::operator=(const UInt4 &rhs) const
5912         {
5913                 Value *value = rhs.loadValue();
5914                 storeValue(value);
5915
5916                 return RValue<UInt4>(value);
5917         }
5918
5919         RValue<UInt4> UInt4::operator=(const Reference<UInt4> &rhs) const
5920         {
5921                 Value *value = rhs.loadValue();
5922                 storeValue(value);
5923
5924                 return RValue<UInt4>(value);
5925         }
5926
5927         RValue<UInt4> operator+(RValue<UInt4> lhs, RValue<UInt4> rhs)
5928         {
5929                 return RValue<UInt4>(Nucleus::createAdd(lhs.value, rhs.value));
5930         }
5931
5932         RValue<UInt4> operator-(RValue<UInt4> lhs, RValue<UInt4> rhs)
5933         {
5934                 return RValue<UInt4>(Nucleus::createSub(lhs.value, rhs.value));
5935         }
5936
5937         RValue<UInt4> operator*(RValue<UInt4> lhs, RValue<UInt4> rhs)
5938         {
5939                 return RValue<UInt4>(Nucleus::createMul(lhs.value, rhs.value));
5940         }
5941
5942         RValue<UInt4> operator/(RValue<UInt4> lhs, RValue<UInt4> rhs)
5943         {
5944                 return RValue<UInt4>(Nucleus::createUDiv(lhs.value, rhs.value));
5945         }
5946
5947         RValue<UInt4> operator%(RValue<UInt4> lhs, RValue<UInt4> rhs)
5948         {
5949                 return RValue<UInt4>(Nucleus::createURem(lhs.value, rhs.value));
5950         }
5951
5952         RValue<UInt4> operator&(RValue<UInt4> lhs, RValue<UInt4> rhs)
5953         {
5954                 return RValue<UInt4>(Nucleus::createAnd(lhs.value, rhs.value));
5955         }
5956
5957         RValue<UInt4> operator|(RValue<UInt4> lhs, RValue<UInt4> rhs)
5958         {
5959                 return RValue<UInt4>(Nucleus::createOr(lhs.value, rhs.value));
5960         }
5961
5962         RValue<UInt4> operator^(RValue<UInt4> lhs, RValue<UInt4> rhs)
5963         {
5964                 return RValue<UInt4>(Nucleus::createXor(lhs.value, rhs.value));
5965         }
5966
5967         RValue<UInt4> operator<<(RValue<UInt4> lhs, unsigned char rhs)
5968         {
5969                 return As<UInt4>(x86::pslld(As<Int4>(lhs), rhs));
5970         }
5971
5972         RValue<UInt4> operator>>(RValue<UInt4> lhs, unsigned char rhs)
5973         {
5974                 return x86::psrld(lhs, rhs);
5975         }
5976
5977         RValue<UInt4> operator<<(RValue<UInt4> lhs, RValue<UInt4> rhs)
5978         {
5979                 return RValue<UInt4>(Nucleus::createShl(lhs.value, rhs.value));
5980         }
5981
5982         RValue<UInt4> operator>>(RValue<UInt4> lhs, RValue<UInt4> rhs)
5983         {
5984                 return RValue<UInt4>(Nucleus::createLShr(lhs.value, rhs.value));
5985         }
5986
5987         RValue<UInt4> operator+=(const UInt4 &lhs, RValue<UInt4> rhs)
5988         {
5989                 return lhs = lhs + rhs;
5990         }
5991
5992         RValue<UInt4> operator-=(const UInt4 &lhs, RValue<UInt4> rhs)
5993         {
5994                 return lhs = lhs - rhs;
5995         }
5996
5997         RValue<UInt4> operator*=(const UInt4 &lhs, RValue<UInt4> rhs)
5998         {
5999                 return lhs = lhs * rhs;
6000         }
6001
6002 //      RValue<UInt4> operator/=(const UInt4 &lhs, RValue<UInt4> rhs)
6003 //      {
6004 //              return lhs = lhs / rhs;
6005 //      }
6006
6007 //      RValue<UInt4> operator%=(const UInt4 &lhs, RValue<UInt4> rhs)
6008 //      {
6009 //              return lhs = lhs % rhs;
6010 //      }
6011
6012         RValue<UInt4> operator&=(const UInt4 &lhs, RValue<UInt4> rhs)
6013         {
6014                 return lhs = lhs & rhs;
6015         }
6016
6017         RValue<UInt4> operator|=(const UInt4 &lhs, RValue<UInt4> rhs)
6018         {
6019                 return lhs = lhs | rhs;
6020         }
6021
6022         RValue<UInt4> operator^=(const UInt4 &lhs, RValue<UInt4> rhs)
6023         {
6024                 return lhs = lhs ^ rhs;
6025         }
6026
6027         RValue<UInt4> operator<<=(const UInt4 &lhs, unsigned char rhs)
6028         {
6029                 return lhs = lhs << rhs;
6030         }
6031
6032         RValue<UInt4> operator>>=(const UInt4 &lhs, unsigned char rhs)
6033         {
6034                 return lhs = lhs >> rhs;
6035         }
6036
6037         RValue<UInt4> operator+(RValue<UInt4> val)
6038         {
6039                 return val;
6040         }
6041
6042         RValue<UInt4> operator-(RValue<UInt4> val)
6043         {
6044                 return RValue<UInt4>(Nucleus::createNeg(val.value));
6045         }
6046
6047         RValue<UInt4> operator~(RValue<UInt4> val)
6048         {
6049                 return RValue<UInt4>(Nucleus::createNot(val.value));
6050         }
6051
6052         RValue<UInt4> CmpEQ(RValue<UInt4> x, RValue<UInt4> y)
6053         {
6054                 // FIXME: An LLVM bug causes SExt(ICmpCC()) to produce 0 or 1 instead of 0 or ~0
6055                 //        Restore the following line when LLVM is updated to a version where this issue is fixed.
6056                 // return RValue<UInt4>(Nucleus::createSExt(Nucleus::createICmpEQ(x.value, y.value), Int4::getType()));
6057                 return RValue<UInt4>(Nucleus::createSExt(Nucleus::createICmpNE(x.value, y.value), Int4::getType())) ^ UInt4(0xFFFFFFFF);
6058         }
6059
6060         RValue<UInt4> CmpLT(RValue<UInt4> x, RValue<UInt4> y)
6061         {
6062                 return RValue<UInt4>(Nucleus::createSExt(Nucleus::createICmpULT(x.value, y.value), Int4::getType()));
6063         }
6064
6065         RValue<UInt4> CmpLE(RValue<UInt4> x, RValue<UInt4> y)
6066         {
6067                 // FIXME: An LLVM bug causes SExt(ICmpCC()) to produce 0 or 1 instead of 0 or ~0
6068                 //        Restore the following line when LLVM is updated to a version where this issue is fixed.
6069                 // return RValue<UInt4>(Nucleus::createSExt(Nucleus::createICmpULE(x.value, y.value), Int4::getType()));
6070                 return RValue<UInt4>(Nucleus::createSExt(Nucleus::createICmpUGT(x.value, y.value), Int4::getType())) ^ UInt4(0xFFFFFFFF);
6071         }
6072
6073         RValue<UInt4> CmpNEQ(RValue<UInt4> x, RValue<UInt4> y)
6074         {
6075                 return RValue<UInt4>(Nucleus::createSExt(Nucleus::createICmpNE(x.value, y.value), Int4::getType()));
6076         }
6077
6078         RValue<UInt4> CmpNLT(RValue<UInt4> x, RValue<UInt4> y)
6079         {
6080                 // FIXME: An LLVM bug causes SExt(ICmpCC()) to produce 0 or 1 instead of 0 or ~0
6081                 //        Restore the following line when LLVM is updated to a version where this issue is fixed.
6082                 // return RValue<UInt4>(Nucleus::createSExt(Nucleus::createICmpUGE(x.value, y.value), Int4::getType()));
6083                 return RValue<UInt4>(Nucleus::createSExt(Nucleus::createICmpULT(x.value, y.value), Int4::getType())) ^ UInt4(0xFFFFFFFF);
6084         }
6085
6086         RValue<UInt4> CmpNLE(RValue<UInt4> x, RValue<UInt4> y)
6087         {
6088                 return RValue<UInt4>(Nucleus::createSExt(Nucleus::createICmpUGT(x.value, y.value), Int4::getType()));
6089         }
6090
6091         RValue<UInt4> Max(RValue<UInt4> x, RValue<UInt4> y)
6092         {
6093                 if(CPUID::supportsSSE4_1())
6094                 {
6095                         return x86::pmaxud(x, y);
6096                 }
6097                 else
6098                 {
6099                         RValue<UInt4> greater = CmpNLE(x, y);
6100                         return x & greater | y & ~greater;
6101                 }
6102         }
6103
6104         RValue<UInt4> Min(RValue<UInt4> x, RValue<UInt4> y)
6105         {
6106                 if(CPUID::supportsSSE4_1())
6107                 {
6108                         return x86::pminud(x, y);
6109                 }
6110                 else
6111                 {
6112                         RValue<UInt4> less = CmpLT(x, y);
6113                         return x & less | y & ~less;
6114                 }
6115         }
6116
6117         RValue<UShort8> Pack(RValue<UInt4> x, RValue<UInt4> y)
6118         {
6119                 return x86::packusdw(x, y);   // FIXME: Fallback required
6120         }
6121
6122         Type *UInt4::getType()
6123         {
6124                 return T(VectorType::get(UInt::getType(), 4));
6125         }
6126
6127         Float::Float(RValue<Int> cast)
6128         {
6129                 Value *integer = Nucleus::createSIToFP(cast.value, Float::getType());
6130
6131                 storeValue(integer);
6132         }
6133
6134         Float::Float()
6135         {
6136
6137         }
6138
6139         Float::Float(float x)
6140         {
6141                 storeValue(Nucleus::createConstantFloat(x));
6142         }
6143
6144         Float::Float(RValue<Float> rhs)
6145         {
6146                 storeValue(rhs.value);
6147         }
6148
6149         Float::Float(const Float &rhs)
6150         {
6151                 Value *value = rhs.loadValue();
6152                 storeValue(value);
6153         }
6154
6155         Float::Float(const Reference<Float> &rhs)
6156         {
6157                 Value *value = rhs.loadValue();
6158                 storeValue(value);
6159         }
6160
6161         RValue<Float> Float::operator=(RValue<Float> rhs) const
6162         {
6163                 storeValue(rhs.value);
6164
6165                 return rhs;
6166         }
6167
6168         RValue<Float> Float::operator=(const Float &rhs) const
6169         {
6170                 Value *value = rhs.loadValue();
6171                 storeValue(value);
6172
6173                 return RValue<Float>(value);
6174         }
6175
6176         RValue<Float> Float::operator=(const Reference<Float> &rhs) const
6177         {
6178                 Value *value = rhs.loadValue();
6179                 storeValue(value);
6180
6181                 return RValue<Float>(value);
6182         }
6183
6184         RValue<Float> operator+(RValue<Float> lhs, RValue<Float> rhs)
6185         {
6186                 return RValue<Float>(Nucleus::createFAdd(lhs.value, rhs.value));
6187         }
6188
6189         RValue<Float> operator-(RValue<Float> lhs, RValue<Float> rhs)
6190         {
6191                 return RValue<Float>(Nucleus::createFSub(lhs.value, rhs.value));
6192         }
6193
6194         RValue<Float> operator*(RValue<Float> lhs, RValue<Float> rhs)
6195         {
6196                 return RValue<Float>(Nucleus::createFMul(lhs.value, rhs.value));
6197         }
6198
6199         RValue<Float> operator/(RValue<Float> lhs, RValue<Float> rhs)
6200         {
6201                 return RValue<Float>(Nucleus::createFDiv(lhs.value, rhs.value));
6202         }
6203
6204         RValue<Float> operator+=(const Float &lhs, RValue<Float> rhs)
6205         {
6206                 return lhs = lhs + rhs;
6207         }
6208
6209         RValue<Float> operator-=(const Float &lhs, RValue<Float> rhs)
6210         {
6211                 return lhs = lhs - rhs;
6212         }
6213
6214         RValue<Float> operator*=(const Float &lhs, RValue<Float> rhs)
6215         {
6216                 return lhs = lhs * rhs;
6217         }
6218
6219         RValue<Float> operator/=(const Float &lhs, RValue<Float> rhs)
6220         {
6221                 return lhs = lhs / rhs;
6222         }
6223
6224         RValue<Float> operator+(RValue<Float> val)
6225         {
6226                 return val;
6227         }
6228
6229         RValue<Float> operator-(RValue<Float> val)
6230         {
6231                 return RValue<Float>(Nucleus::createFNeg(val.value));
6232         }
6233
6234         RValue<Bool> operator<(RValue<Float> lhs, RValue<Float> rhs)
6235         {
6236                 return RValue<Bool>(Nucleus::createFCmpOLT(lhs.value, rhs.value));
6237         }
6238
6239         RValue<Bool> operator<=(RValue<Float> lhs, RValue<Float> rhs)
6240         {
6241                 return RValue<Bool>(Nucleus::createFCmpOLE(lhs.value, rhs.value));
6242         }
6243
6244         RValue<Bool> operator>(RValue<Float> lhs, RValue<Float> rhs)
6245         {
6246                 return RValue<Bool>(Nucleus::createFCmpOGT(lhs.value, rhs.value));
6247         }
6248
6249         RValue<Bool> operator>=(RValue<Float> lhs, RValue<Float> rhs)
6250         {
6251                 return RValue<Bool>(Nucleus::createFCmpOGE(lhs.value, rhs.value));
6252         }
6253
6254         RValue<Bool> operator!=(RValue<Float> lhs, RValue<Float> rhs)
6255         {
6256                 return RValue<Bool>(Nucleus::createFCmpONE(lhs.value, rhs.value));
6257         }
6258
6259         RValue<Bool> operator==(RValue<Float> lhs, RValue<Float> rhs)
6260         {
6261                 return RValue<Bool>(Nucleus::createFCmpOEQ(lhs.value, rhs.value));
6262         }
6263
6264         RValue<Float> Abs(RValue<Float> x)
6265         {
6266                 return IfThenElse(x > 0.0f, x, -x);
6267         }
6268
6269         RValue<Float> Max(RValue<Float> x, RValue<Float> y)
6270         {
6271                 return IfThenElse(x > y, x, y);
6272         }
6273
6274         RValue<Float> Min(RValue<Float> x, RValue<Float> y)
6275         {
6276                 return IfThenElse(x < y, x, y);
6277         }
6278
6279         RValue<Float> Rcp_pp(RValue<Float> x, bool exactAtPow2)
6280         {
6281                 if(exactAtPow2)
6282                 {
6283                         // rcpss uses a piecewise-linear approximation which minimizes the relative error
6284                         // but is not exact at power-of-two values. Rectify by multiplying by the inverse.
6285                         return x86::rcpss(x) * Float(1.0f / _mm_cvtss_f32(_mm_rcp_ss(_mm_set_ps1(1.0f))));
6286                 }
6287                 else
6288                 {
6289                         return x86::rcpss(x);
6290                 }
6291         }
6292
6293         RValue<Float> RcpSqrt_pp(RValue<Float> x)
6294         {
6295                 return x86::rsqrtss(x);
6296         }
6297
6298         RValue<Float> Sqrt(RValue<Float> x)
6299         {
6300                 return x86::sqrtss(x);
6301         }
6302
6303         RValue<Float> Round(RValue<Float> x)
6304         {
6305                 if(CPUID::supportsSSE4_1())
6306                 {
6307                         return x86::roundss(x, 0);
6308                 }
6309                 else
6310                 {
6311                         return Float4(Round(Float4(x))).x;
6312                 }
6313         }
6314
6315         RValue<Float> Trunc(RValue<Float> x)
6316         {
6317                 if(CPUID::supportsSSE4_1())
6318                 {
6319                         return x86::roundss(x, 3);
6320                 }
6321                 else
6322                 {
6323                         return Float(Int(x));   // Rounded toward zero
6324                 }
6325         }
6326
6327         RValue<Float> Frac(RValue<Float> x)
6328         {
6329                 if(CPUID::supportsSSE4_1())
6330                 {
6331                         return x - x86::floorss(x);
6332                 }
6333                 else
6334                 {
6335                         return Float4(Frac(Float4(x))).x;
6336                 }
6337         }
6338
6339         RValue<Float> Floor(RValue<Float> x)
6340         {
6341                 if(CPUID::supportsSSE4_1())
6342                 {
6343                         return x86::floorss(x);
6344                 }
6345                 else
6346                 {
6347                         return Float4(Floor(Float4(x))).x;
6348                 }
6349         }
6350
6351         RValue<Float> Ceil(RValue<Float> x)
6352         {
6353                 if(CPUID::supportsSSE4_1())
6354                 {
6355                         return x86::ceilss(x);
6356                 }
6357                 else
6358                 {
6359                         return Float4(Ceil(Float4(x))).x;
6360                 }
6361         }
6362
6363         Type *Float::getType()
6364         {
6365                 return T(llvm::Type::getFloatTy(*::context));
6366         }
6367
6368         Float2::Float2(RValue<Float4> cast)
6369         {
6370         //      xyzw.parent = this;
6371
6372                 Value *int64x2 = Nucleus::createBitCast(cast.value, Long2::getType());
6373                 Value *int64 = Nucleus::createExtractElement(int64x2, 0);
6374                 Value *float2 = Nucleus::createBitCast(int64, Float2::getType());
6375
6376                 storeValue(float2);
6377         }
6378
6379         Type *Float2::getType()
6380         {
6381                 return T(VectorType::get(Float::getType(), 2));
6382         }
6383
6384         Float4::Float4(RValue<Byte4> cast)
6385         {
6386                 xyzw.parent = this;
6387
6388                 #if 0
6389                         Value *xyzw = Nucleus::createUIToFP(cast.value, Float4::getType());   // FIXME: Crashes
6390                 #elif 0
6391                         Value *vector = loadValue();
6392
6393                         Value *i8x = Nucleus::createExtractElement(cast.value, 0);
6394                         Value *f32x = Nucleus::createUIToFP(i8x, Float::getType());
6395                         Value *x = Nucleus::createInsertElement(vector, f32x, 0);
6396
6397                         Value *i8y = Nucleus::createExtractElement(cast.value, V(Nucleus::createConstantInt(1)));
6398                         Value *f32y = Nucleus::createUIToFP(i8y, Float::getType());
6399                         Value *xy = Nucleus::createInsertElement(x, f32y, V(Nucleus::createConstantInt(1)));
6400
6401                         Value *i8z = Nucleus::createExtractElement(cast.value, Nucleus::createConstantInt(2));
6402                         Value *f32z = Nucleus::createUIToFP(i8z, Float::getType());
6403                         Value *xyz = Nucleus::createInsertElement(xy, f32z, Nucleus::createConstantInt(2));
6404
6405                         Value *i8w = Nucleus::createExtractElement(cast.value, Nucleus::createConstantInt(3));
6406                         Value *f32w = Nucleus::createUIToFP(i8w, Float::getType());
6407                         Value *xyzw = Nucleus::createInsertElement(xyz, f32w, Nucleus::createConstantInt(3));
6408                 #else
6409                         Value *a = Int4(cast).loadValue();
6410                         Value *xyzw = Nucleus::createSIToFP(a, Float4::getType());
6411                 #endif
6412
6413                 storeValue(xyzw);
6414         }
6415
6416         Float4::Float4(RValue<SByte4> cast)
6417         {
6418                 xyzw.parent = this;
6419
6420                 #if 0
6421                         Value *xyzw = Nucleus::createSIToFP(cast.value, Float4::getType());   // FIXME: Crashes
6422                 #elif 0
6423                         Value *vector = loadValue();
6424
6425                         Value *i8x = Nucleus::createExtractElement(cast.value, 0);
6426                         Value *f32x = Nucleus::createSIToFP(i8x, Float::getType());
6427                         Value *x = Nucleus::createInsertElement(vector, f32x, 0);
6428
6429                         Value *i8y = Nucleus::createExtractElement(cast.value, V(Nucleus::createConstantInt(1)));
6430                         Value *f32y = Nucleus::createSIToFP(i8y, Float::getType());
6431                         Value *xy = Nucleus::createInsertElement(x, f32y, V(Nucleus::createConstantInt(1)));
6432
6433                         Value *i8z = Nucleus::createExtractElement(cast.value, Nucleus::createConstantInt(2));
6434                         Value *f32z = Nucleus::createSIToFP(i8z, Float::getType());
6435                         Value *xyz = Nucleus::createInsertElement(xy, f32z, Nucleus::createConstantInt(2));
6436
6437                         Value *i8w = Nucleus::createExtractElement(cast.value, Nucleus::createConstantInt(3));
6438                         Value *f32w = Nucleus::createSIToFP(i8w, Float::getType());
6439                         Value *xyzw = Nucleus::createInsertElement(xyz, f32w, Nucleus::createConstantInt(3));
6440                 #else
6441                         Value *a = Int4(cast).loadValue();
6442                         Value *xyzw = Nucleus::createSIToFP(a, Float4::getType());
6443                 #endif
6444
6445                 storeValue(xyzw);
6446         }
6447
6448         Float4::Float4(RValue<Short4> cast)
6449         {
6450                 xyzw.parent = this;
6451
6452                 Int4 c(cast);
6453                 storeValue(Nucleus::createSIToFP(RValue<Int4>(c).value, Float4::getType()));
6454         }
6455
6456         Float4::Float4(RValue<UShort4> cast)
6457         {
6458                 xyzw.parent = this;
6459
6460                 Int4 c(cast);
6461                 storeValue(Nucleus::createSIToFP(RValue<Int4>(c).value, Float4::getType()));
6462         }
6463
6464         Float4::Float4(RValue<Int4> cast)
6465         {
6466                 xyzw.parent = this;
6467
6468                 Value *xyzw = Nucleus::createSIToFP(cast.value, Float4::getType());
6469
6470                 storeValue(xyzw);
6471         }
6472
6473         Float4::Float4(RValue<UInt4> cast)
6474         {
6475                 xyzw.parent = this;
6476
6477                 Value *xyzw = Nucleus::createUIToFP(cast.value, Float4::getType());
6478
6479                 storeValue(xyzw);
6480         }
6481
6482         Float4::Float4()
6483         {
6484                 xyzw.parent = this;
6485         }
6486
6487         Float4::Float4(float xyzw)
6488         {
6489                 constant(xyzw, xyzw, xyzw, xyzw);
6490         }
6491
6492         Float4::Float4(float x, float yzw)
6493         {
6494                 constant(x, yzw, yzw, yzw);
6495         }
6496
6497         Float4::Float4(float x, float y, float zw)
6498         {
6499                 constant(x, y, zw, zw);
6500         }
6501
6502         Float4::Float4(float x, float y, float z, float w)
6503         {
6504                 constant(x, y, z, w);
6505         }
6506
6507         void Float4::constant(float x, float y, float z, float w)
6508         {
6509                 xyzw.parent = this;
6510
6511                 Constant *constantVector[4];
6512                 constantVector[0] = Nucleus::createConstantFloat(x);
6513                 constantVector[1] = Nucleus::createConstantFloat(y);
6514                 constantVector[2] = Nucleus::createConstantFloat(z);
6515                 constantVector[3] = Nucleus::createConstantFloat(w);
6516
6517                 storeValue(Nucleus::createConstantVector(constantVector, 4));
6518         }
6519
6520         Float4::Float4(RValue<Float4> rhs)
6521         {
6522                 xyzw.parent = this;
6523
6524                 storeValue(rhs.value);
6525         }
6526
6527         Float4::Float4(const Float4 &rhs)
6528         {
6529                 xyzw.parent = this;
6530
6531                 Value *value = rhs.loadValue();
6532                 storeValue(value);
6533         }
6534
6535         Float4::Float4(const Reference<Float4> &rhs)
6536         {
6537                 xyzw.parent = this;
6538
6539                 Value *value = rhs.loadValue();
6540                 storeValue(value);
6541         }
6542
6543         Float4::Float4(RValue<Float> rhs)
6544         {
6545                 xyzw.parent = this;
6546
6547                 Value *vector = loadValue();
6548                 Value *insert = Nucleus::createInsertElement(vector, rhs.value, 0);
6549
6550                 Constant *swizzle[4];
6551                 swizzle[0] = Nucleus::createConstantInt(0);
6552                 swizzle[1] = Nucleus::createConstantInt(0);
6553                 swizzle[2] = Nucleus::createConstantInt(0);
6554                 swizzle[3] = Nucleus::createConstantInt(0);
6555
6556                 Value *replicate = Nucleus::createShuffleVector(insert, V(UndefValue::get(Float4::getType())), V(Nucleus::createConstantVector(swizzle, 4)));
6557
6558                 storeValue(replicate);
6559         }
6560
6561         Float4::Float4(const Float &rhs)
6562         {
6563                 xyzw.parent = this;
6564
6565                 *this = RValue<Float>(rhs.loadValue());
6566         }
6567
6568         Float4::Float4(const Reference<Float> &rhs)
6569         {
6570                 xyzw.parent = this;
6571
6572                 *this = RValue<Float>(rhs.loadValue());
6573         }
6574
6575         RValue<Float4> Float4::operator=(float x) const
6576         {
6577                 return *this = Float4(x, x, x, x);
6578         }
6579
6580         RValue<Float4> Float4::operator=(RValue<Float4> rhs) const
6581         {
6582                 storeValue(rhs.value);
6583
6584                 return rhs;
6585         }
6586
6587         RValue<Float4> Float4::operator=(const Float4 &rhs) const
6588         {
6589                 Value *value = rhs.loadValue();
6590                 storeValue(value);
6591
6592                 return RValue<Float4>(value);
6593         }
6594
6595         RValue<Float4> Float4::operator=(const Reference<Float4> &rhs) const
6596         {
6597                 Value *value = rhs.loadValue();
6598                 storeValue(value);
6599
6600                 return RValue<Float4>(value);
6601         }
6602
6603         RValue<Float4> Float4::operator=(RValue<Float> rhs) const
6604         {
6605                 return *this = Float4(rhs);
6606         }
6607
6608         RValue<Float4> Float4::operator=(const Float &rhs) const
6609         {
6610                 return *this = Float4(rhs);
6611         }
6612
6613         RValue<Float4> Float4::operator=(const Reference<Float> &rhs) const
6614         {
6615                 return *this = Float4(rhs);
6616         }
6617
6618         RValue<Float4> operator+(RValue<Float4> lhs, RValue<Float4> rhs)
6619         {
6620                 return RValue<Float4>(Nucleus::createFAdd(lhs.value, rhs.value));
6621         }
6622
6623         RValue<Float4> operator-(RValue<Float4> lhs, RValue<Float4> rhs)
6624         {
6625                 return RValue<Float4>(Nucleus::createFSub(lhs.value, rhs.value));
6626         }
6627
6628         RValue<Float4> operator*(RValue<Float4> lhs, RValue<Float4> rhs)
6629         {
6630                 return RValue<Float4>(Nucleus::createFMul(lhs.value, rhs.value));
6631         }
6632
6633         RValue<Float4> operator/(RValue<Float4> lhs, RValue<Float4> rhs)
6634         {
6635                 return RValue<Float4>(Nucleus::createFDiv(lhs.value, rhs.value));
6636         }
6637
6638         RValue<Float4> operator%(RValue<Float4> lhs, RValue<Float4> rhs)
6639         {
6640                 return RValue<Float4>(Nucleus::createFRem(lhs.value, rhs.value));
6641         }
6642
6643         RValue<Float4> operator+=(const Float4 &lhs, RValue<Float4> rhs)
6644         {
6645                 return lhs = lhs + rhs;
6646         }
6647
6648         RValue<Float4> operator-=(const Float4 &lhs, RValue<Float4> rhs)
6649         {
6650                 return lhs = lhs - rhs;
6651         }
6652
6653         RValue<Float4> operator*=(const Float4 &lhs, RValue<Float4> rhs)
6654         {
6655                 return lhs = lhs * rhs;
6656         }
6657
6658         RValue<Float4> operator/=(const Float4 &lhs, RValue<Float4> rhs)
6659         {
6660                 return lhs = lhs / rhs;
6661         }
6662
6663         RValue<Float4> operator%=(const Float4 &lhs, RValue<Float4> rhs)
6664         {
6665                 return lhs = lhs % rhs;
6666         }
6667
6668         RValue<Float4> operator+(RValue<Float4> val)
6669         {
6670                 return val;
6671         }
6672
6673         RValue<Float4> operator-(RValue<Float4> val)
6674         {
6675                 return RValue<Float4>(Nucleus::createFNeg(val.value));
6676         }
6677
6678         RValue<Float4> Abs(RValue<Float4> x)
6679         {
6680                 Value *vector = Nucleus::createBitCast(x.value, Int4::getType());
6681
6682                 Constant *constantVector[4];
6683                 constantVector[0] = Nucleus::createConstantInt(0x7FFFFFFF);
6684                 constantVector[1] = Nucleus::createConstantInt(0x7FFFFFFF);
6685                 constantVector[2] = Nucleus::createConstantInt(0x7FFFFFFF);
6686                 constantVector[3] = Nucleus::createConstantInt(0x7FFFFFFF);
6687
6688                 Value *result = Nucleus::createAnd(vector, V(Nucleus::createConstantVector(constantVector, 4)));
6689
6690                 return RValue<Float4>(Nucleus::createBitCast(result, Float4::getType()));
6691         }
6692
6693         RValue<Float4> Max(RValue<Float4> x, RValue<Float4> y)
6694         {
6695                 return x86::maxps(x, y);
6696         }
6697
6698         RValue<Float4> Min(RValue<Float4> x, RValue<Float4> y)
6699         {
6700                 return x86::minps(x, y);
6701         }
6702
6703         RValue<Float4> Rcp_pp(RValue<Float4> x, bool exactAtPow2)
6704         {
6705                 if(exactAtPow2)
6706                 {
6707                         // rcpps uses a piecewise-linear approximation which minimizes the relative error
6708                         // but is not exact at power-of-two values. Rectify by multiplying by the inverse.
6709                         return x86::rcpps(x) * Float4(1.0f / _mm_cvtss_f32(_mm_rcp_ss(_mm_set_ps1(1.0f))));
6710                 }
6711                 else
6712                 {
6713                         return x86::rcpps(x);
6714                 }
6715         }
6716
6717         RValue<Float4> RcpSqrt_pp(RValue<Float4> x)
6718         {
6719                 return x86::rsqrtps(x);
6720         }
6721
6722         RValue<Float4> Sqrt(RValue<Float4> x)
6723         {
6724                 return x86::sqrtps(x);
6725         }
6726
6727         RValue<Float4> Insert(const Float4 &val, RValue<Float> element, int i)
6728         {
6729                 Value *value = val.loadValue();
6730                 Value *insert = Nucleus::createInsertElement(value, element.value, i);
6731
6732                 val = RValue<Float4>(insert);
6733
6734                 return val;
6735         }
6736
6737         RValue<Float> Extract(RValue<Float4> x, int i)
6738         {
6739                 return RValue<Float>(Nucleus::createExtractElement(x.value, i));
6740         }
6741
6742         RValue<Float4> Swizzle(RValue<Float4> x, unsigned char select)
6743         {
6744                 return RValue<Float4>(Nucleus::createSwizzle(x.value, select));
6745         }
6746
6747         RValue<Float4> ShuffleLowHigh(RValue<Float4> x, RValue<Float4> y, unsigned char imm)
6748         {
6749                 Constant *shuffle[4];
6750                 shuffle[0] = Nucleus::createConstantInt(((imm >> 0) & 0x03) + 0);
6751                 shuffle[1] = Nucleus::createConstantInt(((imm >> 2) & 0x03) + 0);
6752                 shuffle[2] = Nucleus::createConstantInt(((imm >> 4) & 0x03) + 4);
6753                 shuffle[3] = Nucleus::createConstantInt(((imm >> 6) & 0x03) + 4);
6754
6755                 return RValue<Float4>(Nucleus::createShuffleVector(x.value, y.value, V(Nucleus::createConstantVector(shuffle, 4))));
6756         }
6757
6758         RValue<Float4> UnpackLow(RValue<Float4> x, RValue<Float4> y)
6759         {
6760                 Constant *shuffle[4];
6761                 shuffle[0] = Nucleus::createConstantInt(0);
6762                 shuffle[1] = Nucleus::createConstantInt(4);
6763                 shuffle[2] = Nucleus::createConstantInt(1);
6764                 shuffle[3] = Nucleus::createConstantInt(5);
6765
6766                 return RValue<Float4>(Nucleus::createShuffleVector(x.value, y.value, V(Nucleus::createConstantVector(shuffle, 4))));
6767         }
6768
6769         RValue<Float4> UnpackHigh(RValue<Float4> x, RValue<Float4> y)
6770         {
6771                 Constant *shuffle[4];
6772                 shuffle[0] = Nucleus::createConstantInt(2);
6773                 shuffle[1] = Nucleus::createConstantInt(6);
6774                 shuffle[2] = Nucleus::createConstantInt(3);
6775                 shuffle[3] = Nucleus::createConstantInt(7);
6776
6777                 return RValue<Float4>(Nucleus::createShuffleVector(x.value, y.value, V(Nucleus::createConstantVector(shuffle, 4))));
6778         }
6779
6780         RValue<Float4> Mask(Float4 &lhs, RValue<Float4> rhs, unsigned char select)
6781         {
6782                 Value *vector = lhs.loadValue();
6783                 Value *shuffle = Nucleus::createMask(vector, rhs.value, select);
6784                 lhs.storeValue(shuffle);
6785
6786                 return RValue<Float4>(shuffle);
6787         }
6788
6789         RValue<Int> SignMask(RValue<Float4> x)
6790         {
6791                 return x86::movmskps(x);
6792         }
6793
6794         RValue<Int4> CmpEQ(RValue<Float4> x, RValue<Float4> y)
6795         {
6796         //      return As<Int4>(x86::cmpeqps(x, y));
6797                 return RValue<Int4>(Nucleus::createSExt(Nucleus::createFCmpOEQ(x.value, y.value), Int4::getType()));
6798         }
6799
6800         RValue<Int4> CmpLT(RValue<Float4> x, RValue<Float4> y)
6801         {
6802         //      return As<Int4>(x86::cmpltps(x, y));
6803                 return RValue<Int4>(Nucleus::createSExt(Nucleus::createFCmpOLT(x.value, y.value), Int4::getType()));
6804         }
6805
6806         RValue<Int4> CmpLE(RValue<Float4> x, RValue<Float4> y)
6807         {
6808         //      return As<Int4>(x86::cmpleps(x, y));
6809                 return RValue<Int4>(Nucleus::createSExt(Nucleus::createFCmpOLE(x.value, y.value), Int4::getType()));
6810         }
6811
6812         RValue<Int4> CmpNEQ(RValue<Float4> x, RValue<Float4> y)
6813         {
6814         //      return As<Int4>(x86::cmpneqps(x, y));
6815                 return RValue<Int4>(Nucleus::createSExt(Nucleus::createFCmpONE(x.value, y.value), Int4::getType()));
6816         }
6817
6818         RValue<Int4> CmpNLT(RValue<Float4> x, RValue<Float4> y)
6819         {
6820         //      return As<Int4>(x86::cmpnltps(x, y));
6821                 return RValue<Int4>(Nucleus::createSExt(Nucleus::createFCmpOGE(x.value, y.value), Int4::getType()));
6822         }
6823
6824         RValue<Int4> CmpNLE(RValue<Float4> x, RValue<Float4> y)
6825         {
6826         //      return As<Int4>(x86::cmpnleps(x, y));
6827                 return RValue<Int4>(Nucleus::createSExt(Nucleus::createFCmpOGT(x.value, y.value), Int4::getType()));
6828         }
6829
6830         RValue<Float4> Round(RValue<Float4> x)
6831         {
6832                 if(CPUID::supportsSSE4_1())
6833                 {
6834                         return x86::roundps(x, 0);
6835                 }
6836                 else
6837                 {
6838                         return Float4(RoundInt(x));
6839                 }
6840         }
6841
6842         RValue<Float4> Trunc(RValue<Float4> x)
6843         {
6844                 if(CPUID::supportsSSE4_1())
6845                 {
6846                         return x86::roundps(x, 3);
6847                 }
6848                 else
6849                 {
6850                         return Float4(Int4(x));   // Rounded toward zero
6851                 }
6852         }
6853
6854         RValue<Float4> Frac(RValue<Float4> x)
6855         {
6856                 if(CPUID::supportsSSE4_1())
6857                 {
6858                         return x - x86::floorps(x);
6859                 }
6860                 else
6861                 {
6862                         Float4 frc = x - Float4(Int4(x));   // Signed fractional part
6863
6864                         return frc + As<Float4>(As<Int4>(CmpNLE(Float4(0.0f), frc)) & As<Int4>(Float4(1, 1, 1, 1)));
6865                 }
6866         }
6867
6868         RValue<Float4> Floor(RValue<Float4> x)
6869         {
6870                 if(CPUID::supportsSSE4_1())
6871                 {
6872                         return x86::floorps(x);
6873                 }
6874                 else
6875                 {
6876                         return x - Frac(x);
6877                 }
6878         }
6879
6880         RValue<Float4> Ceil(RValue<Float4> x)
6881         {
6882                 if(CPUID::supportsSSE4_1())
6883                 {
6884                         return x86::ceilps(x);
6885                 }
6886                 else
6887                 {
6888                         return -Floor(-x);
6889                 }
6890         }
6891
6892         Type *Float4::getType()
6893         {
6894                 return T(VectorType::get(Float::getType(), 4));
6895         }
6896
6897         RValue<Pointer<Byte>> operator+(RValue<Pointer<Byte>> lhs, int offset)
6898         {
6899                 return RValue<Pointer<Byte>>(Nucleus::createGEP(lhs.value, Byte::getType(), V(Nucleus::createConstantInt(offset))));
6900         }
6901
6902         RValue<Pointer<Byte>> operator+(RValue<Pointer<Byte>> lhs, RValue<Int> offset)
6903         {
6904                 return RValue<Pointer<Byte>>(Nucleus::createGEP(lhs.value, Byte::getType(), offset.value));
6905         }
6906
6907         RValue<Pointer<Byte>> operator+(RValue<Pointer<Byte>> lhs, RValue<UInt> offset)
6908         {
6909                 return RValue<Pointer<Byte>>(Nucleus::createGEP(lhs.value, Byte::getType(), offset.value));
6910         }
6911
6912         RValue<Pointer<Byte>> operator+=(const Pointer<Byte> &lhs, int offset)
6913         {
6914                 return lhs = lhs + offset;
6915         }
6916
6917         RValue<Pointer<Byte>> operator+=(const Pointer<Byte> &lhs, RValue<Int> offset)
6918         {
6919                 return lhs = lhs + offset;
6920         }
6921
6922         RValue<Pointer<Byte>> operator+=(const Pointer<Byte> &lhs, RValue<UInt> offset)
6923         {
6924                 return lhs = lhs + offset;
6925         }
6926
6927         RValue<Pointer<Byte>> operator-(RValue<Pointer<Byte>> lhs, int offset)
6928         {
6929                 return lhs + -offset;
6930         }
6931
6932         RValue<Pointer<Byte>> operator-(RValue<Pointer<Byte>> lhs, RValue<Int> offset)
6933         {
6934                 return lhs + -offset;
6935         }
6936
6937         RValue<Pointer<Byte>> operator-(RValue<Pointer<Byte>> lhs, RValue<UInt> offset)
6938         {
6939                 return lhs + -offset;
6940         }
6941
6942         RValue<Pointer<Byte>> operator-=(const Pointer<Byte> &lhs, int offset)
6943         {
6944                 return lhs = lhs - offset;
6945         }
6946
6947         RValue<Pointer<Byte>> operator-=(const Pointer<Byte> &lhs, RValue<Int> offset)
6948         {
6949                 return lhs = lhs - offset;
6950         }
6951
6952         RValue<Pointer<Byte>> operator-=(const Pointer<Byte> &lhs, RValue<UInt> offset)
6953         {
6954                 return lhs = lhs - offset;
6955         }
6956
6957         void Return()
6958         {
6959                 Nucleus::createRetVoid();
6960                 Nucleus::setInsertBlock(Nucleus::createBasicBlock());
6961                 Nucleus::createUnreachable();
6962         }
6963
6964         void Return(bool ret)
6965         {
6966                 Nucleus::createRet(V(Nucleus::createConstantBool(ret)));
6967                 Nucleus::setInsertBlock(Nucleus::createBasicBlock());
6968                 Nucleus::createUnreachable();
6969         }
6970
6971         void Return(const Int &ret)
6972         {
6973                 Nucleus::createRet(ret.loadValue());
6974                 Nucleus::setInsertBlock(Nucleus::createBasicBlock());
6975                 Nucleus::createUnreachable();
6976         }
6977
6978         BasicBlock *beginLoop()
6979         {
6980                 BasicBlock *loopBB = Nucleus::createBasicBlock();
6981
6982                 Nucleus::createBr(loopBB);
6983                 Nucleus::setInsertBlock(loopBB);
6984
6985                 return loopBB;
6986         }
6987
6988         bool branch(RValue<Bool> cmp, BasicBlock *bodyBB, BasicBlock *endBB)
6989         {
6990                 Nucleus::createCondBr(cmp.value, bodyBB, endBB);
6991                 Nucleus::setInsertBlock(bodyBB);
6992
6993                 return true;
6994         }
6995
6996         bool elseBlock(BasicBlock *falseBB)
6997         {
6998                 falseBB->back().eraseFromParent();
6999                 Nucleus::setInsertBlock(falseBB);
7000
7001                 return true;
7002         }
7003
7004         RValue<Long> Ticks()
7005         {
7006                 llvm::Function *rdtsc = Intrinsic::getDeclaration(::module, Intrinsic::readcyclecounter);
7007
7008                 return RValue<Long>(V(::builder->CreateCall(rdtsc)));
7009         }
7010 }
7011
7012 namespace sw
7013 {
7014         namespace x86
7015         {
7016                 RValue<Int> cvtss2si(RValue<Float> val)
7017                 {
7018                         llvm::Function *cvtss2si = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_cvtss2si);
7019
7020                         Float4 vector;
7021                         vector.x = val;
7022
7023                         return RValue<Int>(V(::builder->CreateCall(cvtss2si, RValue<Float4>(vector).value)));
7024                 }
7025
7026                 RValue<Int2> cvtps2pi(RValue<Float4> val)
7027                 {
7028                         llvm::Function *cvtps2pi = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_cvtps2pi);
7029
7030                         return RValue<Int2>(V(::builder->CreateCall(cvtps2pi, val.value)));
7031                 }
7032
7033                 RValue<Int2> cvttps2pi(RValue<Float4> val)
7034                 {
7035                         llvm::Function *cvttps2pi = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_cvttps2pi);
7036
7037                         return RValue<Int2>(V(::builder->CreateCall(cvttps2pi, val.value)));
7038                 }
7039
7040                 RValue<Int4> cvtps2dq(RValue<Float4> val)
7041                 {
7042                         if(CPUID::supportsSSE2())
7043                         {
7044                                 llvm::Function *cvtps2dq = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse2_cvtps2dq);
7045
7046                                 return RValue<Int4>(V(::builder->CreateCall(cvtps2dq, val.value)));
7047                         }
7048                         else
7049                         {
7050                                 Int2 lo = x86::cvtps2pi(val);
7051                                 Int2 hi = x86::cvtps2pi(Swizzle(val, 0xEE));
7052
7053                                 return Int4(lo, hi);
7054                         }
7055                 }
7056
7057                 RValue<Float> rcpss(RValue<Float> val)
7058                 {
7059                         llvm::Function *rcpss = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_rcp_ss);
7060
7061                         Value *vector = Nucleus::createInsertElement(V(UndefValue::get(Float4::getType())), val.value, 0);
7062
7063                         return RValue<Float>(Nucleus::createExtractElement(V(::builder->CreateCall(rcpss, vector)), 0));
7064                 }
7065
7066                 RValue<Float> sqrtss(RValue<Float> val)
7067                 {
7068                         llvm::Function *sqrtss = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_sqrt_ss);
7069
7070                         Value *vector = Nucleus::createInsertElement(V(UndefValue::get(Float4::getType())), val.value, 0);
7071
7072                         return RValue<Float>(Nucleus::createExtractElement(V(::builder->CreateCall(sqrtss, vector)), 0));
7073                 }
7074
7075                 RValue<Float> rsqrtss(RValue<Float> val)
7076                 {
7077                         llvm::Function *rsqrtss = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_rsqrt_ss);
7078
7079                         Value *vector = Nucleus::createInsertElement(V(UndefValue::get(Float4::getType())), val.value, 0);
7080
7081                         return RValue<Float>(Nucleus::createExtractElement(V(::builder->CreateCall(rsqrtss, vector)), 0));
7082                 }
7083
7084                 RValue<Float4> rcpps(RValue<Float4> val)
7085                 {
7086                         llvm::Function *rcpps = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_rcp_ps);
7087
7088                         return RValue<Float4>(V(::builder->CreateCall(rcpps, val.value)));
7089                 }
7090
7091                 RValue<Float4> sqrtps(RValue<Float4> val)
7092                 {
7093                         llvm::Function *sqrtps = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_sqrt_ps);
7094
7095                         return RValue<Float4>(V(::builder->CreateCall(sqrtps, val.value)));
7096                 }
7097
7098                 RValue<Float4> rsqrtps(RValue<Float4> val)
7099                 {
7100                         llvm::Function *rsqrtps = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_rsqrt_ps);
7101
7102                         return RValue<Float4>(V(::builder->CreateCall(rsqrtps, val.value)));
7103                 }
7104
7105                 RValue<Float4> maxps(RValue<Float4> x, RValue<Float4> y)
7106                 {
7107                         llvm::Function *maxps = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_max_ps);
7108
7109                         return RValue<Float4>(V(::builder->CreateCall2(maxps, x.value, y.value)));
7110                 }
7111
7112                 RValue<Float4> minps(RValue<Float4> x, RValue<Float4> y)
7113                 {
7114                         llvm::Function *minps = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_min_ps);
7115
7116                         return RValue<Float4>(V(::builder->CreateCall2(minps, x.value, y.value)));
7117                 }
7118
7119                 RValue<Float> roundss(RValue<Float> val, unsigned char imm)
7120                 {
7121                         llvm::Function *roundss = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse41_round_ss);
7122
7123                         Value *undef = V(UndefValue::get(Float4::getType()));
7124                         Value *vector = Nucleus::createInsertElement(undef, val.value, 0);
7125
7126                         return RValue<Float>(Nucleus::createExtractElement(V(::builder->CreateCall3(roundss, undef, vector, V(Nucleus::createConstantInt(imm)))), 0));
7127                 }
7128
7129                 RValue<Float> floorss(RValue<Float> val)
7130                 {
7131                         return roundss(val, 1);
7132                 }
7133
7134                 RValue<Float> ceilss(RValue<Float> val)
7135                 {
7136                         return roundss(val, 2);
7137                 }
7138
7139                 RValue<Float4> roundps(RValue<Float4> val, unsigned char imm)
7140                 {
7141                         llvm::Function *roundps = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse41_round_ps);
7142
7143                         return RValue<Float4>(V(::builder->CreateCall2(roundps, val.value, V(Nucleus::createConstantInt(imm)))));
7144                 }
7145
7146                 RValue<Float4> floorps(RValue<Float4> val)
7147                 {
7148                         return roundps(val, 1);
7149                 }
7150
7151                 RValue<Float4> ceilps(RValue<Float4> val)
7152                 {
7153                         return roundps(val, 2);
7154                 }
7155
7156                 RValue<Float4> cmpps(RValue<Float4> x, RValue<Float4> y, unsigned char imm)
7157                 {
7158                         llvm::Function *cmpps = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_cmp_ps);
7159
7160                         return RValue<Float4>(V(::builder->CreateCall3(cmpps, x.value, y.value, V(Nucleus::createConstantByte(imm)))));
7161                 }
7162
7163                 RValue<Float4> cmpeqps(RValue<Float4> x, RValue<Float4> y)
7164                 {
7165                         return cmpps(x, y, 0);
7166                 }
7167
7168                 RValue<Float4> cmpltps(RValue<Float4> x, RValue<Float4> y)
7169                 {
7170                         return cmpps(x, y, 1);
7171                 }
7172
7173                 RValue<Float4> cmpleps(RValue<Float4> x, RValue<Float4> y)
7174                 {
7175                         return cmpps(x, y, 2);
7176                 }
7177
7178                 RValue<Float4> cmpunordps(RValue<Float4> x, RValue<Float4> y)
7179                 {
7180                         return cmpps(x, y, 3);
7181                 }
7182
7183                 RValue<Float4> cmpneqps(RValue<Float4> x, RValue<Float4> y)
7184                 {
7185                         return cmpps(x, y, 4);
7186                 }
7187
7188                 RValue<Float4> cmpnltps(RValue<Float4> x, RValue<Float4> y)
7189                 {
7190                         return cmpps(x, y, 5);
7191                 }
7192
7193                 RValue<Float4> cmpnleps(RValue<Float4> x, RValue<Float4> y)
7194                 {
7195                         return cmpps(x, y, 6);
7196                 }
7197
7198                 RValue<Float4> cmpordps(RValue<Float4> x, RValue<Float4> y)
7199                 {
7200                         return cmpps(x, y, 7);
7201                 }
7202
7203                 RValue<Float> cmpss(RValue<Float> x, RValue<Float> y, unsigned char imm)
7204                 {
7205                         llvm::Function *cmpss = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_cmp_ss);
7206
7207                         Value *vector1 = Nucleus::createInsertElement(V(UndefValue::get(Float4::getType())), x.value, 0);
7208                         Value *vector2 = Nucleus::createInsertElement(V(UndefValue::get(Float4::getType())), y.value, 0);
7209
7210                         return RValue<Float>(Nucleus::createExtractElement(V(::builder->CreateCall3(cmpss, vector1, vector2, V(Nucleus::createConstantByte(imm)))), 0));
7211                 }
7212
7213                 RValue<Float> cmpeqss(RValue<Float> x, RValue<Float> y)
7214                 {
7215                         return cmpss(x, y, 0);
7216                 }
7217
7218                 RValue<Float> cmpltss(RValue<Float> x, RValue<Float> y)
7219                 {
7220                         return cmpss(x, y, 1);
7221                 }
7222
7223                 RValue<Float> cmpless(RValue<Float> x, RValue<Float> y)
7224                 {
7225                         return cmpss(x, y, 2);
7226                 }
7227
7228                 RValue<Float> cmpunordss(RValue<Float> x, RValue<Float> y)
7229                 {
7230                         return cmpss(x, y, 3);
7231                 }
7232
7233                 RValue<Float> cmpneqss(RValue<Float> x, RValue<Float> y)
7234                 {
7235                         return cmpss(x, y, 4);
7236                 }
7237
7238                 RValue<Float> cmpnltss(RValue<Float> x, RValue<Float> y)
7239                 {
7240                         return cmpss(x, y, 5);
7241                 }
7242
7243                 RValue<Float> cmpnless(RValue<Float> x, RValue<Float> y)
7244                 {
7245                         return cmpss(x, y, 6);
7246                 }
7247
7248                 RValue<Float> cmpordss(RValue<Float> x, RValue<Float> y)
7249                 {
7250                         return cmpss(x, y, 7);
7251                 }
7252
7253                 RValue<Int4> pabsd(RValue<Int4> x)
7254                 {
7255                         llvm::Function *pabsd = Intrinsic::getDeclaration(::module, Intrinsic::x86_ssse3_pabs_d_128);
7256
7257                         return RValue<Int4>(V(::builder->CreateCall(pabsd, x.value)));
7258                 }
7259
7260                 RValue<Short4> paddsw(RValue<Short4> x, RValue<Short4> y)
7261                 {
7262                         llvm::Function *paddsw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_padds_w);
7263
7264                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(paddsw, As<MMX>(x).value, As<MMX>(y).value))));
7265                 }
7266
7267                 RValue<Short4> psubsw(RValue<Short4> x, RValue<Short4> y)
7268                 {
7269                         llvm::Function *psubsw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psubs_w);
7270
7271                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(psubsw, As<MMX>(x).value, As<MMX>(y).value))));
7272                 }
7273
7274                 RValue<UShort4> paddusw(RValue<UShort4> x, RValue<UShort4> y)
7275                 {
7276                         llvm::Function *paddusw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_paddus_w);
7277
7278                         return As<UShort4>(RValue<MMX>(V(::builder->CreateCall2(paddusw, As<MMX>(x).value, As<MMX>(y).value))));
7279                 }
7280
7281                 RValue<UShort4> psubusw(RValue<UShort4> x, RValue<UShort4> y)
7282                 {
7283                         llvm::Function *psubusw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psubus_w);
7284
7285                         return As<UShort4>(RValue<MMX>(V(::builder->CreateCall2(psubusw, As<MMX>(x).value, As<MMX>(y).value))));
7286                 }
7287
7288                 RValue<SByte8> paddsb(RValue<SByte8> x, RValue<SByte8> y)
7289                 {
7290                         llvm::Function *paddsb = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_padds_b);
7291
7292                         return As<SByte8>(RValue<MMX>(V(::builder->CreateCall2(paddsb, As<MMX>(x).value, As<MMX>(y).value))));
7293                 }
7294
7295                 RValue<SByte8> psubsb(RValue<SByte8> x, RValue<SByte8> y)
7296                 {
7297                         llvm::Function *psubsb = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psubs_b);
7298
7299                         return As<SByte8>(RValue<MMX>(V(::builder->CreateCall2(psubsb, As<MMX>(x).value, As<MMX>(y).value))));
7300                 }
7301
7302                 RValue<Byte8> paddusb(RValue<Byte8> x, RValue<Byte8> y)
7303                 {
7304                         llvm::Function *paddusb = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_paddus_b);
7305
7306                         return As<Byte8>(RValue<MMX>(V(::builder->CreateCall2(paddusb, As<MMX>(x).value, As<MMX>(y).value))));
7307                 }
7308
7309                 RValue<Byte8> psubusb(RValue<Byte8> x, RValue<Byte8> y)
7310                 {
7311                         llvm::Function *psubusb = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psubus_b);
7312
7313                         return As<Byte8>(RValue<MMX>(V(::builder->CreateCall2(psubusb, As<MMX>(x).value, As<MMX>(y).value))));
7314                 }
7315
7316                 RValue<Short4> paddw(RValue<Short4> x, RValue<Short4> y)
7317                 {
7318                         llvm::Function *paddw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_padd_w);
7319
7320                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(paddw, As<MMX>(x).value, As<MMX>(y).value))));
7321                 }
7322
7323                 RValue<Short4> psubw(RValue<Short4> x, RValue<Short4> y)
7324                 {
7325                         llvm::Function *psubw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psub_w);
7326
7327                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(psubw, As<MMX>(x).value, As<MMX>(y).value))));
7328                 }
7329
7330                 RValue<Short4> pmullw(RValue<Short4> x, RValue<Short4> y)
7331                 {
7332                         llvm::Function *pmullw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pmull_w);
7333
7334                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(pmullw, As<MMX>(x).value, As<MMX>(y).value))));
7335                 }
7336
7337                 RValue<Short4> pand(RValue<Short4> x, RValue<Short4> y)
7338                 {
7339                         llvm::Function *pand = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pand);
7340
7341                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(pand, As<MMX>(x).value, As<MMX>(y).value))));
7342                 }
7343
7344                 RValue<Short4> por(RValue<Short4> x, RValue<Short4> y)
7345                 {
7346                         llvm::Function *por = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_por);
7347
7348                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(por, As<MMX>(x).value, As<MMX>(y).value))));
7349                 }
7350
7351                 RValue<Short4> pxor(RValue<Short4> x, RValue<Short4> y)
7352                 {
7353                         llvm::Function *pxor = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pxor);
7354
7355                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(pxor, As<MMX>(x).value, As<MMX>(y).value))));
7356                 }
7357
7358                 RValue<Short4> pshufw(RValue<Short4> x, unsigned char y)
7359                 {
7360                         llvm::Function *pshufw = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_pshuf_w);
7361
7362                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(pshufw, As<MMX>(x).value, V(Nucleus::createConstantByte(y))))));
7363                 }
7364
7365                 RValue<Int2> punpcklwd(RValue<Short4> x, RValue<Short4> y)
7366                 {
7367                         llvm::Function *punpcklwd = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_punpcklwd);
7368
7369                         return As<Int2>(RValue<MMX>(V(::builder->CreateCall2(punpcklwd, As<MMX>(x).value, As<MMX>(y).value))));
7370                 }
7371
7372                 RValue<Int2> punpckhwd(RValue<Short4> x, RValue<Short4> y)
7373                 {
7374                         llvm::Function *punpckhwd = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_punpckhwd);
7375
7376                         return As<Int2>(RValue<MMX>(V(::builder->CreateCall2(punpckhwd, As<MMX>(x).value, As<MMX>(y).value))));
7377                 }
7378
7379                 RValue<Short4> pinsrw(RValue<Short4> x, RValue<Int> y, unsigned int i)
7380                 {
7381                         llvm::Function *pinsrw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pinsr_w);
7382
7383                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall3(pinsrw, As<MMX>(x).value, y.value, V(Nucleus::createConstantInt(i))))));
7384                 }
7385
7386                 RValue<Int> pextrw(RValue<Short4> x, unsigned int i)
7387                 {
7388                         llvm::Function *pextrw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pextr_w);
7389
7390                         return RValue<Int>(V(::builder->CreateCall2(pextrw, As<MMX>(x).value, V(Nucleus::createConstantInt(i)))));
7391                 }
7392
7393                 RValue<Long1> punpckldq(RValue<Int2> x, RValue<Int2> y)
7394                 {
7395                         llvm::Function *punpckldq = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_punpckldq);
7396
7397                         return As<Long1>(RValue<MMX>(V(::builder->CreateCall2(punpckldq, As<MMX>(x).value, As<MMX>(y).value))));
7398                 }
7399
7400                 RValue<Long1> punpckhdq(RValue<Int2> x, RValue<Int2> y)
7401                 {
7402                         llvm::Function *punpckhdq = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_punpckhdq);
7403
7404                         return As<Long1>(RValue<MMX>(V(::builder->CreateCall2(punpckhdq, As<MMX>(x).value, As<MMX>(y).value))));
7405                 }
7406
7407                 RValue<Short4> punpcklbw(RValue<Byte8> x, RValue<Byte8> y)
7408                 {
7409                         llvm::Function *punpcklbw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_punpcklbw);
7410
7411                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(punpcklbw, As<MMX>(x).value, As<MMX>(y).value))));
7412                 }
7413
7414                 RValue<Short4> punpckhbw(RValue<Byte8> x, RValue<Byte8> y)
7415                 {
7416                         llvm::Function *punpckhbw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_punpckhbw);
7417
7418                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(punpckhbw, As<MMX>(x).value, As<MMX>(y).value))));
7419                 }
7420
7421                 RValue<Byte8> paddb(RValue<Byte8> x, RValue<Byte8> y)
7422                 {
7423                         llvm::Function *paddb = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_padd_b);
7424
7425                         return As<Byte8>(RValue<MMX>(V(::builder->CreateCall2(paddb, As<MMX>(x).value, As<MMX>(y).value))));
7426                 }
7427
7428                 RValue<Byte8> psubb(RValue<Byte8> x, RValue<Byte8> y)
7429                 {
7430                         llvm::Function *psubb = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psub_b);
7431
7432                         return As<Byte8>(RValue<MMX>(V(::builder->CreateCall2(psubb, As<MMX>(x).value, As<MMX>(y).value))));
7433                 }
7434
7435                 RValue<Int2> paddd(RValue<Int2> x, RValue<Int2> y)
7436                 {
7437                         llvm::Function *paddd = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_padd_d);
7438
7439                         return As<Int2>(RValue<MMX>(V(::builder->CreateCall2(paddd, As<MMX>(x).value, As<MMX>(y).value))));
7440                 }
7441
7442                 RValue<Int2> psubd(RValue<Int2> x, RValue<Int2> y)
7443                 {
7444                         llvm::Function *psubd = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psub_d);
7445
7446                         return As<Int2>(RValue<MMX>(V(::builder->CreateCall2(psubd, As<MMX>(x).value, As<MMX>(y).value))));
7447                 }
7448
7449                 RValue<UShort4> pavgw(RValue<UShort4> x, RValue<UShort4> y)
7450                 {
7451                         llvm::Function *pavgw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pavg_w);
7452
7453                         return As<UShort4>(RValue<MMX>(V(::builder->CreateCall2(pavgw, As<MMX>(x).value, As<MMX>(y).value))));
7454                 }
7455
7456                 RValue<Short4> pmaxsw(RValue<Short4> x, RValue<Short4> y)
7457                 {
7458                         llvm::Function *pmaxsw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pmaxs_w);
7459
7460                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(pmaxsw, As<MMX>(x).value, As<MMX>(y).value))));
7461                 }
7462
7463                 RValue<Short4> pminsw(RValue<Short4> x, RValue<Short4> y)
7464                 {
7465                         llvm::Function *pminsw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pmins_w);
7466
7467                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(pminsw, As<MMX>(x).value, As<MMX>(y).value))));
7468                 }
7469
7470                 RValue<Short4> pcmpgtw(RValue<Short4> x, RValue<Short4> y)
7471                 {
7472                         llvm::Function *pcmpgtw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pcmpgt_w);
7473
7474                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(pcmpgtw, As<MMX>(x).value, As<MMX>(y).value))));
7475                 }
7476
7477                 RValue<Short4> pcmpeqw(RValue<Short4> x, RValue<Short4> y)
7478                 {
7479                         llvm::Function *pcmpeqw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pcmpeq_w);
7480
7481                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(pcmpeqw, As<MMX>(x).value, As<MMX>(y).value))));
7482                 }
7483
7484                 RValue<Byte8> pcmpgtb(RValue<SByte8> x, RValue<SByte8> y)
7485                 {
7486                         llvm::Function *pcmpgtb = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pcmpgt_b);
7487
7488                         return As<Byte8>(RValue<MMX>(V(::builder->CreateCall2(pcmpgtb, As<MMX>(x).value, As<MMX>(y).value))));
7489                 }
7490
7491                 RValue<Byte8> pcmpeqb(RValue<Byte8> x, RValue<Byte8> y)
7492                 {
7493                         llvm::Function *pcmpeqb = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pcmpeq_b);
7494
7495                         return As<Byte8>(RValue<MMX>(V(::builder->CreateCall2(pcmpeqb, As<MMX>(x).value, As<MMX>(y).value))));
7496                 }
7497
7498                 RValue<Short4> packssdw(RValue<Int2> x, RValue<Int2> y)
7499                 {
7500                         llvm::Function *packssdw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_packssdw);
7501
7502                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(packssdw, As<MMX>(x).value, As<MMX>(y).value))));
7503                 }
7504
7505                 RValue<Short8> packssdw(RValue<Int4> x, RValue<Int4> y)
7506                 {
7507                         if(CPUID::supportsSSE2())
7508                         {
7509                                 llvm::Function *packssdw = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse2_packssdw_128);
7510
7511                                 return RValue<Short8>(V(::builder->CreateCall2(packssdw, x.value, y.value)));
7512                         }
7513                         else
7514                         {
7515                                 Int2 loX = Int2(x);
7516                                 Int2 hiX = Int2(Swizzle(x, 0xEE));
7517
7518                                 Int2 loY = Int2(y);
7519                                 Int2 hiY = Int2(Swizzle(y, 0xEE));
7520
7521                                 Short4 lo = x86::packssdw(loX, hiX);
7522                                 Short4 hi = x86::packssdw(loY, hiY);
7523
7524                                 return Short8(lo, hi);
7525                         }
7526                 }
7527
7528                 RValue<SByte8> packsswb(RValue<Short4> x, RValue<Short4> y)
7529                 {
7530                         llvm::Function *packsswb = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_packsswb);
7531
7532                         return As<SByte8>(RValue<MMX>(V(::builder->CreateCall2(packsswb, As<MMX>(x).value, As<MMX>(y).value))));
7533                 }
7534
7535                 RValue<Byte8> packuswb(RValue<UShort4> x, RValue<UShort4> y)
7536                 {
7537                         llvm::Function *packuswb = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_packuswb);
7538
7539                         return As<Byte8>(RValue<MMX>(V(::builder->CreateCall2(packuswb, As<MMX>(x).value, As<MMX>(y).value))));
7540                 }
7541
7542                 RValue<UShort8> packusdw(RValue<UInt4> x, RValue<UInt4> y)
7543                 {
7544                         if(CPUID::supportsSSE4_1())
7545                         {
7546                                 llvm::Function *packusdw = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse41_packusdw);
7547
7548                                 return RValue<UShort8>(V(::builder->CreateCall2(packusdw, x.value, y.value)));
7549                         }
7550                         else
7551                         {
7552                                 // FIXME: Not an exact replacement!
7553                                 return As<UShort8>(packssdw(As<Int4>(x - UInt4(0x00008000, 0x00008000, 0x00008000, 0x00008000)), As<Int4>(y - UInt4(0x00008000, 0x00008000, 0x00008000, 0x00008000))) + Short8(0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u));
7554                         }
7555                 }
7556
7557                 RValue<UShort4> psrlw(RValue<UShort4> x, unsigned char y)
7558                 {
7559                         llvm::Function *psrlw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psrli_w);
7560
7561                         return As<UShort4>(RValue<MMX>(V(::builder->CreateCall2(psrlw, As<MMX>(x).value, V(Nucleus::createConstantInt(y))))));
7562                 }
7563
7564                 RValue<UShort8> psrlw(RValue<UShort8> x, unsigned char y)
7565                 {
7566                         llvm::Function *psrlw = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse2_psrli_w);
7567
7568                         return RValue<UShort8>(V(::builder->CreateCall2(psrlw, x.value, V(Nucleus::createConstantInt(y)))));
7569                 }
7570
7571                 RValue<Short4> psraw(RValue<Short4> x, unsigned char y)
7572                 {
7573                         llvm::Function *psraw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psrai_w);
7574
7575                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(psraw, As<MMX>(x).value, V(Nucleus::createConstantInt(y))))));
7576                 }
7577
7578                 RValue<Short8> psraw(RValue<Short8> x, unsigned char y)
7579                 {
7580                         llvm::Function *psraw = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse2_psrai_w);
7581
7582                         return RValue<Short8>(V(::builder->CreateCall2(psraw, x.value, V(Nucleus::createConstantInt(y)))));
7583                 }
7584
7585                 RValue<Short4> psllw(RValue<Short4> x, unsigned char y)
7586                 {
7587                         llvm::Function *psllw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pslli_w);
7588
7589                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(psllw, As<MMX>(x).value, V(Nucleus::createConstantInt(y))))));
7590                 }
7591
7592                 RValue<Short8> psllw(RValue<Short8> x, unsigned char y)
7593                 {
7594                         llvm::Function *psllw = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse2_pslli_w);
7595
7596                         return RValue<Short8>(V(::builder->CreateCall2(psllw, x.value, V(Nucleus::createConstantInt(y)))));
7597                 }
7598
7599                 RValue<Int2> pslld(RValue<Int2> x, unsigned char y)
7600                 {
7601                         llvm::Function *pslld = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pslli_d);
7602
7603                         return As<Int2>(RValue<MMX>(V(::builder->CreateCall2(pslld, As<MMX>(x).value, V(Nucleus::createConstantInt(y))))));
7604                 }
7605
7606                 RValue<Int4> pslld(RValue<Int4> x, unsigned char y)
7607                 {
7608                         if(CPUID::supportsSSE2())
7609                         {
7610                                 llvm::Function *pslld = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse2_pslli_d);
7611
7612                                 return RValue<Int4>(V(::builder->CreateCall2(pslld, x.value, V(Nucleus::createConstantInt(y)))));
7613                         }
7614                         else
7615                         {
7616                                 Int2 lo = Int2(x);
7617                                 Int2 hi = Int2(Swizzle(x, 0xEE));
7618
7619                                 lo = x86::pslld(lo, y);
7620                                 hi = x86::pslld(hi, y);
7621
7622                                 return Int4(lo, hi);
7623                         }
7624                 }
7625
7626                 RValue<Int2> psrad(RValue<Int2> x, unsigned char y)
7627                 {
7628                         llvm::Function *psrad = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psrai_d);
7629
7630                         return As<Int2>(RValue<MMX>(V(::builder->CreateCall2(psrad, As<MMX>(x).value, V(Nucleus::createConstantInt(y))))));
7631                 }
7632
7633                 RValue<Int4> psrad(RValue<Int4> x, unsigned char y)
7634                 {
7635                         if(CPUID::supportsSSE2())
7636                         {
7637                                 llvm::Function *psrad = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse2_psrai_d);
7638
7639                                 return RValue<Int4>(V(::builder->CreateCall2(psrad, x.value, V(Nucleus::createConstantInt(y)))));
7640                         }
7641                         else
7642                         {
7643                                 Int2 lo = Int2(x);
7644                                 Int2 hi = Int2(Swizzle(x, 0xEE));
7645
7646                                 lo = x86::psrad(lo, y);
7647                                 hi = x86::psrad(hi, y);
7648
7649                                 return Int4(lo, hi);
7650                         }
7651                 }
7652
7653                 RValue<UInt2> psrld(RValue<UInt2> x, unsigned char y)
7654                 {
7655                         llvm::Function *psrld = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psrli_d);
7656
7657                         return As<UInt2>(RValue<MMX>(V(::builder->CreateCall2(psrld, As<MMX>(x).value, V(Nucleus::createConstantInt(y))))));
7658                 }
7659
7660                 RValue<UInt4> psrld(RValue<UInt4> x, unsigned char y)
7661                 {
7662                         if(CPUID::supportsSSE2())
7663                         {
7664                                 llvm::Function *psrld = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse2_psrli_d);
7665
7666                                 return RValue<UInt4>(V(::builder->CreateCall2(psrld, x.value, V(Nucleus::createConstantInt(y)))));
7667                         }
7668                         else
7669                         {
7670                                 UInt2 lo = As<UInt2>(Int2(As<Int4>(x)));
7671                                 UInt2 hi = As<UInt2>(Int2(Swizzle(As<Int4>(x), 0xEE)));
7672
7673                                 lo = x86::psrld(lo, y);
7674                                 hi = x86::psrld(hi, y);
7675
7676                                 return UInt4(lo, hi);
7677                         }
7678                 }
7679
7680                 RValue<UShort4> psrlw(RValue<UShort4> x, RValue<Long1> y)
7681                 {
7682                         llvm::Function *psrlw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psrl_w);
7683
7684                         return As<UShort4>(RValue<MMX>(V(::builder->CreateCall2(psrlw, As<MMX>(x).value, As<MMX>(y).value))));
7685                 }
7686
7687                 RValue<Short4> psraw(RValue<Short4> x, RValue<Long1> y)
7688                 {
7689                         llvm::Function *psraw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psra_w);
7690
7691                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(psraw, As<MMX>(x).value, As<MMX>(y).value))));
7692                 }
7693
7694                 RValue<Short4> psllw(RValue<Short4> x, RValue<Long1> y)
7695                 {
7696                         llvm::Function *psllw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psll_w);
7697
7698                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(psllw, As<MMX>(x).value, As<MMX>(y).value))));
7699                 }
7700
7701                 RValue<Int2> pslld(RValue<Int2> x, RValue<Long1> y)
7702                 {
7703                         llvm::Function *pslld = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psll_d);
7704
7705                         return As<Int2>(RValue<MMX>(V(::builder->CreateCall2(pslld, As<MMX>(x).value, As<MMX>(y).value))));
7706                 }
7707
7708                 RValue<UInt2> psrld(RValue<UInt2> x, RValue<Long1> y)
7709                 {
7710                         llvm::Function *psrld = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psrl_d);
7711
7712                         return As<UInt2>(RValue<MMX>(V(::builder->CreateCall2(psrld, As<MMX>(x).value, As<MMX>(y).value))));
7713                 }
7714
7715                 RValue<Int2> psrad(RValue<Int2> x, RValue<Long1> y)
7716                 {
7717                         llvm::Function *psrld = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_psra_d);
7718
7719                         return As<Int2>(RValue<MMX>(V(::builder->CreateCall2(psrld, As<MMX>(x).value, As<MMX>(y).value))));
7720                 }
7721
7722                 RValue<Int4> pmaxsd(RValue<Int4> x, RValue<Int4> y)
7723                 {
7724                         llvm::Function *pmaxsd = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse41_pmaxsd);
7725
7726                         return RValue<Int4>(V(::builder->CreateCall2(pmaxsd, x.value, y.value)));
7727                 }
7728
7729                 RValue<Int4> pminsd(RValue<Int4> x, RValue<Int4> y)
7730                 {
7731                         llvm::Function *pminsd = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse41_pminsd);
7732
7733                         return RValue<Int4>(V(::builder->CreateCall2(pminsd, x.value, y.value)));
7734                 }
7735
7736                 RValue<UInt4> pmaxud(RValue<UInt4> x, RValue<UInt4> y)
7737                 {
7738                         llvm::Function *pmaxud = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse41_pmaxud);
7739
7740                         return RValue<UInt4>(V(::builder->CreateCall2(pmaxud, x.value, y.value)));
7741                 }
7742
7743                 RValue<UInt4> pminud(RValue<UInt4> x, RValue<UInt4> y)
7744                 {
7745                         llvm::Function *pminud = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse41_pminud);
7746
7747                         return RValue<UInt4>(V(::builder->CreateCall2(pminud, x.value, y.value)));
7748                 }
7749
7750                 RValue<Short4> pmulhw(RValue<Short4> x, RValue<Short4> y)
7751                 {
7752                         llvm::Function *pmulhw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pmulh_w);
7753
7754                         return As<Short4>(RValue<MMX>(V(::builder->CreateCall2(pmulhw, As<MMX>(x).value, As<MMX>(y).value))));
7755                 }
7756
7757                 RValue<UShort4> pmulhuw(RValue<UShort4> x, RValue<UShort4> y)
7758                 {
7759                         llvm::Function *pmulhuw = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pmulhu_w);
7760
7761                         return As<UShort4>(RValue<MMX>(V(::builder->CreateCall2(pmulhuw, As<MMX>(x).value, As<MMX>(y).value))));
7762                 }
7763
7764                 RValue<Int2> pmaddwd(RValue<Short4> x, RValue<Short4> y)
7765                 {
7766                         llvm::Function *pmaddwd = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pmadd_wd);
7767
7768                         return As<Int2>(RValue<MMX>(V(::builder->CreateCall2(pmaddwd, As<MMX>(x).value, As<MMX>(y).value))));
7769                 }
7770
7771                 RValue<Short8> pmulhw(RValue<Short8> x, RValue<Short8> y)
7772                 {
7773                         llvm::Function *pmulhw = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse2_pmulh_w);
7774
7775                         return RValue<Short8>(V(::builder->CreateCall2(pmulhw, x.value, y.value)));
7776                 }
7777
7778                 RValue<UShort8> pmulhuw(RValue<UShort8> x, RValue<UShort8> y)
7779                 {
7780                         llvm::Function *pmulhuw = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse2_pmulhu_w);
7781
7782                         return RValue<UShort8>(V(::builder->CreateCall2(pmulhuw, x.value, y.value)));
7783                 }
7784
7785                 RValue<Int4> pmaddwd(RValue<Short8> x, RValue<Short8> y)
7786                 {
7787                         llvm::Function *pmaddwd = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse2_pmadd_wd);
7788
7789                         return RValue<Int4>(V(::builder->CreateCall2(pmaddwd, x.value, y.value)));
7790                 }
7791
7792                 RValue<Int> movmskps(RValue<Float4> x)
7793                 {
7794                         llvm::Function *movmskps = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse_movmsk_ps);
7795
7796                         return RValue<Int>(V(::builder->CreateCall(movmskps, x.value)));
7797                 }
7798
7799                 RValue<Int> pmovmskb(RValue<Byte8> x)
7800                 {
7801                         llvm::Function *pmovmskb = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_pmovmskb);
7802
7803                         return RValue<Int>(V(::builder->CreateCall(pmovmskb, As<MMX>(x).value)));
7804                 }
7805
7806                 //RValue<Int2> movd(RValue<Pointer<Int>> x)
7807                 //{
7808                 //      Value *element = Nucleus::createLoad(x.value);
7809
7810                 ////    Value *int2 = UndefValue::get(Int2::getType());
7811                 ////    int2 = Nucleus::createInsertElement(int2, element, ConstantInt::get(Int::getType(), 0));
7812
7813                 //      Value *int2 = Nucleus::createBitCast(Nucleus::createZExt(element, Long::getType()), Int2::getType());
7814
7815                 //      return RValue<Int2>(int2);
7816                 //}
7817
7818                 //RValue<Int2> movdq2q(RValue<Int4> x)
7819                 //{
7820                 //      Value *long2 = Nucleus::createBitCast(x.value, Long2::getType());
7821                 //      Value *element = Nucleus::createExtractElement(long2, ConstantInt::get(Int::getType(), 0));
7822
7823                 //      return RValue<Int2>(Nucleus::createBitCast(element, Int2::getType()));
7824                 //}
7825
7826                 RValue<Int4> pmovzxbd(RValue<Int4> x)
7827                 {
7828                         llvm::Function *pmovzxbd = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse41_pmovzxbd);
7829
7830                         return RValue<Int4>(V(::builder->CreateCall(pmovzxbd, Nucleus::createBitCast(x.value, Byte16::getType()))));
7831                 }
7832
7833                 RValue<Int4> pmovsxbd(RValue<Int4> x)
7834                 {
7835                         llvm::Function *pmovsxbd = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse41_pmovsxbd);
7836
7837                         return RValue<Int4>(V(::builder->CreateCall(pmovsxbd, Nucleus::createBitCast(x.value, SByte16::getType()))));
7838                 }
7839
7840                 RValue<Int4> pmovzxwd(RValue<Int4> x)
7841                 {
7842                         llvm::Function *pmovzxwd = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse41_pmovzxwd);
7843
7844                         return RValue<Int4>(V(::builder->CreateCall(pmovzxwd, Nucleus::createBitCast(x.value, UShort8::getType()))));
7845                 }
7846
7847                 RValue<Int4> pmovsxwd(RValue<Int4> x)
7848                 {
7849                         llvm::Function *pmovsxwd = Intrinsic::getDeclaration(::module, Intrinsic::x86_sse41_pmovsxwd);
7850
7851                         return RValue<Int4>(V(::builder->CreateCall(pmovsxwd, Nucleus::createBitCast(x.value, Short8::getType()))));
7852                 }
7853
7854                 void emms()
7855                 {
7856                         llvm::Function *emms = Intrinsic::getDeclaration(::module, Intrinsic::x86_mmx_emms);
7857
7858                         V(::builder->CreateCall(emms));
7859                 }
7860         }
7861 }