src/Reactor/SubzeroReactor.cpp

   1 // Copyright 2016 The SwiftShader Authors. All Rights Reserved.
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License");
   4 // you may not use this file except in compliance with the License.
   5 // You may obtain a copy of the License at
   6 //
   7 //    http://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS,
  11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 // See the License for the specific language governing permissions and
  13 // limitations under the License.
  14
  15 #include "Nucleus.hpp"
  16
  17 #include "Reactor.hpp"
  18 #include "Routine.hpp"
  19
  20 #include "Optimizer.hpp"
  21
  22 #include "src/IceTypes.h"
  23 #include "src/IceCfg.h"
  24 #include "src/IceELFStreamer.h"
  25 #include "src/IceGlobalContext.h"
  26 #include "src/IceCfgNode.h"
  27 #include "src/IceELFObjectWriter.h"
  28 #include "src/IceGlobalInits.h"
  29
  30 #include "llvm/Support/FileSystem.h"
  31 #include "llvm/Support/raw_os_ostream.h"
  32
  33 #if defined(_WIN32)
  34 #ifndef WIN32_LEAN_AND_MEAN
  35 #define WIN32_LEAN_AND_MEAN
  36 #endif // !WIN32_LEAN_AND_MEAN
  37 #ifndef NOMINMAX
  38 #define NOMINMAX
  39 #endif // !NOMINMAX
  40 #include <Windows.h>
  41 #else
  42 #include <sys/mman.h>
  43 #if !defined(MAP_ANONYMOUS)
  44 #define MAP_ANONYMOUS MAP_ANON
  45 #endif
  46 #endif
  47
  48 #include <mutex>
  49 #include <limits>
  50 #include <iostream>
  51 #include <cassert>
  52
  53 namespace
  54 {
  55         Ice::GlobalContext *context = nullptr;
  56         Ice::Cfg *function = nullptr;
  57         Ice::CfgNode *basicBlock = nullptr;
  58         Ice::CfgLocalAllocatorScope *allocator = nullptr;
  59         sw::Routine *routine = nullptr;
  60
  61         std::mutex codegenMutex;
  62
  63         Ice::ELFFileStreamer *elfFile = nullptr;
  64         Ice::Fdstream *out = nullptr;
  65 }
  66
  67 namespace
  68 {
  69         #if !defined(__i386__) && defined(_M_IX86)
  70                 #define __i386__ 1
  71         #endif
  72
  73         #if !defined(__x86_64__) && (defined(_M_AMD64) || defined (_M_X64))
  74                 #define __x86_64__ 1
  75         #endif
  76
  77         class CPUID
  78         {
  79         public:
  80                 const static bool ARM;
  81                 const static bool SSE4_1;
  82
  83         private:
  84                 static void cpuid(int registers[4], int info)
  85                 {
  86                         #if defined(__i386__) || defined(__x86_64__)
  87                                 #if defined(_WIN32)
  88                                         __cpuid(registers, info);
  89                                 #else
  90                                         __asm volatile("cpuid": "=a" (registers[0]), "=b" (registers[1]), "=c" (registers[2]), "=d" (registers[3]): "a" (info));
  91                                 #endif
  92                         #else
  93                                 registers[0] = 0;
  94                                 registers[1] = 0;
  95                                 registers[2] = 0;
  96                                 registers[3] = 0;
  97                         #endif
  98                 }
  99
 100                 static bool detectARM()
 101                 {
 102                         #if defined(__arm__)
 103                                 return true;
 104                         #elif defined(__i386__) || defined(__x86_64__)
 105                                 return false;
 106                         #else
 107                                 #error "Unknown architecture"
 108                         #endif
 109                 }
 110
 111                 static bool detectSSE4_1()
 112                 {
 113                         #if defined(__i386__) || defined(__x86_64__)
 114                                 int registers[4];
 115                                 cpuid(registers, 1);
 116                                 return (registers[2] & 0x00080000) != 0;
 117                         #else
 118                                 return false;
 119                         #endif
 120                 }
 121         };
 122
 123         const bool CPUID::ARM = CPUID::detectARM();
 124         const bool CPUID::SSE4_1 = CPUID::detectSSE4_1();
 125         const bool emulateIntrinsics = CPUID::ARM;
 126         const bool emulateMismatchedBitCast = CPUID::ARM;
 127 }
 128
 129 namespace sw
 130 {
 131         enum EmulatedType
 132         {
 133                 EmulatedShift = 16,
 134                 EmulatedV2 = 2 << EmulatedShift,
 135                 EmulatedV4 = 4 << EmulatedShift,
 136                 EmulatedV8 = 8 << EmulatedShift,
 137                 EmulatedBits = EmulatedV2 | EmulatedV4 | EmulatedV8,
 138
 139                 Type_v2i32 = Ice::IceType_v4i32 | EmulatedV2,
 140                 Type_v4i16 = Ice::IceType_v8i16 | EmulatedV4,
 141                 Type_v2i16 = Ice::IceType_v8i16 | EmulatedV2,
 142                 Type_v8i8 =  Ice::IceType_v16i8 | EmulatedV8,
 143                 Type_v4i8 =  Ice::IceType_v16i8 | EmulatedV4,
 144                 Type_v2f32 = Ice::IceType_v4f32 | EmulatedV2,
 145         };
 146
 147         class Value : public Ice::Operand {};
 148         class SwitchCases : public Ice::InstSwitch {};
 149         class BasicBlock : public Ice::CfgNode {};
 150
 151         Ice::Type T(Type *t)
 152         {
 153                 static_assert(static_cast<unsigned int>(Ice::IceType_NUM) < static_cast<unsigned int>(EmulatedBits), "Ice::Type overlaps with our emulated types!");
 154                 return (Ice::Type)(reinterpret_cast<std::intptr_t>(t) & ~EmulatedBits);
 155         }
 156
 157         Type *T(Ice::Type t)
 158         {
 159                 return reinterpret_cast<Type*>(t);
 160         }
 161
 162         Type *T(EmulatedType t)
 163         {
 164                 return reinterpret_cast<Type*>(t);
 165         }
 166
 167         Value *V(Ice::Operand *v)
 168         {
 169                 return reinterpret_cast<Value*>(v);
 170         }
 171
 172         BasicBlock *B(Ice::CfgNode *b)
 173         {
 174                 return reinterpret_cast<BasicBlock*>(b);
 175         }
 176
 177         static size_t typeSize(Type *type)
 178         {
 179                 if(reinterpret_cast<std::intptr_t>(type) & EmulatedBits)
 180                 {
 181                         switch(reinterpret_cast<std::intptr_t>(type))
 182                         {
 183                         case Type_v2i32: return 8;
 184                         case Type_v4i16: return 8;
 185                         case Type_v2i16: return 4;
 186                         case Type_v8i8:  return 8;
 187                         case Type_v4i8:  return 4;
 188                         case Type_v2f32: return 8;
 189                         default: assert(false);
 190                         }
 191                 }
 192
 193                 return Ice::typeWidthInBytes(T(type));
 194         }
 195
 196         Optimization optimization[10] = {InstructionCombining, Disabled};
 197
 198         using ElfHeader = std::conditional<sizeof(void*) == 8, Elf64_Ehdr, Elf32_Ehdr>::type;
 199         using SectionHeader = std::conditional<sizeof(void*) == 8, Elf64_Shdr, Elf32_Shdr>::type;
 200
 201         inline const SectionHeader *sectionHeader(const ElfHeader *elfHeader)
 202         {
 203                 return reinterpret_cast<const SectionHeader*>((intptr_t)elfHeader + elfHeader->e_shoff);
 204         }
 205
 206         inline const SectionHeader *elfSection(const ElfHeader *elfHeader, int index)
 207         {
 208                 return &sectionHeader(elfHeader)[index];
 209         }
 210
 211         static void *relocateSymbol(const ElfHeader *elfHeader, const Elf32_Rel &relocation, const SectionHeader &relocationTable)
 212         {
 213                 const SectionHeader *target = elfSection(elfHeader, relocationTable.sh_info);
 214
 215                 intptr_t address = (intptr_t)elfHeader + target->sh_offset;
 216                 int32_t *patchSite = (int*)(address + relocation.r_offset);
 217                 uint32_t index = relocation.getSymbol();
 218                 int table = relocationTable.sh_link;
 219                 void *symbolValue = nullptr;
 220
 221                 if(index != SHN_UNDEF)
 222                 {
 223                         if(table == SHN_UNDEF) return nullptr;
 224                         const SectionHeader *symbolTable = elfSection(elfHeader, table);
 225
 226                         uint32_t symtab_entries = symbolTable->sh_size / symbolTable->sh_entsize;
 227                         if(index >= symtab_entries)
 228                         {
 229                                 assert(index < symtab_entries && "Symbol Index out of range");
 230                                 return nullptr;
 231                         }
 232
 233                         intptr_t symbolAddress = (intptr_t)elfHeader + symbolTable->sh_offset;
 234                         Elf32_Sym &symbol = ((Elf32_Sym*)symbolAddress)[index];
 235                         uint16_t section = symbol.st_shndx;
 236
 237                         if(section != SHN_UNDEF && section < SHN_LORESERVE)
 238                         {
 239                                 const SectionHeader *target = elfSection(elfHeader, symbol.st_shndx);
 240                                 symbolValue = reinterpret_cast<void*>((intptr_t)elfHeader + symbol.st_value + target->sh_offset);
 241                         }
 242                         else
 243                         {
 244                                 return nullptr;
 245                         }
 246                 }
 247
 248                 if(CPUID::ARM)
 249                 {
 250                         switch(relocation.getType())
 251                         {
 252                         case R_ARM_NONE:
 253                                 // No relocation
 254                                 break;
 255                         case R_ARM_MOVW_ABS_NC:
 256                                 {
 257                                         uint32_t thumb = 0;   // Calls to Thumb code not supported.
 258                                         uint32_t lo = (uint32_t)(intptr_t)symbolValue | thumb;
 259                                         *patchSite = (*patchSite & 0xFFF0F000) | ((lo & 0xF000) << 4) | (lo & 0x0FFF);
 260                                 }
 261                                 break;
 262                         case R_ARM_MOVT_ABS:
 263                                 {
 264                                         uint32_t hi = (uint32_t)(intptr_t)(symbolValue) >> 16;
 265                                         *patchSite = (*patchSite & 0xFFF0F000) | ((hi & 0xF000) << 4) | (hi & 0x0FFF);
 266                                 }
 267                                 break;
 268                         default:
 269                                 assert(false && "Unsupported relocation type");
 270                                 return nullptr;
 271                         }
 272                 }
 273                 else
 274                 {
 275                         switch(relocation.getType())
 276                         {
 277                         case R_386_NONE:
 278                                 // No relocation
 279                                 break;
 280                         case R_386_32:
 281                                 *patchSite = (int32_t)((intptr_t)symbolValue + *patchSite);
 282                                 break;
 283                 //      case R_386_PC32:
 284                 //              *patchSite = (int32_t)((intptr_t)symbolValue + *patchSite - (intptr_t)patchSite);
 285                 //              break;
 286                         default:
 287                                 assert(false && "Unsupported relocation type");
 288                                 return nullptr;
 289                         }
 290                 }
 291
 292
 293                 return symbolValue;
 294         }
 295
 296         static void *relocateSymbol(const ElfHeader *elfHeader, const Elf64_Rela &relocation, const SectionHeader &relocationTable)
 297         {
 298                 const SectionHeader *target = elfSection(elfHeader, relocationTable.sh_info);
 299
 300                 intptr_t address = (intptr_t)elfHeader + target->sh_offset;
 301                 int32_t *patchSite = (int*)(address + relocation.r_offset);
 302                 uint32_t index = relocation.getSymbol();
 303                 int table = relocationTable.sh_link;
 304                 void *symbolValue = nullptr;
 305
 306                 if(index != SHN_UNDEF)
 307                 {
 308                         if(table == SHN_UNDEF) return nullptr;
 309                         const SectionHeader *symbolTable = elfSection(elfHeader, table);
 310
 311                         uint32_t symtab_entries = symbolTable->sh_size / symbolTable->sh_entsize;
 312                         if(index >= symtab_entries)
 313                         {
 314                                 assert(index < symtab_entries && "Symbol Index out of range");
 315                                 return nullptr;
 316                         }
 317
 318                         intptr_t symbolAddress = (intptr_t)elfHeader + symbolTable->sh_offset;
 319                         Elf64_Sym &symbol = ((Elf64_Sym*)symbolAddress)[index];
 320                         uint16_t section = symbol.st_shndx;
 321
 322                         if(section != SHN_UNDEF && section < SHN_LORESERVE)
 323                         {
 324                                 const SectionHeader *target = elfSection(elfHeader, symbol.st_shndx);
 325                                 symbolValue = reinterpret_cast<void*>((intptr_t)elfHeader + symbol.st_value + target->sh_offset);
 326                         }
 327                         else
 328                         {
 329                                 return nullptr;
 330                         }
 331                 }
 332
 333                 switch(relocation.getType())
 334                 {
 335                 case R_X86_64_NONE:
 336                         // No relocation
 337                         break;
 338                 case R_X86_64_64:
 339                         *(int64_t*)patchSite = (int64_t)((intptr_t)symbolValue + *(int64_t*)patchSite) + relocation.r_addend;
 340                         break;
 341                 case R_X86_64_PC32:
 342                         *patchSite = (int32_t)((intptr_t)symbolValue + *patchSite - (intptr_t)patchSite) + relocation.r_addend;
 343                         break;
 344                 case R_X86_64_32S:
 345                         *patchSite = (int32_t)((intptr_t)symbolValue + *patchSite) + relocation.r_addend;
 346                         break;
 347                 default:
 348                         assert(false && "Unsupported relocation type");
 349                         return nullptr;
 350                 }
 351
 352                 return symbolValue;
 353         }
 354
 355         void *loadImage(uint8_t *const elfImage, size_t &codeSize)
 356         {
 357                 ElfHeader *elfHeader = (ElfHeader*)elfImage;
 358
 359                 if(!elfHeader->checkMagic())
 360                 {
 361                         return nullptr;
 362                 }
 363
 364                 // Expect ELF bitness to match platform
 365                 assert(sizeof(void*) == 8 ? elfHeader->getFileClass() == ELFCLASS64 : elfHeader->getFileClass() == ELFCLASS32);
 366                 #if defined(__i386__)
 367                         assert(sizeof(void*) == 4 && elfHeader->e_machine == EM_386);
 368                 #elif defined(__x86_64__)
 369                         assert(sizeof(void*) == 8 && elfHeader->e_machine == EM_X86_64);
 370                 #elif defined(__arm__)
 371                         assert(sizeof(void*) == 4 && elfHeader->e_machine == EM_ARM);
 372                 #else
 373                         #error "Unsupported platform"
 374                 #endif
 375
 376                 SectionHeader *sectionHeader = (SectionHeader*)(elfImage + elfHeader->e_shoff);
 377                 void *entry = nullptr;
 378
 379                 for(int i = 0; i < elfHeader->e_shnum; i++)
 380                 {
 381                         if(sectionHeader[i].sh_type == SHT_PROGBITS)
 382                         {
 383                                 if(sectionHeader[i].sh_flags & SHF_EXECINSTR)
 384                                 {
 385                                         entry = elfImage + sectionHeader[i].sh_offset;
 386                                         codeSize = sectionHeader[i].sh_size;
 387                                 }
 388                         }
 389                         else if(sectionHeader[i].sh_type == SHT_REL)
 390                         {
 391                                 assert(sizeof(void*) == 4 && "UNIMPLEMENTED");   // Only expected/implemented for 32-bit code
 392
 393                                 for(Elf32_Word index = 0; index < sectionHeader[i].sh_size / sectionHeader[i].sh_entsize; index++)
 394                                 {
 395                                         const Elf32_Rel &relocation = ((const Elf32_Rel*)(elfImage + sectionHeader[i].sh_offset))[index];
 396                                         relocateSymbol(elfHeader, relocation, sectionHeader[i]);
 397                                 }
 398                         }
 399                         else if(sectionHeader[i].sh_type == SHT_RELA)
 400                         {
 401                                 assert(sizeof(void*) == 8 && "UNIMPLEMENTED");   // Only expected/implemented for 64-bit code
 402
 403                                 for(Elf32_Word index = 0; index < sectionHeader[i].sh_size / sectionHeader[i].sh_entsize; index++)
 404                                 {
 405                                         const Elf64_Rela &relocation = ((const Elf64_Rela*)(elfImage + sectionHeader[i].sh_offset))[index];
 406                                         relocateSymbol(elfHeader, relocation, sectionHeader[i]);
 407                                 }
 408                         }
 409                 }
 410
 411                 return entry;
 412         }
 413
 414         template<typename T>
 415         struct ExecutableAllocator
 416         {
 417                 ExecutableAllocator() {};
 418                 template<class U> ExecutableAllocator(const ExecutableAllocator<U> &other) {};
 419
 420                 using value_type = T;
 421                 using size_type = std::size_t;
 422
 423                 T *allocate(size_type n)
 424                 {
 425                         #if defined(_WIN32)
 426                                 return (T*)VirtualAlloc(NULL, sizeof(T) * n, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
 427                         #else
 428                                 return (T*)mmap(nullptr, sizeof(T) * n, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 429                         #endif
 430                 }
 431
 432                 void deallocate(T *p, size_type n)
 433                 {
 434                         #if defined(_WIN32)
 435                                 VirtualFree(p, 0, MEM_RELEASE);
 436                         #else
 437                                 munmap(p, sizeof(T) * n);
 438                         #endif
 439                 }
 440         };
 441
 442         class ELFMemoryStreamer : public Ice::ELFStreamer, public Routine
 443         {
 444                 ELFMemoryStreamer(const ELFMemoryStreamer &) = delete;
 445                 ELFMemoryStreamer &operator=(const ELFMemoryStreamer &) = delete;
 446
 447         public:
 448                 ELFMemoryStreamer() : Routine(), entry(nullptr)
 449                 {
 450                         position = 0;
 451                         buffer.reserve(0x1000);
 452                 }
 453
 454                 ~ELFMemoryStreamer() override
 455                 {
 456                         #if defined(_WIN32)
 457                                 if(buffer.size() != 0)
 458                                 {
 459                                         DWORD exeProtection;
 460                                         VirtualProtect(&buffer[0], buffer.size(), oldProtection, &exeProtection);
 461                                 }
 462                         #endif
 463                 }
 464
 465                 void write8(uint8_t Value) override
 466                 {
 467                         if(position == (uint64_t)buffer.size())
 468                         {
 469                                 buffer.push_back(Value);
 470                                 position++;
 471                         }
 472                         else if(position < (uint64_t)buffer.size())
 473                         {
 474                                 buffer[position] = Value;
 475                                 position++;
 476                         }
 477                         else assert(false && "UNIMPLEMENTED");
 478                 }
 479
 480                 void writeBytes(llvm::StringRef Bytes) override
 481                 {
 482                         std::size_t oldSize = buffer.size();
 483                         buffer.resize(oldSize + Bytes.size());
 484                         memcpy(&buffer[oldSize], Bytes.begin(), Bytes.size());
 485                         position += Bytes.size();
 486                 }
 487
 488                 uint64_t tell() const override { return position; }
 489
 490                 void seek(uint64_t Off) override { position = Off; }
 491
 492                 const void *getEntry() override
 493                 {
 494                         if(!entry)
 495                         {
 496                                 position = std::numeric_limits<std::size_t>::max();   // Can't stream more data after this
 497
 498                                 size_t codeSize = 0;
 499                                 entry = loadImage(&buffer[0], codeSize);
 500
 501                                 #if defined(_WIN32)
 502                                         VirtualProtect(&buffer[0], buffer.size(), PAGE_EXECUTE_READ, &oldProtection);
 503                                         FlushInstructionCache(GetCurrentProcess(), NULL, 0);
 504                                 #else
 505                                         mprotect(&buffer[0], buffer.size(), PROT_READ | PROT_EXEC);
 506                                         __builtin___clear_cache((char*)entry, (char*)entry + codeSize);
 507                                 #endif
 508                         }
 509
 510                         return entry;
 511                 }
 512
 513         private:
 514                 void *entry;
 515                 std::vector<uint8_t, ExecutableAllocator<uint8_t>> buffer;
 516                 std::size_t position;
 517
 518                 #if defined(_WIN32)
 519                 DWORD oldProtection;
 520                 #endif
 521         };
 522
 523         Nucleus::Nucleus()
 524         {
 525                 ::codegenMutex.lock();   // Reactor is currently not thread safe
 526
 527                 Ice::ClFlags &Flags = Ice::ClFlags::Flags;
 528                 Ice::ClFlags::getParsedClFlags(Flags);
 529
 530                 #if defined(__arm__)
 531                         Flags.setTargetArch(Ice::Target_ARM32);
 532                         Flags.setTargetInstructionSet(Ice::ARM32InstructionSet_HWDivArm);
 533                 #else   // x86
 534                         Flags.setTargetArch(sizeof(void*) == 8 ? Ice::Target_X8664 : Ice::Target_X8632);
 535                         Flags.setTargetInstructionSet(CPUID::SSE4_1 ? Ice::X86InstructionSet_SSE4_1 : Ice::X86InstructionSet_SSE2);
 536                 #endif
 537                 Flags.setOutFileType(Ice::FT_Elf);
 538                 Flags.setOptLevel(Ice::Opt_2);
 539                 Flags.setApplicationBinaryInterface(Ice::ABI_Platform);
 540                 Flags.setVerbose(false ? Ice::IceV_Most : Ice::IceV_None);
 541                 Flags.setDisableHybridAssembly(true);
 542
 543                 static llvm::raw_os_ostream cout(std::cout);
 544                 static llvm::raw_os_ostream cerr(std::cerr);
 545
 546                 if(false)   // Write out to a file
 547                 {
 548                         std::error_code errorCode;
 549                         ::out = new Ice::Fdstream("out.o", errorCode, llvm::sys::fs::F_None);
 550                         ::elfFile = new Ice::ELFFileStreamer(*out);
 551                         ::context = new Ice::GlobalContext(&cout, &cout, &cerr, elfFile);
 552                 }
 553                 else
 554                 {
 555                         ELFMemoryStreamer *elfMemory = new ELFMemoryStreamer();
 556                         ::context = new Ice::GlobalContext(&cout, &cout, &cerr, elfMemory);
 557                         ::routine = elfMemory;
 558                 }
 559         }
 560
 561         Nucleus::~Nucleus()
 562         {
 563                 delete ::routine;
 564
 565                 delete ::allocator;
 566                 delete ::function;
 567                 delete ::context;
 568
 569                 delete ::elfFile;
 570                 delete ::out;
 571
 572                 ::codegenMutex.unlock();
 573         }
 574
 575         Routine *Nucleus::acquireRoutine(const wchar_t *name, bool runOptimizations)
 576         {
 577                 if(basicBlock->getInsts().empty() || basicBlock->getInsts().back().getKind() != Ice::Inst::Ret)
 578                 {
 579                         createRetVoid();
 580                 }
 581
 582                 std::wstring wideName(name);
 583                 std::string asciiName(wideName.begin(), wideName.end());
 584                 ::function->setFunctionName(Ice::GlobalString::createWithString(::context, asciiName));
 585
 586                 optimize();
 587
 588                 ::function->translate();
 589                 assert(!::function->hasError());
 590
 591                 auto globals = ::function->getGlobalInits();
 592
 593                 if(globals && !globals->empty())
 594                 {
 595                         ::context->getGlobals()->merge(globals.get());
 596                 }
 597
 598                 ::context->emitFileHeader();
 599                 ::function->emitIAS();
 600                 auto assembler = ::function->releaseAssembler();
 601                 auto objectWriter = ::context->getObjectWriter();
 602                 assembler->alignFunction();
 603                 objectWriter->writeFunctionCode(::function->getFunctionName(), false, assembler.get());
 604                 ::context->lowerGlobals("last");
 605                 ::context->lowerConstants();
 606                 ::context->lowerJumpTables();
 607                 objectWriter->setUndefinedSyms(::context->getConstantExternSyms());
 608                 objectWriter->writeNonUserSections();
 609
 610                 Routine *handoffRoutine = ::routine;
 611                 ::routine = nullptr;
 612
 613                 return handoffRoutine;
 614         }
 615
 616         void Nucleus::optimize()
 617         {
 618                 sw::optimize(::function);
 619         }
 620
 621         Value *Nucleus::allocateStackVariable(Type *t, int arraySize)
 622         {
 623                 Ice::Type type = T(t);
 624                 int typeSize = Ice::typeWidthInBytes(type);
 625                 int totalSize = typeSize * (arraySize ? arraySize : 1);
 626
 627                 auto bytes = Ice::ConstantInteger32::create(::context, type, totalSize);
 628                 auto address = ::function->makeVariable(T(getPointerType(t)));
 629                 auto alloca = Ice::InstAlloca::create(::function, address, bytes, typeSize);
 630                 ::function->getEntryNode()->getInsts().push_front(alloca);
 631
 632                 return V(address);
 633         }
 634
 635         BasicBlock *Nucleus::createBasicBlock()
 636         {
 637                 return B(::function->makeNode());
 638         }
 639
 640         BasicBlock *Nucleus::getInsertBlock()
 641         {
 642                 return B(::basicBlock);
 643         }
 644
 645         void Nucleus::setInsertBlock(BasicBlock *basicBlock)
 646         {
 647         //      assert(::basicBlock->getInsts().back().getTerminatorEdges().size() >= 0 && "Previous basic block must have a terminator");
 648                 ::basicBlock = basicBlock;
 649         }
 650
 651         void Nucleus::createFunction(Type *ReturnType, std::vector<Type*> &Params)
 652         {
 653                 uint32_t sequenceNumber = 0;
 654                 ::function = Ice::Cfg::create(::context, sequenceNumber).release();
 655                 ::allocator = new Ice::CfgLocalAllocatorScope(::function);
 656
 657                 for(Type *type : Params)
 658                 {
 659                         Ice::Variable *arg = ::function->makeVariable(T(type));
 660                         ::function->addArg(arg);
 661                 }
 662
 663                 Ice::CfgNode *node = ::function->makeNode();
 664                 ::function->setEntryNode(node);
 665                 ::basicBlock = node;
 666         }
 667
 668         Value *Nucleus::getArgument(unsigned int index)
 669         {
 670                 return V(::function->getArgs()[index]);
 671         }
 672
 673         void Nucleus::createRetVoid()
 674         {
 675                 Ice::InstRet *ret = Ice::InstRet::create(::function);
 676                 ::basicBlock->appendInst(ret);
 677         }
 678
 679         void Nucleus::createRet(Value *v)
 680         {
 681                 Ice::InstRet *ret = Ice::InstRet::create(::function, v);
 682                 ::basicBlock->appendInst(ret);
 683         }
 684
 685         void Nucleus::createBr(BasicBlock *dest)
 686         {
 687                 auto br = Ice::InstBr::create(::function, dest);
 688                 ::basicBlock->appendInst(br);
 689         }
 690
 691         void Nucleus::createCondBr(Value *cond, BasicBlock *ifTrue, BasicBlock *ifFalse)
 692         {
 693                 auto br = Ice::InstBr::create(::function, cond, ifTrue, ifFalse);
 694                 ::basicBlock->appendInst(br);
 695         }
 696
 697         static bool isCommutative(Ice::InstArithmetic::OpKind op)
 698         {
 699                 switch(op)
 700                 {
 701                 case Ice::InstArithmetic::Add:
 702                 case Ice::InstArithmetic::Fadd:
 703                 case Ice::InstArithmetic::Mul:
 704                 case Ice::InstArithmetic::Fmul:
 705                 case Ice::InstArithmetic::And:
 706                 case Ice::InstArithmetic::Or:
 707                 case Ice::InstArithmetic::Xor:
 708                         return true;
 709                 default:
 710                         return false;
 711                 }
 712         }
 713
 714         static Value *createArithmetic(Ice::InstArithmetic::OpKind op, Value *lhs, Value *rhs)
 715         {
 716                 assert(lhs->getType() == rhs->getType() || (llvm::isa<Ice::Constant>(rhs) && (op == Ice::InstArithmetic::Shl || Ice::InstArithmetic::Lshr || Ice::InstArithmetic::Ashr)));
 717
 718                 bool swapOperands = llvm::isa<Ice::Constant>(lhs) && isCommutative(op);
 719
 720                 Ice::Variable *result = ::function->makeVariable(lhs->getType());
 721                 Ice::InstArithmetic *arithmetic = Ice::InstArithmetic::create(::function, op, result, swapOperands ? rhs : lhs, swapOperands ? lhs : rhs);
 722                 ::basicBlock->appendInst(arithmetic);
 723
 724                 return V(result);
 725         }
 726
 727         Value *Nucleus::createAdd(Value *lhs, Value *rhs)
 728         {
 729                 return createArithmetic(Ice::InstArithmetic::Add, lhs, rhs);
 730         }
 731
 732         Value *Nucleus::createSub(Value *lhs, Value *rhs)
 733         {
 734                 return createArithmetic(Ice::InstArithmetic::Sub, lhs, rhs);
 735         }
 736
 737         Value *Nucleus::createMul(Value *lhs, Value *rhs)
 738         {
 739                 return createArithmetic(Ice::InstArithmetic::Mul, lhs, rhs);
 740         }
 741
 742         Value *Nucleus::createUDiv(Value *lhs, Value *rhs)
 743         {
 744                 return createArithmetic(Ice::InstArithmetic::Udiv, lhs, rhs);
 745         }
 746
 747         Value *Nucleus::createSDiv(Value *lhs, Value *rhs)
 748         {
 749                 return createArithmetic(Ice::InstArithmetic::Sdiv, lhs, rhs);
 750         }
 751
 752         Value *Nucleus::createFAdd(Value *lhs, Value *rhs)
 753         {
 754                 return createArithmetic(Ice::InstArithmetic::Fadd, lhs, rhs);
 755         }
 756
 757         Value *Nucleus::createFSub(Value *lhs, Value *rhs)
 758         {
 759                 return createArithmetic(Ice::InstArithmetic::Fsub, lhs, rhs);
 760         }
 761
 762         Value *Nucleus::createFMul(Value *lhs, Value *rhs)
 763         {
 764                 return createArithmetic(Ice::InstArithmetic::Fmul, lhs, rhs);
 765         }
 766
 767         Value *Nucleus::createFDiv(Value *lhs, Value *rhs)
 768         {
 769                 return createArithmetic(Ice::InstArithmetic::Fdiv, lhs, rhs);
 770         }
 771
 772         Value *Nucleus::createURem(Value *lhs, Value *rhs)
 773         {
 774                 return createArithmetic(Ice::InstArithmetic::Urem, lhs, rhs);
 775         }
 776
 777         Value *Nucleus::createSRem(Value *lhs, Value *rhs)
 778         {
 779                 return createArithmetic(Ice::InstArithmetic::Srem, lhs, rhs);
 780         }
 781
 782         Value *Nucleus::createFRem(Value *lhs, Value *rhs)
 783         {
 784                 return createArithmetic(Ice::InstArithmetic::Frem, lhs, rhs);
 785         }
 786
 787         Value *Nucleus::createShl(Value *lhs, Value *rhs)
 788         {
 789                 return createArithmetic(Ice::InstArithmetic::Shl, lhs, rhs);
 790         }
 791
 792         Value *Nucleus::createLShr(Value *lhs, Value *rhs)
 793         {
 794                 return createArithmetic(Ice::InstArithmetic::Lshr, lhs, rhs);
 795         }
 796
 797         Value *Nucleus::createAShr(Value *lhs, Value *rhs)
 798         {
 799                 return createArithmetic(Ice::InstArithmetic::Ashr, lhs, rhs);
 800         }
 801
 802         Value *Nucleus::createAnd(Value *lhs, Value *rhs)
 803         {
 804                 return createArithmetic(Ice::InstArithmetic::And, lhs, rhs);
 805         }
 806
 807         Value *Nucleus::createOr(Value *lhs, Value *rhs)
 808         {
 809                 return createArithmetic(Ice::InstArithmetic::Or, lhs, rhs);
 810         }
 811
 812         Value *Nucleus::createXor(Value *lhs, Value *rhs)
 813         {
 814                 return createArithmetic(Ice::InstArithmetic::Xor, lhs, rhs);
 815         }
 816
 817         Value *Nucleus::createNeg(Value *v)
 818         {
 819                 return createSub(createNullValue(T(v->getType())), v);
 820         }
 821
 822         Value *Nucleus::createFNeg(Value *v)
 823         {
 824                 double c[4] = {-0.0, -0.0, -0.0, -0.0};
 825                 Value *negativeZero = Ice::isVectorType(v->getType()) ?
 826                                       createConstantVector(c, T(v->getType())) :
 827                                       V(::context->getConstantFloat(-0.0f));
 828
 829                 return createFSub(negativeZero, v);
 830         }
 831
 832         Value *Nucleus::createNot(Value *v)
 833         {
 834                 if(Ice::isScalarIntegerType(v->getType()))
 835                 {
 836                         return createXor(v, V(::context->getConstantInt(v->getType(), -1)));
 837                 }
 838                 else   // Vector
 839                 {
 840                         int64_t c[16] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1};
 841                         return createXor(v, createConstantVector(c, T(v->getType())));
 842                 }
 843         }
 844
 845         Value *Nucleus::createLoad(Value *ptr, Type *type, bool isVolatile, unsigned int align)
 846         {
 847                 int valueType = (int)reinterpret_cast<intptr_t>(type);
 848                 Ice::Variable *result = ::function->makeVariable(T(type));
 849
 850                 if(valueType & EmulatedBits)
 851                 {
 852                         if(emulateIntrinsics)
 853                         {
 854                                 if(typeSize(type) == 4)
 855                                 {
 856                                         auto pointer = RValue<Pointer<Byte>>(ptr);
 857                                         Int x = *Pointer<Int>(pointer +1-1);
 858
 859                                         Int4 vector;
 860                                         vector = Insert(vector, x, 0);
 861
 862                                         auto bitcast = Ice::InstCast::create(::function, Ice::InstCast::Bitcast, result, vector.loadValue());
 863                                         ::basicBlock->appendInst(bitcast);
 864                                 }
 865                                 else if(typeSize(type) == 8)
 866                                 {
 867                                         auto pointer = RValue<Pointer<Byte>>(ptr);
 868                                         Int x = *Pointer<Int>(pointer +1-1);
 869                                         Int y = *Pointer<Int>(pointer + 4);
 870
 871                                         Int4 vector;
 872                                         vector = Insert(vector, x, 0);
 873                                         vector = Insert(vector, y, 1);
 874
 875                                         auto bitcast = Ice::InstCast::create(::function, Ice::InstCast::Bitcast, result, vector.loadValue());
 876                                         ::basicBlock->appendInst(bitcast);
 877                                 }
 878                                 else assert(false);
 879                         }
 880                         else
 881                         {
 882                                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::LoadSubVector, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
 883                                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
 884                                 auto load = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
 885                                 load->addArg(ptr);
 886                                 load->addArg(::context->getConstantInt32(typeSize(type)));
 887                                 ::basicBlock->appendInst(load);
 888                         }
 889                 }
 890                 else
 891                 {
 892                         auto load = Ice::InstLoad::create(::function, result, ptr, align);
 893                         ::basicBlock->appendInst(load);
 894                 }
 895
 896                 return V(result);
 897         }
 898
 899         Value *Nucleus::createStore(Value *value, Value *ptr, Type *type, bool isVolatile, unsigned int align)
 900         {
 901                 int valueType = (int)reinterpret_cast<intptr_t>(type);
 902
 903                 if(valueType & EmulatedBits)
 904                 {
 905                         if(emulateIntrinsics)
 906                         {
 907                                 if(typeSize(type) == 4)
 908                                 {
 909                                         Ice::Variable *vector = ::function->makeVariable(Ice::IceType_v4i32);
 910                                         auto bitcast = Ice::InstCast::create(::function, Ice::InstCast::Bitcast, vector, value);
 911                                         ::basicBlock->appendInst(bitcast);
 912
 913                                         RValue<Int4> v(V(vector));
 914
 915                                         auto pointer = RValue<Pointer<Byte>>(ptr);
 916                                         Int x = Extract(v, 0);
 917                                         *Pointer<Int>(pointer) = x;
 918                                 }
 919                                 else if(typeSize(type) == 8)
 920                                 {
 921                                         Ice::Variable *vector = ::function->makeVariable(Ice::IceType_v4i32);
 922                                         auto bitcast = Ice::InstCast::create(::function, Ice::InstCast::Bitcast, vector, value);
 923                                         ::basicBlock->appendInst(bitcast);
 924
 925                                         RValue<Int4> v(V(vector));
 926
 927                                         auto pointer = RValue<Pointer<Byte>>(ptr);
 928                                         Int x = Extract(v, 0);
 929                                         *Pointer<Int>(pointer) = x;
 930                                         Int y = Extract(v, 1);
 931                                         *Pointer<Int>(pointer + 4) = y;
 932                                 }
 933                                 else assert(false);
 934                         }
 935                         else
 936                         {
 937                                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::StoreSubVector, Ice::Intrinsics::SideEffects_T, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_T};
 938                                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
 939                                 auto store = Ice::InstIntrinsicCall::create(::function, 3, nullptr, target, intrinsic);
 940                                 store->addArg(value);
 941                                 store->addArg(ptr);
 942                                 store->addArg(::context->getConstantInt32(typeSize(type)));
 943                                 ::basicBlock->appendInst(store);
 944                         }
 945                 }
 946                 else
 947                 {
 948                         assert(T(value->getType()) == type);
 949
 950                         auto store = Ice::InstStore::create(::function, value, ptr, align);
 951                         ::basicBlock->appendInst(store);
 952                 }
 953
 954                 return value;
 955         }
 956
 957         Value *Nucleus::createGEP(Value *ptr, Type *type, Value *index, bool unsignedIndex)
 958         {
 959                 assert(index->getType() == Ice::IceType_i32);
 960
 961                 if(auto *constant = llvm::dyn_cast<Ice::ConstantInteger32>(index))
 962                 {
 963                         int32_t offset = constant->getValue() * (int)typeSize(type);
 964
 965                         if(offset == 0)
 966                         {
 967                                 return ptr;
 968                         }
 969
 970                         return createAdd(ptr, createConstantInt(offset));
 971                 }
 972
 973                 if(!Ice::isByteSizedType(T(type)))
 974                 {
 975                         index = createMul(index, createConstantInt((int)typeSize(type)));
 976                 }
 977
 978                 if(sizeof(void*) == 8)
 979                 {
 980                         if(unsignedIndex)
 981                         {
 982                                 index = createZExt(index, T(Ice::IceType_i64));
 983                         }
 984                         else
 985                         {
 986                                 index = createSExt(index, T(Ice::IceType_i64));
 987                         }
 988                 }
 989
 990                 return createAdd(ptr, index);
 991         }
 992
 993         Value *Nucleus::createAtomicAdd(Value *ptr, Value *value)
 994         {
 995                 assert(false && "UNIMPLEMENTED"); return nullptr;
 996         }
 997
 998         static Value *createCast(Ice::InstCast::OpKind op, Value *v, Type *destType)
 999         {
1000                 if(v->getType() == T(destType))
1001                 {
1002                         return v;
1003                 }
1004
1005                 Ice::Variable *result = ::function->makeVariable(T(destType));
1006                 Ice::InstCast *cast = Ice::InstCast::create(::function, op, result, v);
1007                 ::basicBlock->appendInst(cast);
1008
1009                 return V(result);
1010         }
1011
1012         Value *Nucleus::createTrunc(Value *v, Type *destType)
1013         {
1014                 return createCast(Ice::InstCast::Trunc, v, destType);
1015         }
1016
1017         Value *Nucleus::createZExt(Value *v, Type *destType)
1018         {
1019                 return createCast(Ice::InstCast::Zext, v, destType);
1020         }
1021
1022         Value *Nucleus::createSExt(Value *v, Type *destType)
1023         {
1024                 return createCast(Ice::InstCast::Sext, v, destType);
1025         }
1026
1027         Value *Nucleus::createFPToSI(Value *v, Type *destType)
1028         {
1029                 return createCast(Ice::InstCast::Fptosi, v, destType);
1030         }
1031
1032         Value *Nucleus::createSIToFP(Value *v, Type *destType)
1033         {
1034                 return createCast(Ice::InstCast::Sitofp, v, destType);
1035         }
1036
1037         Value *Nucleus::createFPTrunc(Value *v, Type *destType)
1038         {
1039                 return createCast(Ice::InstCast::Fptrunc, v, destType);
1040         }
1041
1042         Value *Nucleus::createFPExt(Value *v, Type *destType)
1043         {
1044                 return createCast(Ice::InstCast::Fpext, v, destType);
1045         }
1046
1047         Value *Nucleus::createBitCast(Value *v, Type *destType)
1048         {
1049                 // Bitcasts must be between types of the same logical size. But with emulated narrow vectors we need
1050                 // support for casting between scalars and wide vectors. For platforms where this is not supported,
1051                 // emulate them by writing to the stack and reading back as the destination type.
1052                 if(emulateMismatchedBitCast)
1053                 {
1054                         if(!Ice::isVectorType(v->getType()) && Ice::isVectorType(T(destType)))
1055                         {
1056                                 Value *address = allocateStackVariable(destType);
1057                                 createStore(v, address, T(v->getType()));
1058                                 return createLoad(address, destType);
1059                         }
1060                         else if(Ice::isVectorType(v->getType()) && !Ice::isVectorType(T(destType)))
1061                         {
1062                                 Value *address = allocateStackVariable(T(v->getType()));
1063                                 createStore(v, address, T(v->getType()));
1064                                 return createLoad(address, destType);
1065                         }
1066                 }
1067
1068                 return createCast(Ice::InstCast::Bitcast, v, destType);
1069         }
1070
1071         static Value *createIntCompare(Ice::InstIcmp::ICond condition, Value *lhs, Value *rhs)
1072         {
1073                 assert(lhs->getType() == rhs->getType());
1074
1075                 auto result = ::function->makeVariable(Ice::isScalarIntegerType(lhs->getType()) ? Ice::IceType_i1 : lhs->getType());
1076                 auto cmp = Ice::InstIcmp::create(::function, condition, result, lhs, rhs);
1077                 ::basicBlock->appendInst(cmp);
1078
1079                 return V(result);
1080         }
1081
1082         Value *Nucleus::createICmpEQ(Value *lhs, Value *rhs)
1083         {
1084                 return createIntCompare(Ice::InstIcmp::Eq, lhs, rhs);
1085         }
1086
1087         Value *Nucleus::createICmpNE(Value *lhs, Value *rhs)
1088         {
1089                 return createIntCompare(Ice::InstIcmp::Ne, lhs, rhs);
1090         }
1091
1092         Value *Nucleus::createICmpUGT(Value *lhs, Value *rhs)
1093         {
1094                 return createIntCompare(Ice::InstIcmp::Ugt, lhs, rhs);
1095         }
1096
1097         Value *Nucleus::createICmpUGE(Value *lhs, Value *rhs)
1098         {
1099                 return createIntCompare(Ice::InstIcmp::Uge, lhs, rhs);
1100         }
1101
1102         Value *Nucleus::createICmpULT(Value *lhs, Value *rhs)
1103         {
1104                 return createIntCompare(Ice::InstIcmp::Ult, lhs, rhs);
1105         }
1106
1107         Value *Nucleus::createICmpULE(Value *lhs, Value *rhs)
1108         {
1109                 return createIntCompare(Ice::InstIcmp::Ule, lhs, rhs);
1110         }
1111
1112         Value *Nucleus::createICmpSGT(Value *lhs, Value *rhs)
1113         {
1114                 return createIntCompare(Ice::InstIcmp::Sgt, lhs, rhs);
1115         }
1116
1117         Value *Nucleus::createICmpSGE(Value *lhs, Value *rhs)
1118         {
1119                 return createIntCompare(Ice::InstIcmp::Sge, lhs, rhs);
1120         }
1121
1122         Value *Nucleus::createICmpSLT(Value *lhs, Value *rhs)
1123         {
1124                 return createIntCompare(Ice::InstIcmp::Slt, lhs, rhs);
1125         }
1126
1127         Value *Nucleus::createICmpSLE(Value *lhs, Value *rhs)
1128         {
1129                 return createIntCompare(Ice::InstIcmp::Sle, lhs, rhs);
1130         }
1131
1132         static Value *createFloatCompare(Ice::InstFcmp::FCond condition, Value *lhs, Value *rhs)
1133         {
1134                 assert(lhs->getType() == rhs->getType());
1135                 assert(Ice::isScalarFloatingType(lhs->getType()) || lhs->getType() == Ice::IceType_v4f32);
1136
1137                 auto result = ::function->makeVariable(Ice::isScalarFloatingType(lhs->getType()) ? Ice::IceType_i1 : Ice::IceType_v4i32);
1138                 auto cmp = Ice::InstFcmp::create(::function, condition, result, lhs, rhs);
1139                 ::basicBlock->appendInst(cmp);
1140
1141                 return V(result);
1142         }
1143
1144         Value *Nucleus::createFCmpOEQ(Value *lhs, Value *rhs)
1145         {
1146                 return createFloatCompare(Ice::InstFcmp::Oeq, lhs, rhs);
1147         }
1148
1149         Value *Nucleus::createFCmpOGT(Value *lhs, Value *rhs)
1150         {
1151                 return createFloatCompare(Ice::InstFcmp::Ogt, lhs, rhs);
1152         }
1153
1154         Value *Nucleus::createFCmpOGE(Value *lhs, Value *rhs)
1155         {
1156                 return createFloatCompare(Ice::InstFcmp::Oge, lhs, rhs);
1157         }
1158
1159         Value *Nucleus::createFCmpOLT(Value *lhs, Value *rhs)
1160         {
1161                 return createFloatCompare(Ice::InstFcmp::Olt, lhs, rhs);
1162         }
1163
1164         Value *Nucleus::createFCmpOLE(Value *lhs, Value *rhs)
1165         {
1166                 return createFloatCompare(Ice::InstFcmp::Ole, lhs, rhs);
1167         }
1168
1169         Value *Nucleus::createFCmpONE(Value *lhs, Value *rhs)
1170         {
1171                 return createFloatCompare(Ice::InstFcmp::One, lhs, rhs);
1172         }
1173
1174         Value *Nucleus::createFCmpORD(Value *lhs, Value *rhs)
1175         {
1176                 return createFloatCompare(Ice::InstFcmp::Ord, lhs, rhs);
1177         }
1178
1179         Value *Nucleus::createFCmpUNO(Value *lhs, Value *rhs)
1180         {
1181                 return createFloatCompare(Ice::InstFcmp::Uno, lhs, rhs);
1182         }
1183
1184         Value *Nucleus::createFCmpUEQ(Value *lhs, Value *rhs)
1185         {
1186                 return createFloatCompare(Ice::InstFcmp::Ueq, lhs, rhs);
1187         }
1188
1189         Value *Nucleus::createFCmpUGT(Value *lhs, Value *rhs)
1190         {
1191                 return createFloatCompare(Ice::InstFcmp::Ugt, lhs, rhs);
1192         }
1193
1194         Value *Nucleus::createFCmpUGE(Value *lhs, Value *rhs)
1195         {
1196                 return createFloatCompare(Ice::InstFcmp::Uge, lhs, rhs);
1197         }
1198
1199         Value *Nucleus::createFCmpULT(Value *lhs, Value *rhs)
1200         {
1201                 return createFloatCompare(Ice::InstFcmp::Ult, lhs, rhs);
1202         }
1203
1204         Value *Nucleus::createFCmpULE(Value *lhs, Value *rhs)
1205         {
1206                 return createFloatCompare(Ice::InstFcmp::Ule, lhs, rhs);
1207         }
1208
1209         Value *Nucleus::createFCmpUNE(Value *lhs, Value *rhs)
1210         {
1211                 return createFloatCompare(Ice::InstFcmp::Une, lhs, rhs);
1212         }
1213
1214         Value *Nucleus::createExtractElement(Value *vector, Type *type, int index)
1215         {
1216                 auto result = ::function->makeVariable(T(type));
1217                 auto extract = Ice::InstExtractElement::create(::function, result, vector, ::context->getConstantInt32(index));
1218                 ::basicBlock->appendInst(extract);
1219
1220                 return V(result);
1221         }
1222
1223         Value *Nucleus::createInsertElement(Value *vector, Value *element, int index)
1224         {
1225                 auto result = ::function->makeVariable(vector->getType());
1226                 auto insert = Ice::InstInsertElement::create(::function, result, vector, element, ::context->getConstantInt32(index));
1227                 ::basicBlock->appendInst(insert);
1228
1229                 return V(result);
1230         }
1231
1232         Value *Nucleus::createShuffleVector(Value *V1, Value *V2, const int *select)
1233         {
1234                 assert(V1->getType() == V2->getType());
1235
1236                 int size = Ice::typeNumElements(V1->getType());
1237                 auto result = ::function->makeVariable(V1->getType());
1238                 auto shuffle = Ice::InstShuffleVector::create(::function, result, V1, V2);
1239
1240                 for(int i = 0; i < size; i++)
1241                 {
1242                         shuffle->addIndex(llvm::cast<Ice::ConstantInteger32>(::context->getConstantInt32(select[i])));
1243                 }
1244
1245                 ::basicBlock->appendInst(shuffle);
1246
1247                 return V(result);
1248         }
1249
1250         Value *Nucleus::createSelect(Value *C, Value *ifTrue, Value *ifFalse)
1251         {
1252                 assert(ifTrue->getType() == ifFalse->getType());
1253
1254                 auto result = ::function->makeVariable(ifTrue->getType());
1255                 auto *select = Ice::InstSelect::create(::function, result, C, ifTrue, ifFalse);
1256                 ::basicBlock->appendInst(select);
1257
1258                 return V(result);
1259         }
1260
1261         SwitchCases *Nucleus::createSwitch(Value *control, BasicBlock *defaultBranch, unsigned numCases)
1262         {
1263                 auto switchInst = Ice::InstSwitch::create(::function, numCases, control, defaultBranch);
1264                 ::basicBlock->appendInst(switchInst);
1265
1266                 return reinterpret_cast<SwitchCases*>(switchInst);
1267         }
1268
1269         void Nucleus::addSwitchCase(SwitchCases *switchCases, int label, BasicBlock *branch)
1270         {
1271                 switchCases->addBranch(label, label, branch);
1272         }
1273
1274         void Nucleus::createUnreachable()
1275         {
1276                 Ice::InstUnreachable *unreachable = Ice::InstUnreachable::create(::function);
1277                 ::basicBlock->appendInst(unreachable);
1278         }
1279
1280         static Value *createSwizzle4(Value *val, unsigned char select)
1281         {
1282                 int swizzle[4] =
1283                 {
1284                         (select >> 0) & 0x03,
1285                         (select >> 2) & 0x03,
1286                         (select >> 4) & 0x03,
1287                         (select >> 6) & 0x03,
1288                 };
1289
1290                 return Nucleus::createShuffleVector(val, val, swizzle);
1291         }
1292
1293         static Value *createMask4(Value *lhs, Value *rhs, unsigned char select)
1294         {
1295                 int64_t mask[4] = {0, 0, 0, 0};
1296
1297                 mask[(select >> 0) & 0x03] = -1;
1298                 mask[(select >> 2) & 0x03] = -1;
1299                 mask[(select >> 4) & 0x03] = -1;
1300                 mask[(select >> 6) & 0x03] = -1;
1301
1302                 Value *condition = Nucleus::createConstantVector(mask, T(Ice::IceType_v4i1));
1303                 Value *result = Nucleus::createSelect(condition, rhs, lhs);
1304
1305                 return result;
1306         }
1307
1308         Type *Nucleus::getPointerType(Type *ElementType)
1309         {
1310                 if(sizeof(void*) == 8)
1311                 {
1312                         return T(Ice::IceType_i64);
1313                 }
1314                 else
1315                 {
1316                         return T(Ice::IceType_i32);
1317                 }
1318         }
1319
1320         Value *Nucleus::createNullValue(Type *Ty)
1321         {
1322                 if(Ice::isVectorType(T(Ty)))
1323                 {
1324                         assert(Ice::typeNumElements(T(Ty)) <= 16);
1325                         int64_t c[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
1326                         return createConstantVector(c, Ty);
1327                 }
1328                 else
1329                 {
1330                         return V(::context->getConstantZero(T(Ty)));
1331                 }
1332         }
1333
1334         Value *Nucleus::createConstantLong(int64_t i)
1335         {
1336                 return V(::context->getConstantInt64(i));
1337         }
1338
1339         Value *Nucleus::createConstantInt(int i)
1340         {
1341                 return V(::context->getConstantInt32(i));
1342         }
1343
1344         Value *Nucleus::createConstantInt(unsigned int i)
1345         {
1346                 return V(::context->getConstantInt32(i));
1347         }
1348
1349         Value *Nucleus::createConstantBool(bool b)
1350         {
1351                 return V(::context->getConstantInt1(b));
1352         }
1353
1354         Value *Nucleus::createConstantByte(signed char i)
1355         {
1356                 return V(::context->getConstantInt8(i));
1357         }
1358
1359         Value *Nucleus::createConstantByte(unsigned char i)
1360         {
1361                 return V(::context->getConstantInt8(i));
1362         }
1363
1364         Value *Nucleus::createConstantShort(short i)
1365         {
1366                 return V(::context->getConstantInt16(i));
1367         }
1368
1369         Value *Nucleus::createConstantShort(unsigned short i)
1370         {
1371                 return V(::context->getConstantInt16(i));
1372         }
1373
1374         Value *Nucleus::createConstantFloat(float x)
1375         {
1376                 return V(::context->getConstantFloat(x));
1377         }
1378
1379         Value *Nucleus::createNullPointer(Type *Ty)
1380         {
1381                 return createNullValue(T(sizeof(void*) == 8 ? Ice::IceType_i64 : Ice::IceType_i32));
1382         }
1383
1384         Value *Nucleus::createConstantVector(const int64_t *constants, Type *type)
1385         {
1386                 const int vectorSize = 16;
1387                 assert(Ice::typeWidthInBytes(T(type)) == vectorSize);
1388                 const int alignment = vectorSize;
1389                 auto globalPool = ::function->getGlobalPool();
1390
1391                 const int64_t *i = constants;
1392                 const double *f = reinterpret_cast<const double*>(constants);
1393                 Ice::VariableDeclaration::DataInitializer *dataInitializer = nullptr;
1394
1395                 switch((int)reinterpret_cast<intptr_t>(type))
1396                 {
1397                 case Ice::IceType_v4i32:
1398                 case Ice::IceType_v4i1:
1399                         {
1400                                 const int initializer[4] = {(int)i[0], (int)i[1], (int)i[2], (int)i[3]};
1401                                 static_assert(sizeof(initializer) == vectorSize, "!");
1402                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1403                         }
1404                         break;
1405                 case Ice::IceType_v4f32:
1406                         {
1407                                 const float initializer[4] = {(float)f[0], (float)f[1], (float)f[2], (float)f[3]};
1408                                 static_assert(sizeof(initializer) == vectorSize, "!");
1409                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1410                         }
1411                         break;
1412                 case Ice::IceType_v8i16:
1413                 case Ice::IceType_v8i1:
1414                         {
1415                                 const short initializer[8] = {(short)i[0], (short)i[1], (short)i[2], (short)i[3], (short)i[4], (short)i[5], (short)i[6], (short)i[7]};
1416                                 static_assert(sizeof(initializer) == vectorSize, "!");
1417                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1418                         }
1419                         break;
1420                 case Ice::IceType_v16i8:
1421                 case Ice::IceType_v16i1:
1422                         {
1423                                 const char initializer[16] = {(char)i[0], (char)i[1], (char)i[2], (char)i[3], (char)i[4], (char)i[5], (char)i[6], (char)i[7], (char)i[8], (char)i[9], (char)i[10], (char)i[11], (char)i[12], (char)i[13], (char)i[14], (char)i[15]};
1424                                 static_assert(sizeof(initializer) == vectorSize, "!");
1425                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1426                         }
1427                         break;
1428                 case Type_v2i32:
1429                         {
1430                                 const int initializer[4] = {(int)i[0], (int)i[1], (int)i[0], (int)i[1]};
1431                                 static_assert(sizeof(initializer) == vectorSize, "!");
1432                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1433                         }
1434                         break;
1435                 case Type_v2f32:
1436                         {
1437                                 const float initializer[4] = {(float)f[0], (float)f[1], (float)f[0], (float)f[1]};
1438                                 static_assert(sizeof(initializer) == vectorSize, "!");
1439                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1440                         }
1441                         break;
1442                 case Type_v4i16:
1443                         {
1444                                 const short initializer[8] = {(short)i[0], (short)i[1], (short)i[2], (short)i[3], (short)i[0], (short)i[1], (short)i[2], (short)i[3]};
1445                                 static_assert(sizeof(initializer) == vectorSize, "!");
1446                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1447                         }
1448                         break;
1449                 case Type_v8i8:
1450                         {
1451                                 const char initializer[16] = {(char)i[0], (char)i[1], (char)i[2], (char)i[3], (char)i[4], (char)i[5], (char)i[6], (char)i[7], (char)i[0], (char)i[1], (char)i[2], (char)i[3], (char)i[4], (char)i[5], (char)i[6], (char)i[7]};
1452                                 static_assert(sizeof(initializer) == vectorSize, "!");
1453                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1454                         }
1455                         break;
1456                 case Type_v4i8:
1457                         {
1458                                 const char initializer[16] = {(char)i[0], (char)i[1], (char)i[2], (char)i[3], (char)i[0], (char)i[1], (char)i[2], (char)i[3], (char)i[0], (char)i[1], (char)i[2], (char)i[3], (char)i[0], (char)i[1], (char)i[2], (char)i[3]};
1459                                 static_assert(sizeof(initializer) == vectorSize, "!");
1460                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1461                         }
1462                         break;
1463                 default:
1464                         assert(false && "Unknown constant vector type" && type);
1465                 }
1466
1467                 auto name = Ice::GlobalString::createWithoutString(::context);
1468                 auto *variableDeclaration = Ice::VariableDeclaration::create(globalPool);
1469                 variableDeclaration->setName(name);
1470                 variableDeclaration->setAlignment(alignment);
1471                 variableDeclaration->setIsConstant(true);
1472                 variableDeclaration->addInitializer(dataInitializer);
1473
1474                 ::function->addGlobal(variableDeclaration);
1475
1476                 constexpr int32_t offset = 0;
1477                 Ice::Operand *ptr = ::context->getConstantSym(offset, name);
1478
1479                 Ice::Variable *result = ::function->makeVariable(T(type));
1480                 auto load = Ice::InstLoad::create(::function, result, ptr, alignment);
1481                 ::basicBlock->appendInst(load);
1482
1483                 return V(result);
1484         }
1485
1486         Value *Nucleus::createConstantVector(const double *constants, Type *type)
1487         {
1488                 return createConstantVector((const int64_t*)constants, type);
1489         }
1490
1491         Type *Void::getType()
1492         {
1493                 return T(Ice::IceType_void);
1494         }
1495
1496         Bool::Bool(Argument<Bool> argument)
1497         {
1498                 storeValue(argument.value);
1499         }
1500
1501         Bool::Bool(bool x)
1502         {
1503                 storeValue(Nucleus::createConstantBool(x));
1504         }
1505
1506         Bool::Bool(RValue<Bool> rhs)
1507         {
1508                 storeValue(rhs.value);
1509         }
1510
1511         Bool::Bool(const Bool &rhs)
1512         {
1513                 Value *value = rhs.loadValue();
1514                 storeValue(value);
1515         }
1516
1517         Bool::Bool(const Reference<Bool> &rhs)
1518         {
1519                 Value *value = rhs.loadValue();
1520                 storeValue(value);
1521         }
1522
1523         RValue<Bool> Bool::operator=(RValue<Bool> rhs)
1524         {
1525                 storeValue(rhs.value);
1526
1527                 return rhs;
1528         }
1529
1530         RValue<Bool> Bool::operator=(const Bool &rhs)
1531         {
1532                 Value *value = rhs.loadValue();
1533                 storeValue(value);
1534
1535                 return RValue<Bool>(value);
1536         }
1537
1538         RValue<Bool> Bool::operator=(const Reference<Bool> &rhs)
1539         {
1540                 Value *value = rhs.loadValue();
1541                 storeValue(value);
1542
1543                 return RValue<Bool>(value);
1544         }
1545
1546         RValue<Bool> operator!(RValue<Bool> val)
1547         {
1548                 return RValue<Bool>(Nucleus::createNot(val.value));
1549         }
1550
1551         RValue<Bool> operator&&(RValue<Bool> lhs, RValue<Bool> rhs)
1552         {
1553                 return RValue<Bool>(Nucleus::createAnd(lhs.value, rhs.value));
1554         }
1555
1556         RValue<Bool> operator||(RValue<Bool> lhs, RValue<Bool> rhs)
1557         {
1558                 return RValue<Bool>(Nucleus::createOr(lhs.value, rhs.value));
1559         }
1560
1561         Type *Bool::getType()
1562         {
1563                 return T(Ice::IceType_i1);
1564         }
1565
1566         Byte::Byte(Argument<Byte> argument)
1567         {
1568                 storeValue(argument.value);
1569         }
1570
1571         Byte::Byte(RValue<Int> cast)
1572         {
1573                 Value *integer = Nucleus::createTrunc(cast.value, Byte::getType());
1574
1575                 storeValue(integer);
1576         }
1577
1578         Byte::Byte(RValue<UInt> cast)
1579         {
1580                 Value *integer = Nucleus::createTrunc(cast.value, Byte::getType());
1581
1582                 storeValue(integer);
1583         }
1584
1585         Byte::Byte(RValue<UShort> cast)
1586         {
1587                 Value *integer = Nucleus::createTrunc(cast.value, Byte::getType());
1588
1589                 storeValue(integer);
1590         }
1591
1592         Byte::Byte(int x)
1593         {
1594                 storeValue(Nucleus::createConstantByte((unsigned char)x));
1595         }
1596
1597         Byte::Byte(unsigned char x)
1598         {
1599                 storeValue(Nucleus::createConstantByte(x));
1600         }
1601
1602         Byte::Byte(RValue<Byte> rhs)
1603         {
1604                 storeValue(rhs.value);
1605         }
1606
1607         Byte::Byte(const Byte &rhs)
1608         {
1609                 Value *value = rhs.loadValue();
1610                 storeValue(value);
1611         }
1612
1613         Byte::Byte(const Reference<Byte> &rhs)
1614         {
1615                 Value *value = rhs.loadValue();
1616                 storeValue(value);
1617         }
1618
1619         RValue<Byte> Byte::operator=(RValue<Byte> rhs)
1620         {
1621                 storeValue(rhs.value);
1622
1623                 return rhs;
1624         }
1625
1626         RValue<Byte> Byte::operator=(const Byte &rhs)
1627         {
1628                 Value *value = rhs.loadValue();
1629                 storeValue(value);
1630
1631                 return RValue<Byte>(value);
1632         }
1633
1634         RValue<Byte> Byte::operator=(const Reference<Byte> &rhs)
1635         {
1636                 Value *value = rhs.loadValue();
1637                 storeValue(value);
1638
1639                 return RValue<Byte>(value);
1640         }
1641
1642         RValue<Byte> operator+(RValue<Byte> lhs, RValue<Byte> rhs)
1643         {
1644                 return RValue<Byte>(Nucleus::createAdd(lhs.value, rhs.value));
1645         }
1646
1647         RValue<Byte> operator-(RValue<Byte> lhs, RValue<Byte> rhs)
1648         {
1649                 return RValue<Byte>(Nucleus::createSub(lhs.value, rhs.value));
1650         }
1651
1652         RValue<Byte> operator*(RValue<Byte> lhs, RValue<Byte> rhs)
1653         {
1654                 return RValue<Byte>(Nucleus::createMul(lhs.value, rhs.value));
1655         }
1656
1657         RValue<Byte> operator/(RValue<Byte> lhs, RValue<Byte> rhs)
1658         {
1659                 return RValue<Byte>(Nucleus::createUDiv(lhs.value, rhs.value));
1660         }
1661
1662         RValue<Byte> operator%(RValue<Byte> lhs, RValue<Byte> rhs)
1663         {
1664                 return RValue<Byte>(Nucleus::createURem(lhs.value, rhs.value));
1665         }
1666
1667         RValue<Byte> operator&(RValue<Byte> lhs, RValue<Byte> rhs)
1668         {
1669                 return RValue<Byte>(Nucleus::createAnd(lhs.value, rhs.value));
1670         }
1671
1672         RValue<Byte> operator|(RValue<Byte> lhs, RValue<Byte> rhs)
1673         {
1674                 return RValue<Byte>(Nucleus::createOr(lhs.value, rhs.value));
1675         }
1676
1677         RValue<Byte> operator^(RValue<Byte> lhs, RValue<Byte> rhs)
1678         {
1679                 return RValue<Byte>(Nucleus::createXor(lhs.value, rhs.value));
1680         }
1681
1682         RValue<Byte> operator<<(RValue<Byte> lhs, RValue<Byte> rhs)
1683         {
1684                 return RValue<Byte>(Nucleus::createShl(lhs.value, rhs.value));
1685         }
1686
1687         RValue<Byte> operator>>(RValue<Byte> lhs, RValue<Byte> rhs)
1688         {
1689                 return RValue<Byte>(Nucleus::createLShr(lhs.value, rhs.value));
1690         }
1691
1692         RValue<Byte> operator+=(Byte &lhs, RValue<Byte> rhs)
1693         {
1694                 return lhs = lhs + rhs;
1695         }
1696
1697         RValue<Byte> operator-=(Byte &lhs, RValue<Byte> rhs)
1698         {
1699                 return lhs = lhs - rhs;
1700         }
1701
1702         RValue<Byte> operator*=(Byte &lhs, RValue<Byte> rhs)
1703         {
1704                 return lhs = lhs * rhs;
1705         }
1706
1707         RValue<Byte> operator/=(Byte &lhs, RValue<Byte> rhs)
1708         {
1709                 return lhs = lhs / rhs;
1710         }
1711
1712         RValue<Byte> operator%=(Byte &lhs, RValue<Byte> rhs)
1713         {
1714                 return lhs = lhs % rhs;
1715         }
1716
1717         RValue<Byte> operator&=(Byte &lhs, RValue<Byte> rhs)
1718         {
1719                 return lhs = lhs & rhs;
1720         }
1721
1722         RValue<Byte> operator|=(Byte &lhs, RValue<Byte> rhs)
1723         {
1724                 return lhs = lhs | rhs;
1725         }
1726
1727         RValue<Byte> operator^=(Byte &lhs, RValue<Byte> rhs)
1728         {
1729                 return lhs = lhs ^ rhs;
1730         }
1731
1732         RValue<Byte> operator<<=(Byte &lhs, RValue<Byte> rhs)
1733         {
1734                 return lhs = lhs << rhs;
1735         }
1736
1737         RValue<Byte> operator>>=(Byte &lhs, RValue<Byte> rhs)
1738         {
1739                 return lhs = lhs >> rhs;
1740         }
1741
1742         RValue<Byte> operator+(RValue<Byte> val)
1743         {
1744                 return val;
1745         }
1746
1747         RValue<Byte> operator-(RValue<Byte> val)
1748         {
1749                 return RValue<Byte>(Nucleus::createNeg(val.value));
1750         }
1751
1752         RValue<Byte> operator~(RValue<Byte> val)
1753         {
1754                 return RValue<Byte>(Nucleus::createNot(val.value));
1755         }
1756
1757         RValue<Byte> operator++(Byte &val, int)   // Post-increment
1758         {
1759                 RValue<Byte> res = val;
1760                 val += Byte(1);
1761                 return res;
1762         }
1763
1764         const Byte &operator++(Byte &val)   // Pre-increment
1765         {
1766                 val += Byte(1);
1767                 return val;
1768         }
1769
1770         RValue<Byte> operator--(Byte &val, int)   // Post-decrement
1771         {
1772                 RValue<Byte> res = val;
1773                 val -= Byte(1);
1774                 return res;
1775         }
1776
1777         const Byte &operator--(Byte &val)   // Pre-decrement
1778         {
1779                 val -= Byte(1);
1780                 return val;
1781         }
1782
1783         RValue<Bool> operator<(RValue<Byte> lhs, RValue<Byte> rhs)
1784         {
1785                 return RValue<Bool>(Nucleus::createICmpULT(lhs.value, rhs.value));
1786         }
1787
1788         RValue<Bool> operator<=(RValue<Byte> lhs, RValue<Byte> rhs)
1789         {
1790                 return RValue<Bool>(Nucleus::createICmpULE(lhs.value, rhs.value));
1791         }
1792
1793         RValue<Bool> operator>(RValue<Byte> lhs, RValue<Byte> rhs)
1794         {
1795                 return RValue<Bool>(Nucleus::createICmpUGT(lhs.value, rhs.value));
1796         }
1797
1798         RValue<Bool> operator>=(RValue<Byte> lhs, RValue<Byte> rhs)
1799         {
1800                 return RValue<Bool>(Nucleus::createICmpUGE(lhs.value, rhs.value));
1801         }
1802
1803         RValue<Bool> operator!=(RValue<Byte> lhs, RValue<Byte> rhs)
1804         {
1805                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
1806         }
1807
1808         RValue<Bool> operator==(RValue<Byte> lhs, RValue<Byte> rhs)
1809         {
1810                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
1811         }
1812
1813         Type *Byte::getType()
1814         {
1815                 return T(Ice::IceType_i8);
1816         }
1817
1818         SByte::SByte(Argument<SByte> argument)
1819         {
1820                 storeValue(argument.value);
1821         }
1822
1823         SByte::SByte(RValue<Int> cast)
1824         {
1825                 Value *integer = Nucleus::createTrunc(cast.value, SByte::getType());
1826
1827                 storeValue(integer);
1828         }
1829
1830         SByte::SByte(RValue<Short> cast)
1831         {
1832                 Value *integer = Nucleus::createTrunc(cast.value, SByte::getType());
1833
1834                 storeValue(integer);
1835         }
1836
1837         SByte::SByte(signed char x)
1838         {
1839                 storeValue(Nucleus::createConstantByte(x));
1840         }
1841
1842         SByte::SByte(RValue<SByte> rhs)
1843         {
1844                 storeValue(rhs.value);
1845         }
1846
1847         SByte::SByte(const SByte &rhs)
1848         {
1849                 Value *value = rhs.loadValue();
1850                 storeValue(value);
1851         }
1852
1853         SByte::SByte(const Reference<SByte> &rhs)
1854         {
1855                 Value *value = rhs.loadValue();
1856                 storeValue(value);
1857         }
1858
1859         RValue<SByte> SByte::operator=(RValue<SByte> rhs)
1860         {
1861                 storeValue(rhs.value);
1862
1863                 return rhs;
1864         }
1865
1866         RValue<SByte> SByte::operator=(const SByte &rhs)
1867         {
1868                 Value *value = rhs.loadValue();
1869                 storeValue(value);
1870
1871                 return RValue<SByte>(value);
1872         }
1873
1874         RValue<SByte> SByte::operator=(const Reference<SByte> &rhs)
1875         {
1876                 Value *value = rhs.loadValue();
1877                 storeValue(value);
1878
1879                 return RValue<SByte>(value);
1880         }
1881
1882         RValue<SByte> operator+(RValue<SByte> lhs, RValue<SByte> rhs)
1883         {
1884                 return RValue<SByte>(Nucleus::createAdd(lhs.value, rhs.value));
1885         }
1886
1887         RValue<SByte> operator-(RValue<SByte> lhs, RValue<SByte> rhs)
1888         {
1889                 return RValue<SByte>(Nucleus::createSub(lhs.value, rhs.value));
1890         }
1891
1892         RValue<SByte> operator*(RValue<SByte> lhs, RValue<SByte> rhs)
1893         {
1894                 return RValue<SByte>(Nucleus::createMul(lhs.value, rhs.value));
1895         }
1896
1897         RValue<SByte> operator/(RValue<SByte> lhs, RValue<SByte> rhs)
1898         {
1899                 return RValue<SByte>(Nucleus::createSDiv(lhs.value, rhs.value));
1900         }
1901
1902         RValue<SByte> operator%(RValue<SByte> lhs, RValue<SByte> rhs)
1903         {
1904                 return RValue<SByte>(Nucleus::createSRem(lhs.value, rhs.value));
1905         }
1906
1907         RValue<SByte> operator&(RValue<SByte> lhs, RValue<SByte> rhs)
1908         {
1909                 return RValue<SByte>(Nucleus::createAnd(lhs.value, rhs.value));
1910         }
1911
1912         RValue<SByte> operator|(RValue<SByte> lhs, RValue<SByte> rhs)
1913         {
1914                 return RValue<SByte>(Nucleus::createOr(lhs.value, rhs.value));
1915         }
1916
1917         RValue<SByte> operator^(RValue<SByte> lhs, RValue<SByte> rhs)
1918         {
1919                 return RValue<SByte>(Nucleus::createXor(lhs.value, rhs.value));
1920         }
1921
1922         RValue<SByte> operator<<(RValue<SByte> lhs, RValue<SByte> rhs)
1923         {
1924                 return RValue<SByte>(Nucleus::createShl(lhs.value, rhs.value));
1925         }
1926
1927         RValue<SByte> operator>>(RValue<SByte> lhs, RValue<SByte> rhs)
1928         {
1929                 return RValue<SByte>(Nucleus::createAShr(lhs.value, rhs.value));
1930         }
1931
1932         RValue<SByte> operator+=(SByte &lhs, RValue<SByte> rhs)
1933         {
1934                 return lhs = lhs + rhs;
1935         }
1936
1937         RValue<SByte> operator-=(SByte &lhs, RValue<SByte> rhs)
1938         {
1939                 return lhs = lhs - rhs;
1940         }
1941
1942         RValue<SByte> operator*=(SByte &lhs, RValue<SByte> rhs)
1943         {
1944                 return lhs = lhs * rhs;
1945         }
1946
1947         RValue<SByte> operator/=(SByte &lhs, RValue<SByte> rhs)
1948         {
1949                 return lhs = lhs / rhs;
1950         }
1951
1952         RValue<SByte> operator%=(SByte &lhs, RValue<SByte> rhs)
1953         {
1954                 return lhs = lhs % rhs;
1955         }
1956
1957         RValue<SByte> operator&=(SByte &lhs, RValue<SByte> rhs)
1958         {
1959                 return lhs = lhs & rhs;
1960         }
1961
1962         RValue<SByte> operator|=(SByte &lhs, RValue<SByte> rhs)
1963         {
1964                 return lhs = lhs | rhs;
1965         }
1966
1967         RValue<SByte> operator^=(SByte &lhs, RValue<SByte> rhs)
1968         {
1969                 return lhs = lhs ^ rhs;
1970         }
1971
1972         RValue<SByte> operator<<=(SByte &lhs, RValue<SByte> rhs)
1973         {
1974                 return lhs = lhs << rhs;
1975         }
1976
1977         RValue<SByte> operator>>=(SByte &lhs, RValue<SByte> rhs)
1978         {
1979                 return lhs = lhs >> rhs;
1980         }
1981
1982         RValue<SByte> operator+(RValue<SByte> val)
1983         {
1984                 return val;
1985         }
1986
1987         RValue<SByte> operator-(RValue<SByte> val)
1988         {
1989                 return RValue<SByte>(Nucleus::createNeg(val.value));
1990         }
1991
1992         RValue<SByte> operator~(RValue<SByte> val)
1993         {
1994                 return RValue<SByte>(Nucleus::createNot(val.value));
1995         }
1996
1997         RValue<SByte> operator++(SByte &val, int)   // Post-increment
1998         {
1999                 RValue<SByte> res = val;
2000                 val += SByte(1);
2001                 return res;
2002         }
2003
2004         const SByte &operator++(SByte &val)   // Pre-increment
2005         {
2006                 val += SByte(1);
2007                 return val;
2008         }
2009
2010         RValue<SByte> operator--(SByte &val, int)   // Post-decrement
2011         {
2012                 RValue<SByte> res = val;
2013                 val -= SByte(1);
2014                 return res;
2015         }
2016
2017         const SByte &operator--(SByte &val)   // Pre-decrement
2018         {
2019                 val -= SByte(1);
2020                 return val;
2021         }
2022
2023         RValue<Bool> operator<(RValue<SByte> lhs, RValue<SByte> rhs)
2024         {
2025                 return RValue<Bool>(Nucleus::createICmpSLT(lhs.value, rhs.value));
2026         }
2027
2028         RValue<Bool> operator<=(RValue<SByte> lhs, RValue<SByte> rhs)
2029         {
2030                 return RValue<Bool>(Nucleus::createICmpSLE(lhs.value, rhs.value));
2031         }
2032
2033         RValue<Bool> operator>(RValue<SByte> lhs, RValue<SByte> rhs)
2034         {
2035                 return RValue<Bool>(Nucleus::createICmpSGT(lhs.value, rhs.value));
2036         }
2037
2038         RValue<Bool> operator>=(RValue<SByte> lhs, RValue<SByte> rhs)
2039         {
2040                 return RValue<Bool>(Nucleus::createICmpSGE(lhs.value, rhs.value));
2041         }
2042
2043         RValue<Bool> operator!=(RValue<SByte> lhs, RValue<SByte> rhs)
2044         {
2045                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
2046         }
2047
2048         RValue<Bool> operator==(RValue<SByte> lhs, RValue<SByte> rhs)
2049         {
2050                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
2051         }
2052
2053         Type *SByte::getType()
2054         {
2055                 return T(Ice::IceType_i8);
2056         }
2057
2058         Short::Short(Argument<Short> argument)
2059         {
2060                 storeValue(argument.value);
2061         }
2062
2063         Short::Short(RValue<Int> cast)
2064         {
2065                 Value *integer = Nucleus::createTrunc(cast.value, Short::getType());
2066
2067                 storeValue(integer);
2068         }
2069
2070         Short::Short(short x)
2071         {
2072                 storeValue(Nucleus::createConstantShort(x));
2073         }
2074
2075         Short::Short(RValue<Short> rhs)
2076         {
2077                 storeValue(rhs.value);
2078         }
2079
2080         Short::Short(const Short &rhs)
2081         {
2082                 Value *value = rhs.loadValue();
2083                 storeValue(value);
2084         }
2085
2086         Short::Short(const Reference<Short> &rhs)
2087         {
2088                 Value *value = rhs.loadValue();
2089                 storeValue(value);
2090         }
2091
2092         RValue<Short> Short::operator=(RValue<Short> rhs)
2093         {
2094                 storeValue(rhs.value);
2095
2096                 return rhs;
2097         }
2098
2099         RValue<Short> Short::operator=(const Short &rhs)
2100         {
2101                 Value *value = rhs.loadValue();
2102                 storeValue(value);
2103
2104                 return RValue<Short>(value);
2105         }
2106
2107         RValue<Short> Short::operator=(const Reference<Short> &rhs)
2108         {
2109                 Value *value = rhs.loadValue();
2110                 storeValue(value);
2111
2112                 return RValue<Short>(value);
2113         }
2114
2115         RValue<Short> operator+(RValue<Short> lhs, RValue<Short> rhs)
2116         {
2117                 return RValue<Short>(Nucleus::createAdd(lhs.value, rhs.value));
2118         }
2119
2120         RValue<Short> operator-(RValue<Short> lhs, RValue<Short> rhs)
2121         {
2122                 return RValue<Short>(Nucleus::createSub(lhs.value, rhs.value));
2123         }
2124
2125         RValue<Short> operator*(RValue<Short> lhs, RValue<Short> rhs)
2126         {
2127                 return RValue<Short>(Nucleus::createMul(lhs.value, rhs.value));
2128         }
2129
2130         RValue<Short> operator/(RValue<Short> lhs, RValue<Short> rhs)
2131         {
2132                 return RValue<Short>(Nucleus::createSDiv(lhs.value, rhs.value));
2133         }
2134
2135         RValue<Short> operator%(RValue<Short> lhs, RValue<Short> rhs)
2136         {
2137                 return RValue<Short>(Nucleus::createSRem(lhs.value, rhs.value));
2138         }
2139
2140         RValue<Short> operator&(RValue<Short> lhs, RValue<Short> rhs)
2141         {
2142                 return RValue<Short>(Nucleus::createAnd(lhs.value, rhs.value));
2143         }
2144
2145         RValue<Short> operator|(RValue<Short> lhs, RValue<Short> rhs)
2146         {
2147                 return RValue<Short>(Nucleus::createOr(lhs.value, rhs.value));
2148         }
2149
2150         RValue<Short> operator^(RValue<Short> lhs, RValue<Short> rhs)
2151         {
2152                 return RValue<Short>(Nucleus::createXor(lhs.value, rhs.value));
2153         }
2154
2155         RValue<Short> operator<<(RValue<Short> lhs, RValue<Short> rhs)
2156         {
2157                 return RValue<Short>(Nucleus::createShl(lhs.value, rhs.value));
2158         }
2159
2160         RValue<Short> operator>>(RValue<Short> lhs, RValue<Short> rhs)
2161         {
2162                 return RValue<Short>(Nucleus::createAShr(lhs.value, rhs.value));
2163         }
2164
2165         RValue<Short> operator+=(Short &lhs, RValue<Short> rhs)
2166         {
2167                 return lhs = lhs + rhs;
2168         }
2169
2170         RValue<Short> operator-=(Short &lhs, RValue<Short> rhs)
2171         {
2172                 return lhs = lhs - rhs;
2173         }
2174
2175         RValue<Short> operator*=(Short &lhs, RValue<Short> rhs)
2176         {
2177                 return lhs = lhs * rhs;
2178         }
2179
2180         RValue<Short> operator/=(Short &lhs, RValue<Short> rhs)
2181         {
2182                 return lhs = lhs / rhs;
2183         }
2184
2185         RValue<Short> operator%=(Short &lhs, RValue<Short> rhs)
2186         {
2187                 return lhs = lhs % rhs;
2188         }
2189
2190         RValue<Short> operator&=(Short &lhs, RValue<Short> rhs)
2191         {
2192                 return lhs = lhs & rhs;
2193         }
2194
2195         RValue<Short> operator|=(Short &lhs, RValue<Short> rhs)
2196         {
2197                 return lhs = lhs | rhs;
2198         }
2199
2200         RValue<Short> operator^=(Short &lhs, RValue<Short> rhs)
2201         {
2202                 return lhs = lhs ^ rhs;
2203         }
2204
2205         RValue<Short> operator<<=(Short &lhs, RValue<Short> rhs)
2206         {
2207                 return lhs = lhs << rhs;
2208         }
2209
2210         RValue<Short> operator>>=(Short &lhs, RValue<Short> rhs)
2211         {
2212                 return lhs = lhs >> rhs;
2213         }
2214
2215         RValue<Short> operator+(RValue<Short> val)
2216         {
2217                 return val;
2218         }
2219
2220         RValue<Short> operator-(RValue<Short> val)
2221         {
2222                 return RValue<Short>(Nucleus::createNeg(val.value));
2223         }
2224
2225         RValue<Short> operator~(RValue<Short> val)
2226         {
2227                 return RValue<Short>(Nucleus::createNot(val.value));
2228         }
2229
2230         RValue<Short> operator++(Short &val, int)   // Post-increment
2231         {
2232                 RValue<Short> res = val;
2233                 val += Short(1);
2234                 return res;
2235         }
2236
2237         const Short &operator++(Short &val)   // Pre-increment
2238         {
2239                 val += Short(1);
2240                 return val;
2241         }
2242
2243         RValue<Short> operator--(Short &val, int)   // Post-decrement
2244         {
2245                 RValue<Short> res = val;
2246                 val -= Short(1);
2247                 return res;
2248         }
2249
2250         const Short &operator--(Short &val)   // Pre-decrement
2251         {
2252                 val -= Short(1);
2253                 return val;
2254         }
2255
2256         RValue<Bool> operator<(RValue<Short> lhs, RValue<Short> rhs)
2257         {
2258                 return RValue<Bool>(Nucleus::createICmpSLT(lhs.value, rhs.value));
2259         }
2260
2261         RValue<Bool> operator<=(RValue<Short> lhs, RValue<Short> rhs)
2262         {
2263                 return RValue<Bool>(Nucleus::createICmpSLE(lhs.value, rhs.value));
2264         }
2265
2266         RValue<Bool> operator>(RValue<Short> lhs, RValue<Short> rhs)
2267         {
2268                 return RValue<Bool>(Nucleus::createICmpSGT(lhs.value, rhs.value));
2269         }
2270
2271         RValue<Bool> operator>=(RValue<Short> lhs, RValue<Short> rhs)
2272         {
2273                 return RValue<Bool>(Nucleus::createICmpSGE(lhs.value, rhs.value));
2274         }
2275
2276         RValue<Bool> operator!=(RValue<Short> lhs, RValue<Short> rhs)
2277         {
2278                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
2279         }
2280
2281         RValue<Bool> operator==(RValue<Short> lhs, RValue<Short> rhs)
2282         {
2283                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
2284         }
2285
2286         Type *Short::getType()
2287         {
2288                 return T(Ice::IceType_i16);
2289         }
2290
2291         UShort::UShort(Argument<UShort> argument)
2292         {
2293                 storeValue(argument.value);
2294         }
2295
2296         UShort::UShort(RValue<UInt> cast)
2297         {
2298                 Value *integer = Nucleus::createTrunc(cast.value, UShort::getType());
2299
2300                 storeValue(integer);
2301         }
2302
2303         UShort::UShort(RValue<Int> cast)
2304         {
2305                 Value *integer = Nucleus::createTrunc(cast.value, UShort::getType());
2306
2307                 storeValue(integer);
2308         }
2309
2310         UShort::UShort(unsigned short x)
2311         {
2312                 storeValue(Nucleus::createConstantShort(x));
2313         }
2314
2315         UShort::UShort(RValue<UShort> rhs)
2316         {
2317                 storeValue(rhs.value);
2318         }
2319
2320         UShort::UShort(const UShort &rhs)
2321         {
2322                 Value *value = rhs.loadValue();
2323                 storeValue(value);
2324         }
2325
2326         UShort::UShort(const Reference<UShort> &rhs)
2327         {
2328                 Value *value = rhs.loadValue();
2329                 storeValue(value);
2330         }
2331
2332         RValue<UShort> UShort::operator=(RValue<UShort> rhs)
2333         {
2334                 storeValue(rhs.value);
2335
2336                 return rhs;
2337         }
2338
2339         RValue<UShort> UShort::operator=(const UShort &rhs)
2340         {
2341                 Value *value = rhs.loadValue();
2342                 storeValue(value);
2343
2344                 return RValue<UShort>(value);
2345         }
2346
2347         RValue<UShort> UShort::operator=(const Reference<UShort> &rhs)
2348         {
2349                 Value *value = rhs.loadValue();
2350                 storeValue(value);
2351
2352                 return RValue<UShort>(value);
2353         }
2354
2355         RValue<UShort> operator+(RValue<UShort> lhs, RValue<UShort> rhs)
2356         {
2357                 return RValue<UShort>(Nucleus::createAdd(lhs.value, rhs.value));
2358         }
2359
2360         RValue<UShort> operator-(RValue<UShort> lhs, RValue<UShort> rhs)
2361         {
2362                 return RValue<UShort>(Nucleus::createSub(lhs.value, rhs.value));
2363         }
2364
2365         RValue<UShort> operator*(RValue<UShort> lhs, RValue<UShort> rhs)
2366         {
2367                 return RValue<UShort>(Nucleus::createMul(lhs.value, rhs.value));
2368         }
2369
2370         RValue<UShort> operator/(RValue<UShort> lhs, RValue<UShort> rhs)
2371         {
2372                 return RValue<UShort>(Nucleus::createUDiv(lhs.value, rhs.value));
2373         }
2374
2375         RValue<UShort> operator%(RValue<UShort> lhs, RValue<UShort> rhs)
2376         {
2377                 return RValue<UShort>(Nucleus::createURem(lhs.value, rhs.value));
2378         }
2379
2380         RValue<UShort> operator&(RValue<UShort> lhs, RValue<UShort> rhs)
2381         {
2382                 return RValue<UShort>(Nucleus::createAnd(lhs.value, rhs.value));
2383         }
2384
2385         RValue<UShort> operator|(RValue<UShort> lhs, RValue<UShort> rhs)
2386         {
2387                 return RValue<UShort>(Nucleus::createOr(lhs.value, rhs.value));
2388         }
2389
2390         RValue<UShort> operator^(RValue<UShort> lhs, RValue<UShort> rhs)
2391         {
2392                 return RValue<UShort>(Nucleus::createXor(lhs.value, rhs.value));
2393         }
2394
2395         RValue<UShort> operator<<(RValue<UShort> lhs, RValue<UShort> rhs)
2396         {
2397                 return RValue<UShort>(Nucleus::createShl(lhs.value, rhs.value));
2398         }
2399
2400         RValue<UShort> operator>>(RValue<UShort> lhs, RValue<UShort> rhs)
2401         {
2402                 return RValue<UShort>(Nucleus::createLShr(lhs.value, rhs.value));
2403         }
2404
2405         RValue<UShort> operator+=(UShort &lhs, RValue<UShort> rhs)
2406         {
2407                 return lhs = lhs + rhs;
2408         }
2409
2410         RValue<UShort> operator-=(UShort &lhs, RValue<UShort> rhs)
2411         {
2412                 return lhs = lhs - rhs;
2413         }
2414
2415         RValue<UShort> operator*=(UShort &lhs, RValue<UShort> rhs)
2416         {
2417                 return lhs = lhs * rhs;
2418         }
2419
2420         RValue<UShort> operator/=(UShort &lhs, RValue<UShort> rhs)
2421         {
2422                 return lhs = lhs / rhs;
2423         }
2424
2425         RValue<UShort> operator%=(UShort &lhs, RValue<UShort> rhs)
2426         {
2427                 return lhs = lhs % rhs;
2428         }
2429
2430         RValue<UShort> operator&=(UShort &lhs, RValue<UShort> rhs)
2431         {
2432                 return lhs = lhs & rhs;
2433         }
2434
2435         RValue<UShort> operator|=(UShort &lhs, RValue<UShort> rhs)
2436         {
2437                 return lhs = lhs | rhs;
2438         }
2439
2440         RValue<UShort> operator^=(UShort &lhs, RValue<UShort> rhs)
2441         {
2442                 return lhs = lhs ^ rhs;
2443         }
2444
2445         RValue<UShort> operator<<=(UShort &lhs, RValue<UShort> rhs)
2446         {
2447                 return lhs = lhs << rhs;
2448         }
2449
2450         RValue<UShort> operator>>=(UShort &lhs, RValue<UShort> rhs)
2451         {
2452                 return lhs = lhs >> rhs;
2453         }
2454
2455         RValue<UShort> operator+(RValue<UShort> val)
2456         {
2457                 return val;
2458         }
2459
2460         RValue<UShort> operator-(RValue<UShort> val)
2461         {
2462                 return RValue<UShort>(Nucleus::createNeg(val.value));
2463         }
2464
2465         RValue<UShort> operator~(RValue<UShort> val)
2466         {
2467                 return RValue<UShort>(Nucleus::createNot(val.value));
2468         }
2469
2470         RValue<UShort> operator++(UShort &val, int)   // Post-increment
2471         {
2472                 RValue<UShort> res = val;
2473                 val += UShort(1);
2474                 return res;
2475         }
2476
2477         const UShort &operator++(UShort &val)   // Pre-increment
2478         {
2479                 val += UShort(1);
2480                 return val;
2481         }
2482
2483         RValue<UShort> operator--(UShort &val, int)   // Post-decrement
2484         {
2485                 RValue<UShort> res = val;
2486                 val -= UShort(1);
2487                 return res;
2488         }
2489
2490         const UShort &operator--(UShort &val)   // Pre-decrement
2491         {
2492                 val -= UShort(1);
2493                 return val;
2494         }
2495
2496         RValue<Bool> operator<(RValue<UShort> lhs, RValue<UShort> rhs)
2497         {
2498                 return RValue<Bool>(Nucleus::createICmpULT(lhs.value, rhs.value));
2499         }
2500
2501         RValue<Bool> operator<=(RValue<UShort> lhs, RValue<UShort> rhs)
2502         {
2503                 return RValue<Bool>(Nucleus::createICmpULE(lhs.value, rhs.value));
2504         }
2505
2506         RValue<Bool> operator>(RValue<UShort> lhs, RValue<UShort> rhs)
2507         {
2508                 return RValue<Bool>(Nucleus::createICmpUGT(lhs.value, rhs.value));
2509         }
2510
2511         RValue<Bool> operator>=(RValue<UShort> lhs, RValue<UShort> rhs)
2512         {
2513                 return RValue<Bool>(Nucleus::createICmpUGE(lhs.value, rhs.value));
2514         }
2515
2516         RValue<Bool> operator!=(RValue<UShort> lhs, RValue<UShort> rhs)
2517         {
2518                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
2519         }
2520
2521         RValue<Bool> operator==(RValue<UShort> lhs, RValue<UShort> rhs)
2522         {
2523                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
2524         }
2525
2526         Type *UShort::getType()
2527         {
2528                 return T(Ice::IceType_i16);
2529         }
2530
2531         Byte4::Byte4(RValue<Byte8> cast)
2532         {
2533                 storeValue(Nucleus::createBitCast(cast.value, getType()));
2534         }
2535
2536         Byte4::Byte4(const Reference<Byte4> &rhs)
2537         {
2538                 Value *value = rhs.loadValue();
2539                 storeValue(value);
2540         }
2541
2542         Type *Byte4::getType()
2543         {
2544                 return T(Type_v4i8);
2545         }
2546
2547         Type *SByte4::getType()
2548         {
2549                 return T(Type_v4i8);
2550         }
2551
2552         Byte8::Byte8(uint8_t x0, uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, uint8_t x5, uint8_t x6, uint8_t x7)
2553         {
2554                 int64_t constantVector[8] = {x0, x1, x2, x3, x4, x5, x6, x7};
2555                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
2556         }
2557
2558         Byte8::Byte8(RValue<Byte8> rhs)
2559         {
2560                 storeValue(rhs.value);
2561         }
2562
2563         Byte8::Byte8(const Byte8 &rhs)
2564         {
2565                 Value *value = rhs.loadValue();
2566                 storeValue(value);
2567         }
2568
2569         Byte8::Byte8(const Reference<Byte8> &rhs)
2570         {
2571                 Value *value = rhs.loadValue();
2572                 storeValue(value);
2573         }
2574
2575         RValue<Byte8> Byte8::operator=(RValue<Byte8> rhs)
2576         {
2577                 storeValue(rhs.value);
2578
2579                 return rhs;
2580         }
2581
2582         RValue<Byte8> Byte8::operator=(const Byte8 &rhs)
2583         {
2584                 Value *value = rhs.loadValue();
2585                 storeValue(value);
2586
2587                 return RValue<Byte8>(value);
2588         }
2589
2590         RValue<Byte8> Byte8::operator=(const Reference<Byte8> &rhs)
2591         {
2592                 Value *value = rhs.loadValue();
2593                 storeValue(value);
2594
2595                 return RValue<Byte8>(value);
2596         }
2597
2598         RValue<Byte8> operator+(RValue<Byte8> lhs, RValue<Byte8> rhs)
2599         {
2600                 return RValue<Byte8>(Nucleus::createAdd(lhs.value, rhs.value));
2601         }
2602
2603         RValue<Byte8> operator-(RValue<Byte8> lhs, RValue<Byte8> rhs)
2604         {
2605                 return RValue<Byte8>(Nucleus::createSub(lhs.value, rhs.value));
2606         }
2607
2608 //      RValue<Byte8> operator*(RValue<Byte8> lhs, RValue<Byte8> rhs)
2609 //      {
2610 //              return RValue<Byte8>(Nucleus::createMul(lhs.value, rhs.value));
2611 //      }
2612
2613 //      RValue<Byte8> operator/(RValue<Byte8> lhs, RValue<Byte8> rhs)
2614 //      {
2615 //              return RValue<Byte8>(Nucleus::createUDiv(lhs.value, rhs.value));
2616 //      }
2617
2618 //      RValue<Byte8> operator%(RValue<Byte8> lhs, RValue<Byte8> rhs)
2619 //      {
2620 //              return RValue<Byte8>(Nucleus::createURem(lhs.value, rhs.value));
2621 //      }
2622
2623         RValue<Byte8> operator&(RValue<Byte8> lhs, RValue<Byte8> rhs)
2624         {
2625                 return RValue<Byte8>(Nucleus::createAnd(lhs.value, rhs.value));
2626         }
2627
2628         RValue<Byte8> operator|(RValue<Byte8> lhs, RValue<Byte8> rhs)
2629         {
2630                 return RValue<Byte8>(Nucleus::createOr(lhs.value, rhs.value));
2631         }
2632
2633         RValue<Byte8> operator^(RValue<Byte8> lhs, RValue<Byte8> rhs)
2634         {
2635                 return RValue<Byte8>(Nucleus::createXor(lhs.value, rhs.value));
2636         }
2637
2638 //      RValue<Byte8> operator<<(RValue<Byte8> lhs, unsigned char rhs)
2639 //      {
2640 //              return RValue<Byte8>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
2641 //      }
2642
2643 //      RValue<Byte8> operator>>(RValue<Byte8> lhs, unsigned char rhs)
2644 //      {
2645 //              return RValue<Byte8>(Nucleus::createLShr(lhs.value, V(::context->getConstantInt32(rhs))));
2646 //      }
2647
2648         RValue<Byte8> operator+=(Byte8 &lhs, RValue<Byte8> rhs)
2649         {
2650                 return lhs = lhs + rhs;
2651         }
2652
2653         RValue<Byte8> operator-=(Byte8 &lhs, RValue<Byte8> rhs)
2654         {
2655                 return lhs = lhs - rhs;
2656         }
2657
2658 //      RValue<Byte8> operator*=(Byte8 &lhs, RValue<Byte8> rhs)
2659 //      {
2660 //              return lhs = lhs * rhs;
2661 //      }
2662
2663 //      RValue<Byte8> operator/=(Byte8 &lhs, RValue<Byte8> rhs)
2664 //      {
2665 //              return lhs = lhs / rhs;
2666 //      }
2667
2668 //      RValue<Byte8> operator%=(Byte8 &lhs, RValue<Byte8> rhs)
2669 //      {
2670 //              return lhs = lhs % rhs;
2671 //      }
2672
2673         RValue<Byte8> operator&=(Byte8 &lhs, RValue<Byte8> rhs)
2674         {
2675                 return lhs = lhs & rhs;
2676         }
2677
2678         RValue<Byte8> operator|=(Byte8 &lhs, RValue<Byte8> rhs)
2679         {
2680                 return lhs = lhs | rhs;
2681         }
2682
2683         RValue<Byte8> operator^=(Byte8 &lhs, RValue<Byte8> rhs)
2684         {
2685                 return lhs = lhs ^ rhs;
2686         }
2687
2688 //      RValue<Byte8> operator<<=(Byte8 &lhs, RValue<Byte8> rhs)
2689 //      {
2690 //              return lhs = lhs << rhs;
2691 //      }
2692
2693 //      RValue<Byte8> operator>>=(Byte8 &lhs, RValue<Byte8> rhs)
2694 //      {
2695 //              return lhs = lhs >> rhs;
2696 //      }
2697
2698 //      RValue<Byte8> operator+(RValue<Byte8> val)
2699 //      {
2700 //              return val;
2701 //      }
2702
2703 //      RValue<Byte8> operator-(RValue<Byte8> val)
2704 //      {
2705 //              return RValue<Byte8>(Nucleus::createNeg(val.value));
2706 //      }
2707
2708         RValue<Byte8> operator~(RValue<Byte8> val)
2709         {
2710                 return RValue<Byte8>(Nucleus::createNot(val.value));
2711         }
2712
2713         RValue<Byte> Extract(RValue<Byte8> val, int i)
2714         {
2715                 return RValue<Byte>(Nucleus::createExtractElement(val.value, Byte::getType(), i));
2716         }
2717
2718         RValue<Byte8> Insert(RValue<Byte8> val, RValue<Byte> element, int i)
2719         {
2720                 return RValue<Byte8>(Nucleus::createInsertElement(val.value, element.value, i));
2721         }
2722
2723         RValue<Byte> Saturate(RValue<UShort> x)
2724         {
2725                 return Byte(IfThenElse(Int(x) > 0xFF, Int(0xFF), Int(x)));
2726         }
2727
2728         RValue<Byte8> AddSat(RValue<Byte8> x, RValue<Byte8> y)
2729         {
2730                 if(emulateIntrinsics)
2731                 {
2732                         Byte8 result;
2733                         result = Insert(result, Saturate(UShort(Int(Extract(x, 0))) + UShort(Int(Extract(y, 0)))), 0);
2734                         result = Insert(result, Saturate(UShort(Int(Extract(x, 1))) + UShort(Int(Extract(y, 1)))), 1);
2735                         result = Insert(result, Saturate(UShort(Int(Extract(x, 2))) + UShort(Int(Extract(y, 2)))), 2);
2736                         result = Insert(result, Saturate(UShort(Int(Extract(x, 3))) + UShort(Int(Extract(y, 3)))), 3);
2737                         result = Insert(result, Saturate(UShort(Int(Extract(x, 4))) + UShort(Int(Extract(y, 4)))), 4);
2738                         result = Insert(result, Saturate(UShort(Int(Extract(x, 5))) + UShort(Int(Extract(y, 5)))), 5);
2739                         result = Insert(result, Saturate(UShort(Int(Extract(x, 6))) + UShort(Int(Extract(y, 6)))), 6);
2740                         result = Insert(result, Saturate(UShort(Int(Extract(x, 7))) + UShort(Int(Extract(y, 7)))), 7);
2741
2742                         return result;
2743                 }
2744                 else
2745                 {
2746                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
2747                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::AddSaturateUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
2748                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
2749                         auto paddusb = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
2750                         paddusb->addArg(x.value);
2751                         paddusb->addArg(y.value);
2752                         ::basicBlock->appendInst(paddusb);
2753
2754                         return RValue<Byte8>(V(result));
2755                 }
2756         }
2757
2758         RValue<Byte8> SubSat(RValue<Byte8> x, RValue<Byte8> y)
2759         {
2760                 if(emulateIntrinsics)
2761                 {
2762                         Byte8 result;
2763                         result = Insert(result, Saturate(UShort(Int(Extract(x, 0))) - UShort(Int(Extract(y, 0)))), 0);
2764                         result = Insert(result, Saturate(UShort(Int(Extract(x, 1))) - UShort(Int(Extract(y, 1)))), 1);
2765                         result = Insert(result, Saturate(UShort(Int(Extract(x, 2))) - UShort(Int(Extract(y, 2)))), 2);
2766                         result = Insert(result, Saturate(UShort(Int(Extract(x, 3))) - UShort(Int(Extract(y, 3)))), 3);
2767                         result = Insert(result, Saturate(UShort(Int(Extract(x, 4))) - UShort(Int(Extract(y, 4)))), 4);
2768                         result = Insert(result, Saturate(UShort(Int(Extract(x, 5))) - UShort(Int(Extract(y, 5)))), 5);
2769                         result = Insert(result, Saturate(UShort(Int(Extract(x, 6))) - UShort(Int(Extract(y, 6)))), 6);
2770                         result = Insert(result, Saturate(UShort(Int(Extract(x, 7))) - UShort(Int(Extract(y, 7)))), 7);
2771
2772                         return result;
2773                 }
2774                 else
2775                 {
2776                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
2777                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SubtractSaturateUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
2778                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
2779                         auto psubusw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
2780                         psubusw->addArg(x.value);
2781                         psubusw->addArg(y.value);
2782                         ::basicBlock->appendInst(psubusw);
2783
2784                         return RValue<Byte8>(V(result));
2785                 }
2786         }
2787
2788         RValue<Short4> Unpack(RValue<Byte4> x)
2789         {
2790                 int shuffle[16] = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7};   // Real type is v16i8
2791                 return As<Short4>(Nucleus::createShuffleVector(x.value, x.value, shuffle));
2792         }
2793
2794         RValue<Short4> Unpack(RValue<Byte4> x, RValue<Byte4> y)
2795         {
2796                 return UnpackLow(As<Byte8>(x), As<Byte8>(y));
2797         }
2798
2799         RValue<Short4> UnpackLow(RValue<Byte8> x, RValue<Byte8> y)
2800         {
2801                 int shuffle[16] = {0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23};   // Real type is v16i8
2802                 return As<Short4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
2803         }
2804
2805         RValue<Short4> UnpackHigh(RValue<Byte8> x, RValue<Byte8> y)
2806         {
2807                 int shuffle[16] = {0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23};   // Real type is v16i8
2808                 auto lowHigh = RValue<Byte16>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
2809                 return As<Short4>(Swizzle(As<Int4>(lowHigh), 0xEE));
2810         }
2811
2812         RValue<SByte> Extract(RValue<SByte8> val, int i)
2813         {
2814                 return RValue<SByte>(Nucleus::createExtractElement(val.value, SByte::getType(), i));
2815         }
2816
2817         RValue<SByte8> Insert(RValue<SByte8> val, RValue<SByte> element, int i)
2818         {
2819                 return RValue<SByte8>(Nucleus::createInsertElement(val.value, element.value, i));
2820         }
2821
2822         RValue<SByte8> operator>>(RValue<SByte8> lhs, unsigned char rhs)
2823         {
2824                 if(emulateIntrinsics)
2825                 {
2826                         SByte8 result;
2827                         result = Insert(result, Extract(lhs, 0) >> SByte(rhs), 0);
2828                         result = Insert(result, Extract(lhs, 1) >> SByte(rhs), 1);
2829                         result = Insert(result, Extract(lhs, 2) >> SByte(rhs), 2);
2830                         result = Insert(result, Extract(lhs, 3) >> SByte(rhs), 3);
2831                         result = Insert(result, Extract(lhs, 4) >> SByte(rhs), 4);
2832                         result = Insert(result, Extract(lhs, 5) >> SByte(rhs), 5);
2833                         result = Insert(result, Extract(lhs, 6) >> SByte(rhs), 6);
2834                         result = Insert(result, Extract(lhs, 7) >> SByte(rhs), 7);
2835
2836                         return result;
2837                 }
2838                 else
2839                 {
2840                         #if defined(__i386__) || defined(__x86_64__)
2841                                 // SSE2 doesn't support byte vector shifts, so shift as shorts and recombine.
2842                                 RValue<Short4> hi = (As<Short4>(lhs) >> rhs) & Short4(0xFF00);
2843                                 RValue<Short4> lo = As<Short4>(As<UShort4>((As<Short4>(lhs) << 8) >> rhs) >> 8);
2844
2845                                 return As<SByte8>(hi | lo);
2846                         #else
2847                                 return RValue<SByte8>(Nucleus::createAShr(lhs.value, V(::context->getConstantInt32(rhs))));
2848                         #endif
2849                 }
2850         }
2851
2852         RValue<Int> SignMask(RValue<Byte8> x)
2853         {
2854                 if(emulateIntrinsics)
2855                 {
2856                         Byte8 xx = As<Byte8>(As<SByte8>(x) >> 7) & Byte8(0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80);
2857                         return Int(Extract(xx, 0)) | Int(Extract(xx, 1)) | Int(Extract(xx, 2)) | Int(Extract(xx, 3)) | Int(Extract(xx, 4)) | Int(Extract(xx, 5)) | Int(Extract(xx, 6)) | Int(Extract(xx, 7));
2858                 }
2859                 else
2860                 {
2861                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_i32);
2862                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SignMask, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
2863                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
2864                         auto movmsk = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
2865                         movmsk->addArg(x.value);
2866                         ::basicBlock->appendInst(movmsk);
2867
2868                         return RValue<Int>(V(result));
2869                 }
2870         }
2871
2872 //      RValue<Byte8> CmpGT(RValue<Byte8> x, RValue<Byte8> y)
2873 //      {
2874 //              return RValue<Byte8>(createIntCompare(Ice::InstIcmp::Ugt, x.value, y.value));
2875 //      }
2876
2877         RValue<Byte8> CmpEQ(RValue<Byte8> x, RValue<Byte8> y)
2878         {
2879                 return RValue<Byte8>(Nucleus::createICmpEQ(x.value, y.value));
2880         }
2881
2882         Type *Byte8::getType()
2883         {
2884                 return T(Type_v8i8);
2885         }
2886
2887         SByte8::SByte8(uint8_t x0, uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, uint8_t x5, uint8_t x6, uint8_t x7)
2888         {
2889                 int64_t constantVector[8] = { x0, x1, x2, x3, x4, x5, x6, x7 };
2890                 Value *vector = V(Nucleus::createConstantVector(constantVector, getType()));
2891
2892                 storeValue(Nucleus::createBitCast(vector, getType()));
2893         }
2894
2895         SByte8::SByte8(RValue<SByte8> rhs)
2896         {
2897                 storeValue(rhs.value);
2898         }
2899
2900         SByte8::SByte8(const SByte8 &rhs)
2901         {
2902                 Value *value = rhs.loadValue();
2903                 storeValue(value);
2904         }
2905
2906         SByte8::SByte8(const Reference<SByte8> &rhs)
2907         {
2908                 Value *value = rhs.loadValue();
2909                 storeValue(value);
2910         }
2911
2912         RValue<SByte8> SByte8::operator=(RValue<SByte8> rhs)
2913         {
2914                 storeValue(rhs.value);
2915
2916                 return rhs;
2917         }
2918
2919         RValue<SByte8> SByte8::operator=(const SByte8 &rhs)
2920         {
2921                 Value *value = rhs.loadValue();
2922                 storeValue(value);
2923
2924                 return RValue<SByte8>(value);
2925         }
2926
2927         RValue<SByte8> SByte8::operator=(const Reference<SByte8> &rhs)
2928         {
2929                 Value *value = rhs.loadValue();
2930                 storeValue(value);
2931
2932                 return RValue<SByte8>(value);
2933         }
2934
2935         RValue<SByte8> operator+(RValue<SByte8> lhs, RValue<SByte8> rhs)
2936         {
2937                 return RValue<SByte8>(Nucleus::createAdd(lhs.value, rhs.value));
2938         }
2939
2940         RValue<SByte8> operator-(RValue<SByte8> lhs, RValue<SByte8> rhs)
2941         {
2942                 return RValue<SByte8>(Nucleus::createSub(lhs.value, rhs.value));
2943         }
2944
2945 //      RValue<SByte8> operator*(RValue<SByte8> lhs, RValue<SByte8> rhs)
2946 //      {
2947 //              return RValue<SByte8>(Nucleus::createMul(lhs.value, rhs.value));
2948 //      }
2949
2950 //      RValue<SByte8> operator/(RValue<SByte8> lhs, RValue<SByte8> rhs)
2951 //      {
2952 //              return RValue<SByte8>(Nucleus::createSDiv(lhs.value, rhs.value));
2953 //      }
2954
2955 //      RValue<SByte8> operator%(RValue<SByte8> lhs, RValue<SByte8> rhs)
2956 //      {
2957 //              return RValue<SByte8>(Nucleus::createSRem(lhs.value, rhs.value));
2958 //      }
2959
2960         RValue<SByte8> operator&(RValue<SByte8> lhs, RValue<SByte8> rhs)
2961         {
2962                 return RValue<SByte8>(Nucleus::createAnd(lhs.value, rhs.value));
2963         }
2964
2965         RValue<SByte8> operator|(RValue<SByte8> lhs, RValue<SByte8> rhs)
2966         {
2967                 return RValue<SByte8>(Nucleus::createOr(lhs.value, rhs.value));
2968         }
2969
2970         RValue<SByte8> operator^(RValue<SByte8> lhs, RValue<SByte8> rhs)
2971         {
2972                 return RValue<SByte8>(Nucleus::createXor(lhs.value, rhs.value));
2973         }
2974
2975 //      RValue<SByte8> operator<<(RValue<SByte8> lhs, unsigned char rhs)
2976 //      {
2977 //              return RValue<SByte8>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
2978 //      }
2979
2980 //      RValue<SByte8> operator>>(RValue<SByte8> lhs, unsigned char rhs)
2981 //      {
2982 //              return RValue<SByte8>(Nucleus::createAShr(lhs.value, V(::context->getConstantInt32(rhs))));
2983 //      }
2984
2985         RValue<SByte8> operator+=(SByte8 &lhs, RValue<SByte8> rhs)
2986         {
2987                 return lhs = lhs + rhs;
2988         }
2989
2990         RValue<SByte8> operator-=(SByte8 &lhs, RValue<SByte8> rhs)
2991         {
2992                 return lhs = lhs - rhs;
2993         }
2994
2995 //      RValue<SByte8> operator*=(SByte8 &lhs, RValue<SByte8> rhs)
2996 //      {
2997 //              return lhs = lhs * rhs;
2998 //      }
2999
3000 //      RValue<SByte8> operator/=(SByte8 &lhs, RValue<SByte8> rhs)
3001 //      {
3002 //              return lhs = lhs / rhs;
3003 //      }
3004
3005 //      RValue<SByte8> operator%=(SByte8 &lhs, RValue<SByte8> rhs)
3006 //      {
3007 //              return lhs = lhs % rhs;
3008 //      }
3009
3010         RValue<SByte8> operator&=(SByte8 &lhs, RValue<SByte8> rhs)
3011         {
3012                 return lhs = lhs & rhs;
3013         }
3014
3015         RValue<SByte8> operator|=(SByte8 &lhs, RValue<SByte8> rhs)
3016         {
3017                 return lhs = lhs | rhs;
3018         }
3019
3020         RValue<SByte8> operator^=(SByte8 &lhs, RValue<SByte8> rhs)
3021         {
3022                 return lhs = lhs ^ rhs;
3023         }
3024
3025 //      RValue<SByte8> operator<<=(SByte8 &lhs, RValue<SByte8> rhs)
3026 //      {
3027 //              return lhs = lhs << rhs;
3028 //      }
3029
3030 //      RValue<SByte8> operator>>=(SByte8 &lhs, RValue<SByte8> rhs)
3031 //      {
3032 //              return lhs = lhs >> rhs;
3033 //      }
3034
3035 //      RValue<SByte8> operator+(RValue<SByte8> val)
3036 //      {
3037 //              return val;
3038 //      }
3039
3040 //      RValue<SByte8> operator-(RValue<SByte8> val)
3041 //      {
3042 //              return RValue<SByte8>(Nucleus::createNeg(val.value));
3043 //      }
3044
3045         RValue<SByte8> operator~(RValue<SByte8> val)
3046         {
3047                 return RValue<SByte8>(Nucleus::createNot(val.value));
3048         }
3049
3050         RValue<SByte> Saturate(RValue<Short> x)
3051         {
3052                 return SByte(IfThenElse(Int(x) > 0x7F, Int(0x7F), IfThenElse(Int(x) < -0x80, Int(0x80), Int(x))));
3053         }
3054
3055         RValue<SByte8> AddSat(RValue<SByte8> x, RValue<SByte8> y)
3056         {
3057                 if(emulateIntrinsics)
3058                 {
3059                         SByte8 result;
3060                         result = Insert(result, Saturate(Short(Int(Extract(x, 0))) + Short(Int(Extract(y, 0)))), 0);
3061                         result = Insert(result, Saturate(Short(Int(Extract(x, 1))) + Short(Int(Extract(y, 1)))), 1);
3062                         result = Insert(result, Saturate(Short(Int(Extract(x, 2))) + Short(Int(Extract(y, 2)))), 2);
3063                         result = Insert(result, Saturate(Short(Int(Extract(x, 3))) + Short(Int(Extract(y, 3)))), 3);
3064                         result = Insert(result, Saturate(Short(Int(Extract(x, 4))) + Short(Int(Extract(y, 4)))), 4);
3065                         result = Insert(result, Saturate(Short(Int(Extract(x, 5))) + Short(Int(Extract(y, 5)))), 5);
3066                         result = Insert(result, Saturate(Short(Int(Extract(x, 6))) + Short(Int(Extract(y, 6)))), 6);
3067                         result = Insert(result, Saturate(Short(Int(Extract(x, 7))) + Short(Int(Extract(y, 7)))), 7);
3068
3069                         return result;
3070                 }
3071                 else
3072                 {
3073                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
3074                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::AddSaturateSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3075                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
3076                         auto paddsb = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3077                         paddsb->addArg(x.value);
3078                         paddsb->addArg(y.value);
3079                         ::basicBlock->appendInst(paddsb);
3080
3081                         return RValue<SByte8>(V(result));
3082                 }
3083         }
3084
3085         RValue<SByte8> SubSat(RValue<SByte8> x, RValue<SByte8> y)
3086         {
3087                 if(emulateIntrinsics)
3088                 {
3089                         SByte8 result;
3090                         result = Insert(result, Saturate(Short(Int(Extract(x, 0))) - Short(Int(Extract(y, 0)))), 0);
3091                         result = Insert(result, Saturate(Short(Int(Extract(x, 1))) - Short(Int(Extract(y, 1)))), 1);
3092                         result = Insert(result, Saturate(Short(Int(Extract(x, 2))) - Short(Int(Extract(y, 2)))), 2);
3093                         result = Insert(result, Saturate(Short(Int(Extract(x, 3))) - Short(Int(Extract(y, 3)))), 3);
3094                         result = Insert(result, Saturate(Short(Int(Extract(x, 4))) - Short(Int(Extract(y, 4)))), 4);
3095                         result = Insert(result, Saturate(Short(Int(Extract(x, 5))) - Short(Int(Extract(y, 5)))), 5);
3096                         result = Insert(result, Saturate(Short(Int(Extract(x, 6))) - Short(Int(Extract(y, 6)))), 6);
3097                         result = Insert(result, Saturate(Short(Int(Extract(x, 7))) - Short(Int(Extract(y, 7)))), 7);
3098
3099                         return result;
3100                 }
3101                 else
3102                 {
3103                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
3104                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SubtractSaturateSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3105                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
3106                         auto psubsb = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3107                         psubsb->addArg(x.value);
3108                         psubsb->addArg(y.value);
3109                         ::basicBlock->appendInst(psubsb);
3110
3111                         return RValue<SByte8>(V(result));
3112                 }
3113         }
3114
3115         RValue<Short4> UnpackLow(RValue<SByte8> x, RValue<SByte8> y)
3116         {
3117                 int shuffle[16] = {0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23};   // Real type is v16i8
3118                 return As<Short4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
3119         }
3120
3121         RValue<Short4> UnpackHigh(RValue<SByte8> x, RValue<SByte8> y)
3122         {
3123                 int shuffle[16] = {0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23};   // Real type is v16i8
3124                 auto lowHigh = RValue<Byte16>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
3125                 return As<Short4>(Swizzle(As<Int4>(lowHigh), 0xEE));
3126         }
3127
3128         RValue<Int> SignMask(RValue<SByte8> x)
3129         {
3130                 if(emulateIntrinsics)
3131                 {
3132                         SByte8 xx = (x >> 7) & SByte8(0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80);
3133                         return Int(Extract(xx, 0)) | Int(Extract(xx, 1)) | Int(Extract(xx, 2)) | Int(Extract(xx, 3)) | Int(Extract(xx, 4)) | Int(Extract(xx, 5)) | Int(Extract(xx, 6)) | Int(Extract(xx, 7));
3134                 }
3135                 else
3136                 {
3137                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_i32);
3138                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SignMask, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3139                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
3140                         auto movmsk = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
3141                         movmsk->addArg(x.value);
3142                         ::basicBlock->appendInst(movmsk);
3143
3144                         return RValue<Int>(V(result));
3145                 }
3146         }
3147
3148         RValue<Byte8> CmpGT(RValue<SByte8> x, RValue<SByte8> y)
3149         {
3150                 return RValue<Byte8>(createIntCompare(Ice::InstIcmp::Sgt, x.value, y.value));
3151         }
3152
3153         RValue<Byte8> CmpEQ(RValue<SByte8> x, RValue<SByte8> y)
3154         {
3155                 return RValue<Byte8>(Nucleus::createICmpEQ(x.value, y.value));
3156         }
3157
3158         Type *SByte8::getType()
3159         {
3160                 return T(Type_v8i8);
3161         }
3162
3163         Byte16::Byte16(RValue<Byte16> rhs)
3164         {
3165                 storeValue(rhs.value);
3166         }
3167
3168         Byte16::Byte16(const Byte16 &rhs)
3169         {
3170                 Value *value = rhs.loadValue();
3171                 storeValue(value);
3172         }
3173
3174         Byte16::Byte16(const Reference<Byte16> &rhs)
3175         {
3176                 Value *value = rhs.loadValue();
3177                 storeValue(value);
3178         }
3179
3180         RValue<Byte16> Byte16::operator=(RValue<Byte16> rhs)
3181         {
3182                 storeValue(rhs.value);
3183
3184                 return rhs;
3185         }
3186
3187         RValue<Byte16> Byte16::operator=(const Byte16 &rhs)
3188         {
3189                 Value *value = rhs.loadValue();
3190                 storeValue(value);
3191
3192                 return RValue<Byte16>(value);
3193         }
3194
3195         RValue<Byte16> Byte16::operator=(const Reference<Byte16> &rhs)
3196         {
3197                 Value *value = rhs.loadValue();
3198                 storeValue(value);
3199
3200                 return RValue<Byte16>(value);
3201         }
3202
3203         Type *Byte16::getType()
3204         {
3205                 return T(Ice::IceType_v16i8);
3206         }
3207
3208         Type *SByte16::getType()
3209         {
3210                 return T(Ice::IceType_v16i8);
3211         }
3212
3213         Short2::Short2(RValue<Short4> cast)
3214         {
3215                 storeValue(Nucleus::createBitCast(cast.value, getType()));
3216         }
3217
3218         Type *Short2::getType()
3219         {
3220                 return T(Type_v2i16);
3221         }
3222
3223         UShort2::UShort2(RValue<UShort4> cast)
3224         {
3225                 storeValue(Nucleus::createBitCast(cast.value, getType()));
3226         }
3227
3228         Type *UShort2::getType()
3229         {
3230                 return T(Type_v2i16);
3231         }
3232
3233         Short4::Short4(RValue<Int> cast)
3234         {
3235                 Value *vector = loadValue();
3236                 Value *element = Nucleus::createTrunc(cast.value, Short::getType());
3237                 Value *insert = Nucleus::createInsertElement(vector, element, 0);
3238                 Value *swizzle = Swizzle(RValue<Short4>(insert), 0x00).value;
3239
3240                 storeValue(swizzle);
3241         }
3242
3243         Short4::Short4(RValue<Int4> cast)
3244         {
3245                 int select[8] = {0, 2, 4, 6, 0, 2, 4, 6};
3246                 Value *short8 = Nucleus::createBitCast(cast.value, Short8::getType());
3247                 Value *packed = Nucleus::createShuffleVector(short8, short8, select);
3248
3249                 Value *int2 = RValue<Int2>(Int2(As<Int4>(packed))).value;
3250                 Value *short4 = Nucleus::createBitCast(int2, Short4::getType());
3251
3252                 storeValue(short4);
3253         }
3254
3255 //      Short4::Short4(RValue<Float> cast)
3256 //      {
3257 //      }
3258
3259         Short4::Short4(RValue<Float4> cast)
3260         {
3261                 assert(false && "UNIMPLEMENTED");
3262         }
3263
3264         Short4::Short4(short xyzw)
3265         {
3266                 int64_t constantVector[4] = {xyzw, xyzw, xyzw, xyzw};
3267                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
3268         }
3269
3270         Short4::Short4(short x, short y, short z, short w)
3271         {
3272                 int64_t constantVector[4] = {x, y, z, w};
3273                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
3274         }
3275
3276         Short4::Short4(RValue<Short4> rhs)
3277         {
3278                 storeValue(rhs.value);
3279         }
3280
3281         Short4::Short4(const Short4 &rhs)
3282         {
3283                 Value *value = rhs.loadValue();
3284                 storeValue(value);
3285         }
3286
3287         Short4::Short4(const Reference<Short4> &rhs)
3288         {
3289                 Value *value = rhs.loadValue();
3290                 storeValue(value);
3291         }
3292
3293         Short4::Short4(RValue<UShort4> rhs)
3294         {
3295                 storeValue(rhs.value);
3296         }
3297
3298         Short4::Short4(const UShort4 &rhs)
3299         {
3300                 storeValue(rhs.loadValue());
3301         }
3302
3303         Short4::Short4(const Reference<UShort4> &rhs)
3304         {
3305                 storeValue(rhs.loadValue());
3306         }
3307
3308         RValue<Short4> Short4::operator=(RValue<Short4> rhs)
3309         {
3310                 storeValue(rhs.value);
3311
3312                 return rhs;
3313         }
3314
3315         RValue<Short4> Short4::operator=(const Short4 &rhs)
3316         {
3317                 Value *value = rhs.loadValue();
3318                 storeValue(value);
3319
3320                 return RValue<Short4>(value);
3321         }
3322
3323         RValue<Short4> Short4::operator=(const Reference<Short4> &rhs)
3324         {
3325                 Value *value = rhs.loadValue();
3326                 storeValue(value);
3327
3328                 return RValue<Short4>(value);
3329         }
3330
3331         RValue<Short4> Short4::operator=(RValue<UShort4> rhs)
3332         {
3333                 storeValue(rhs.value);
3334
3335                 return RValue<Short4>(rhs);
3336         }
3337
3338         RValue<Short4> Short4::operator=(const UShort4 &rhs)
3339         {
3340                 Value *value = rhs.loadValue();
3341                 storeValue(value);
3342
3343                 return RValue<Short4>(value);
3344         }
3345
3346         RValue<Short4> Short4::operator=(const Reference<UShort4> &rhs)
3347         {
3348                 Value *value = rhs.loadValue();
3349                 storeValue(value);
3350
3351                 return RValue<Short4>(value);
3352         }
3353
3354         RValue<Short4> operator+(RValue<Short4> lhs, RValue<Short4> rhs)
3355         {
3356                 return RValue<Short4>(Nucleus::createAdd(lhs.value, rhs.value));
3357         }
3358
3359         RValue<Short4> operator-(RValue<Short4> lhs, RValue<Short4> rhs)
3360         {
3361                 return RValue<Short4>(Nucleus::createSub(lhs.value, rhs.value));
3362         }
3363
3364         RValue<Short4> operator*(RValue<Short4> lhs, RValue<Short4> rhs)
3365         {
3366                 return RValue<Short4>(Nucleus::createMul(lhs.value, rhs.value));
3367         }
3368
3369 //      RValue<Short4> operator/(RValue<Short4> lhs, RValue<Short4> rhs)
3370 //      {
3371 //              return RValue<Short4>(Nucleus::createSDiv(lhs.value, rhs.value));
3372 //      }
3373
3374 //      RValue<Short4> operator%(RValue<Short4> lhs, RValue<Short4> rhs)
3375 //      {
3376 //              return RValue<Short4>(Nucleus::createSRem(lhs.value, rhs.value));
3377 //      }
3378
3379         RValue<Short4> operator&(RValue<Short4> lhs, RValue<Short4> rhs)
3380         {
3381                 return RValue<Short4>(Nucleus::createAnd(lhs.value, rhs.value));
3382         }
3383
3384         RValue<Short4> operator|(RValue<Short4> lhs, RValue<Short4> rhs)
3385         {
3386                 return RValue<Short4>(Nucleus::createOr(lhs.value, rhs.value));
3387         }
3388
3389         RValue<Short4> operator^(RValue<Short4> lhs, RValue<Short4> rhs)
3390         {
3391                 return RValue<Short4>(Nucleus::createXor(lhs.value, rhs.value));
3392         }
3393
3394         RValue<Short4> operator<<(RValue<Short4> lhs, unsigned char rhs)
3395         {
3396                 if(emulateIntrinsics)
3397                 {
3398                         Short4 result;
3399                         result = Insert(result, Extract(lhs, 0) << Short(rhs), 0);
3400                         result = Insert(result, Extract(lhs, 1) << Short(rhs), 1);
3401                         result = Insert(result, Extract(lhs, 2) << Short(rhs), 2);
3402                         result = Insert(result, Extract(lhs, 3) << Short(rhs), 3);
3403
3404                         return result;
3405                 }
3406                 else
3407                 {
3408                         return RValue<Short4>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
3409                 }
3410         }
3411
3412         RValue<Short4> operator>>(RValue<Short4> lhs, unsigned char rhs)
3413         {
3414                 if(emulateIntrinsics)
3415                 {
3416                         Short4 result;
3417                         result = Insert(result, Extract(lhs, 0) >> Short(rhs), 0);
3418                         result = Insert(result, Extract(lhs, 1) >> Short(rhs), 1);
3419                         result = Insert(result, Extract(lhs, 2) >> Short(rhs), 2);
3420                         result = Insert(result, Extract(lhs, 3) >> Short(rhs), 3);
3421
3422                         return result;
3423                 }
3424                 else
3425                 {
3426                         return RValue<Short4>(Nucleus::createAShr(lhs.value, V(::context->getConstantInt32(rhs))));
3427                 }
3428         }
3429
3430         RValue<Short4> operator+=(Short4 &lhs, RValue<Short4> rhs)
3431         {
3432                 return lhs = lhs + rhs;
3433         }
3434
3435         RValue<Short4> operator-=(Short4 &lhs, RValue<Short4> rhs)
3436         {
3437                 return lhs = lhs - rhs;
3438         }
3439
3440         RValue<Short4> operator*=(Short4 &lhs, RValue<Short4> rhs)
3441         {
3442                 return lhs = lhs * rhs;
3443         }
3444
3445 //      RValue<Short4> operator/=(Short4 &lhs, RValue<Short4> rhs)
3446 //      {
3447 //              return lhs = lhs / rhs;
3448 //      }
3449
3450 //      RValue<Short4> operator%=(Short4 &lhs, RValue<Short4> rhs)
3451 //      {
3452 //              return lhs = lhs % rhs;
3453 //      }
3454
3455         RValue<Short4> operator&=(Short4 &lhs, RValue<Short4> rhs)
3456         {
3457                 return lhs = lhs & rhs;
3458         }
3459
3460         RValue<Short4> operator|=(Short4 &lhs, RValue<Short4> rhs)
3461         {
3462                 return lhs = lhs | rhs;
3463         }
3464
3465         RValue<Short4> operator^=(Short4 &lhs, RValue<Short4> rhs)
3466         {
3467                 return lhs = lhs ^ rhs;
3468         }
3469
3470         RValue<Short4> operator<<=(Short4 &lhs, unsigned char rhs)
3471         {
3472                 return lhs = lhs << rhs;
3473         }
3474
3475         RValue<Short4> operator>>=(Short4 &lhs, unsigned char rhs)
3476         {
3477                 return lhs = lhs >> rhs;
3478         }
3479
3480 //      RValue<Short4> operator+(RValue<Short4> val)
3481 //      {
3482 //              return val;
3483 //      }
3484
3485         RValue<Short4> operator-(RValue<Short4> val)
3486         {
3487                 return RValue<Short4>(Nucleus::createNeg(val.value));
3488         }
3489
3490         RValue<Short4> operator~(RValue<Short4> val)
3491         {
3492                 return RValue<Short4>(Nucleus::createNot(val.value));
3493         }
3494
3495         RValue<Short4> RoundShort4(RValue<Float4> cast)
3496         {
3497                 RValue<Int4> int4 = RoundInt(cast);
3498                 return As<Short4>(Pack(int4, int4));
3499         }
3500
3501         RValue<Short4> Max(RValue<Short4> x, RValue<Short4> y)
3502         {
3503                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v8i1);
3504                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Sle, condition, x.value, y.value);
3505                 ::basicBlock->appendInst(cmp);
3506
3507                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3508                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
3509                 ::basicBlock->appendInst(select);
3510
3511                 return RValue<Short4>(V(result));
3512         }
3513
3514         RValue<Short4> Min(RValue<Short4> x, RValue<Short4> y)
3515         {
3516                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v8i1);
3517                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Sgt, condition, x.value, y.value);
3518                 ::basicBlock->appendInst(cmp);
3519
3520                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3521                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
3522                 ::basicBlock->appendInst(select);
3523
3524                 return RValue<Short4>(V(result));
3525         }
3526
3527         RValue<Short> Saturate(RValue<Int> x)
3528         {
3529                 return Short(IfThenElse(x > 0x7FFF, Int(0x7FFF), IfThenElse(x < -0x8000, Int(0x8000), x)));
3530         }
3531
3532         RValue<Short4> AddSat(RValue<Short4> x, RValue<Short4> y)
3533         {
3534                 if(emulateIntrinsics)
3535                 {
3536                         Short4 result;
3537                         result = Insert(result, Saturate(Int(Extract(x, 0)) + Int(Extract(y, 0))), 0);
3538                         result = Insert(result, Saturate(Int(Extract(x, 1)) + Int(Extract(y, 1))), 1);
3539                         result = Insert(result, Saturate(Int(Extract(x, 2)) + Int(Extract(y, 2))), 2);
3540                         result = Insert(result, Saturate(Int(Extract(x, 3)) + Int(Extract(y, 3))), 3);
3541
3542                         return result;
3543                 }
3544                 else
3545                 {
3546                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3547                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::AddSaturateSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3548                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
3549                         auto paddsw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3550                         paddsw->addArg(x.value);
3551                         paddsw->addArg(y.value);
3552                         ::basicBlock->appendInst(paddsw);
3553
3554                         return RValue<Short4>(V(result));
3555                 }
3556         }
3557
3558         RValue<Short4> SubSat(RValue<Short4> x, RValue<Short4> y)
3559         {
3560                 if(emulateIntrinsics)
3561                 {
3562                         Short4 result;
3563                         result = Insert(result, Saturate(Int(Extract(x, 0)) - Int(Extract(y, 0))), 0);
3564                         result = Insert(result, Saturate(Int(Extract(x, 1)) - Int(Extract(y, 1))), 1);
3565                         result = Insert(result, Saturate(Int(Extract(x, 2)) - Int(Extract(y, 2))), 2);
3566                         result = Insert(result, Saturate(Int(Extract(x, 3)) - Int(Extract(y, 3))), 3);
3567
3568                         return result;
3569                 }
3570                 else
3571                 {
3572                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3573                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SubtractSaturateSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3574                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
3575                         auto psubsw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3576                         psubsw->addArg(x.value);
3577                         psubsw->addArg(y.value);
3578                         ::basicBlock->appendInst(psubsw);
3579
3580                         return RValue<Short4>(V(result));
3581                 }
3582         }
3583
3584         RValue<Short4> MulHigh(RValue<Short4> x, RValue<Short4> y)
3585         {
3586                 if(emulateIntrinsics)
3587                 {
3588                         Short4 result;
3589                         result = Insert(result, Short((Int(Extract(x, 0)) * Int(Extract(y, 0))) >> 16), 0);
3590                         result = Insert(result, Short((Int(Extract(x, 1)) * Int(Extract(y, 1))) >> 16), 1);
3591                         result = Insert(result, Short((Int(Extract(x, 2)) * Int(Extract(y, 2))) >> 16), 2);
3592                         result = Insert(result, Short((Int(Extract(x, 3)) * Int(Extract(y, 3))) >> 16), 3);
3593
3594                         return result;
3595                 }
3596                 else
3597                 {
3598                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3599                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::MultiplyHighSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3600                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
3601                         auto pmulhw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3602                         pmulhw->addArg(x.value);
3603                         pmulhw->addArg(y.value);
3604                         ::basicBlock->appendInst(pmulhw);
3605
3606                         return RValue<Short4>(V(result));
3607                 }
3608         }
3609
3610         RValue<Int2> MulAdd(RValue<Short4> x, RValue<Short4> y)
3611         {
3612                 if(emulateIntrinsics)
3613                 {
3614                         Int2 result;
3615                         result = Insert(result, Int(Extract(x, 0)) * Int(Extract(y, 0)) + Int(Extract(x, 1)) * Int(Extract(y, 1)), 0);
3616                         result = Insert(result, Int(Extract(x, 2)) * Int(Extract(y, 2)) + Int(Extract(x, 3)) * Int(Extract(y, 3)), 1);
3617
3618                         return result;
3619                 }
3620                 else
3621                 {
3622                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3623                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::MultiplyAddPairs, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3624                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
3625                         auto pmaddwd = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3626                         pmaddwd->addArg(x.value);
3627                         pmaddwd->addArg(y.value);
3628                         ::basicBlock->appendInst(pmaddwd);
3629
3630                         return As<Int2>(V(result));
3631                 }
3632         }
3633
3634         RValue<SByte8> Pack(RValue<Short4> x, RValue<Short4> y)
3635         {
3636                 if(emulateIntrinsics)
3637                 {
3638                         SByte8 result;
3639                         result = Insert(result, Saturate(Extract(x, 0)), 0);
3640                         result = Insert(result, Saturate(Extract(x, 1)), 1);
3641                         result = Insert(result, Saturate(Extract(x, 2)), 2);
3642                         result = Insert(result, Saturate(Extract(x, 3)), 3);
3643                         result = Insert(result, Saturate(Extract(y, 0)), 4);
3644                         result = Insert(result, Saturate(Extract(y, 1)), 5);
3645                         result = Insert(result, Saturate(Extract(y, 2)), 6);
3646                         result = Insert(result, Saturate(Extract(y, 3)), 7);
3647
3648                         return result;
3649                 }
3650                 else
3651                 {
3652                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
3653                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::VectorPackSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3654                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
3655                         auto pack = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3656                         pack->addArg(x.value);
3657                         pack->addArg(y.value);
3658                         ::basicBlock->appendInst(pack);
3659
3660                         return As<SByte8>(Swizzle(As<Int4>(V(result)), 0x88));
3661                 }
3662         }
3663
3664         RValue<Int2> UnpackLow(RValue<Short4> x, RValue<Short4> y)
3665         {
3666                 int shuffle[8] = {0, 8, 1, 9, 2, 10, 3, 11};   // Real type is v8i16
3667                 return As<Int2>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
3668         }
3669
3670         RValue<Int2> UnpackHigh(RValue<Short4> x, RValue<Short4> y)
3671         {
3672                 int shuffle[8] = {0, 8, 1, 9, 2, 10, 3, 11};   // Real type is v8i16
3673                 auto lowHigh = RValue<Short8>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
3674                 return As<Int2>(Swizzle(As<Int4>(lowHigh), 0xEE));
3675         }
3676
3677         RValue<Short4> Swizzle(RValue<Short4> x, unsigned char select)
3678         {
3679                 // Real type is v8i16
3680                 int shuffle[8] =
3681                 {
3682                         (select >> 0) & 0x03,
3683                         (select >> 2) & 0x03,
3684                         (select >> 4) & 0x03,
3685                         (select >> 6) & 0x03,
3686                         (select >> 0) & 0x03,
3687                         (select >> 2) & 0x03,
3688                         (select >> 4) & 0x03,
3689                         (select >> 6) & 0x03,
3690                 };
3691
3692                 return RValue<Short4>(Nucleus::createShuffleVector(x.value, x.value, shuffle));
3693         }
3694
3695         RValue<Short4> Insert(RValue<Short4> val, RValue<Short> element, int i)
3696         {
3697                 return RValue<Short4>(Nucleus::createInsertElement(val.value, element.value, i));
3698         }
3699
3700         RValue<Short> Extract(RValue<Short4> val, int i)
3701         {
3702                 return RValue<Short>(Nucleus::createExtractElement(val.value, Short::getType(), i));
3703         }
3704
3705         RValue<Short4> CmpGT(RValue<Short4> x, RValue<Short4> y)
3706         {
3707                 return RValue<Short4>(createIntCompare(Ice::InstIcmp::Sgt, x.value, y.value));
3708         }
3709
3710         RValue<Short4> CmpEQ(RValue<Short4> x, RValue<Short4> y)
3711         {
3712                 return RValue<Short4>(Nucleus::createICmpEQ(x.value, y.value));
3713         }
3714
3715         Type *Short4::getType()
3716         {
3717                 return T(Type_v4i16);
3718         }
3719
3720         UShort4::UShort4(RValue<Int4> cast)
3721         {
3722                 *this = Short4(cast);
3723         }
3724
3725         UShort4::UShort4(RValue<Float4> cast, bool saturate)
3726         {
3727                 if(saturate)
3728                 {
3729                         if(CPUID::SSE4_1)
3730                         {
3731                                 Int4 int4(Min(cast, Float4(0xFFFF)));   // packusdw takes care of 0x0000 saturation
3732                                 *this = As<Short4>(Pack(As<UInt4>(int4), As<UInt4>(int4)));
3733                         }
3734                         else
3735                         {
3736                                 *this = Short4(Int4(Max(Min(cast, Float4(0xFFFF)), Float4(0x0000))));
3737                         }
3738                 }
3739                 else
3740                 {
3741                         *this = Short4(Int4(cast));
3742                 }
3743         }
3744
3745         UShort4::UShort4(unsigned short xyzw)
3746         {
3747                 int64_t constantVector[4] = {xyzw, xyzw, xyzw, xyzw};
3748                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
3749         }
3750
3751         UShort4::UShort4(unsigned short x, unsigned short y, unsigned short z, unsigned short w)
3752         {
3753                 int64_t constantVector[4] = {x, y, z, w};
3754                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
3755         }
3756
3757         UShort4::UShort4(RValue<UShort4> rhs)
3758         {
3759                 storeValue(rhs.value);
3760         }
3761
3762         UShort4::UShort4(const UShort4 &rhs)
3763         {
3764                 Value *value = rhs.loadValue();
3765                 storeValue(value);
3766         }
3767
3768         UShort4::UShort4(const Reference<UShort4> &rhs)
3769         {
3770                 Value *value = rhs.loadValue();
3771                 storeValue(value);
3772         }
3773
3774         UShort4::UShort4(RValue<Short4> rhs)
3775         {
3776                 storeValue(rhs.value);
3777         }
3778
3779         UShort4::UShort4(const Short4 &rhs)
3780         {
3781                 Value *value = rhs.loadValue();
3782                 storeValue(value);
3783         }
3784
3785         UShort4::UShort4(const Reference<Short4> &rhs)
3786         {
3787                 Value *value = rhs.loadValue();
3788                 storeValue(value);
3789         }
3790
3791         RValue<UShort4> UShort4::operator=(RValue<UShort4> rhs)
3792         {
3793                 storeValue(rhs.value);
3794
3795                 return rhs;
3796         }
3797
3798         RValue<UShort4> UShort4::operator=(const UShort4 &rhs)
3799         {
3800                 Value *value = rhs.loadValue();
3801                 storeValue(value);
3802
3803                 return RValue<UShort4>(value);
3804         }
3805
3806         RValue<UShort4> UShort4::operator=(const Reference<UShort4> &rhs)
3807         {
3808                 Value *value = rhs.loadValue();
3809                 storeValue(value);
3810
3811                 return RValue<UShort4>(value);
3812         }
3813
3814         RValue<UShort4> UShort4::operator=(RValue<Short4> rhs)
3815         {
3816                 storeValue(rhs.value);
3817
3818                 return RValue<UShort4>(rhs);
3819         }
3820
3821         RValue<UShort4> UShort4::operator=(const Short4 &rhs)
3822         {
3823                 Value *value = rhs.loadValue();
3824                 storeValue(value);
3825
3826                 return RValue<UShort4>(value);
3827         }
3828
3829         RValue<UShort4> UShort4::operator=(const Reference<Short4> &rhs)
3830         {
3831                 Value *value = rhs.loadValue();
3832                 storeValue(value);
3833
3834                 return RValue<UShort4>(value);
3835         }
3836
3837         RValue<UShort4> operator+(RValue<UShort4> lhs, RValue<UShort4> rhs)
3838         {
3839                 return RValue<UShort4>(Nucleus::createAdd(lhs.value, rhs.value));
3840         }
3841
3842         RValue<UShort4> operator-(RValue<UShort4> lhs, RValue<UShort4> rhs)
3843         {
3844                 return RValue<UShort4>(Nucleus::createSub(lhs.value, rhs.value));
3845         }
3846
3847         RValue<UShort4> operator*(RValue<UShort4> lhs, RValue<UShort4> rhs)
3848         {
3849                 return RValue<UShort4>(Nucleus::createMul(lhs.value, rhs.value));
3850         }
3851
3852         RValue<UShort4> operator&(RValue<UShort4> lhs, RValue<UShort4> rhs)
3853         {
3854                 return RValue<UShort4>(Nucleus::createAnd(lhs.value, rhs.value));
3855         }
3856
3857         RValue<UShort4> operator|(RValue<UShort4> lhs, RValue<UShort4> rhs)
3858         {
3859                 return RValue<UShort4>(Nucleus::createOr(lhs.value, rhs.value));
3860         }
3861
3862         RValue<UShort4> operator^(RValue<UShort4> lhs, RValue<UShort4> rhs)
3863         {
3864                 return RValue<UShort4>(Nucleus::createXor(lhs.value, rhs.value));
3865         }
3866
3867         RValue<UShort> Extract(RValue<UShort4> val, int i)
3868         {
3869                 return RValue<UShort>(Nucleus::createExtractElement(val.value, UShort::getType(), i));
3870         }
3871
3872         RValue<UShort4> Insert(RValue<UShort4> val, RValue<UShort> element, int i)
3873         {
3874                 return RValue<UShort4>(Nucleus::createInsertElement(val.value, element.value, i));
3875         }
3876
3877         RValue<UShort4> operator<<(RValue<UShort4> lhs, unsigned char rhs)
3878         {
3879                 if(emulateIntrinsics)
3880                 {
3881                         UShort4 result;
3882                         result = Insert(result, Extract(lhs, 0) << UShort(rhs), 0);
3883                         result = Insert(result, Extract(lhs, 1) << UShort(rhs), 1);
3884                         result = Insert(result, Extract(lhs, 2) << UShort(rhs), 2);
3885                         result = Insert(result, Extract(lhs, 3) << UShort(rhs), 3);
3886
3887                         return result;
3888                 }
3889                 else
3890                 {
3891                         return RValue<UShort4>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
3892                 }
3893         }
3894
3895         RValue<UShort4> operator>>(RValue<UShort4> lhs, unsigned char rhs)
3896         {
3897                 if(emulateIntrinsics)
3898                 {
3899                         UShort4 result;
3900                         result = Insert(result, Extract(lhs, 0) >> UShort(rhs), 0);
3901                         result = Insert(result, Extract(lhs, 1) >> UShort(rhs), 1);
3902                         result = Insert(result, Extract(lhs, 2) >> UShort(rhs), 2);
3903                         result = Insert(result, Extract(lhs, 3) >> UShort(rhs), 3);
3904
3905                         return result;
3906                 }
3907                 else
3908                 {
3909                         return RValue<UShort4>(Nucleus::createLShr(lhs.value, V(::context->getConstantInt32(rhs))));
3910                 }
3911         }
3912
3913         RValue<UShort4> operator<<=(UShort4 &lhs, unsigned char rhs)
3914         {
3915                 return lhs = lhs << rhs;
3916         }
3917
3918         RValue<UShort4> operator>>=(UShort4 &lhs, unsigned char rhs)
3919         {
3920                 return lhs = lhs >> rhs;
3921         }
3922
3923         RValue<UShort4> operator~(RValue<UShort4> val)
3924         {
3925                 return RValue<UShort4>(Nucleus::createNot(val.value));
3926         }
3927
3928         RValue<UShort4> Max(RValue<UShort4> x, RValue<UShort4> y)
3929         {
3930                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v8i1);
3931                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Ule, condition, x.value, y.value);
3932                 ::basicBlock->appendInst(cmp);
3933
3934                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3935                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
3936                 ::basicBlock->appendInst(select);
3937
3938                 return RValue<UShort4>(V(result));
3939         }
3940
3941         RValue<UShort4> Min(RValue<UShort4> x, RValue<UShort4> y)
3942         {
3943                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v8i1);
3944                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Ugt, condition, x.value, y.value);
3945                 ::basicBlock->appendInst(cmp);
3946
3947                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3948                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
3949                 ::basicBlock->appendInst(select);
3950
3951                 return RValue<UShort4>(V(result));
3952         }
3953
3954         RValue<UShort> SaturateUShort(RValue<Int> x)
3955         {
3956                 return UShort(IfThenElse(x > 0xFFFF, Int(0xFFFF), IfThenElse(x < 0, Int(0), x)));
3957         }
3958
3959         RValue<UShort4> AddSat(RValue<UShort4> x, RValue<UShort4> y)
3960         {
3961                 if(emulateIntrinsics)
3962                 {
3963                         UShort4 result;
3964                         result = Insert(result, SaturateUShort(Int(Extract(x, 0)) + Int(Extract(y, 0))), 0);
3965                         result = Insert(result, SaturateUShort(Int(Extract(x, 1)) + Int(Extract(y, 1))), 1);
3966                         result = Insert(result, SaturateUShort(Int(Extract(x, 2)) + Int(Extract(y, 2))), 2);
3967                         result = Insert(result, SaturateUShort(Int(Extract(x, 3)) + Int(Extract(y, 3))), 3);
3968
3969                         return result;
3970                 }
3971                 else
3972                 {
3973                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3974                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::AddSaturateUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3975                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
3976                         auto paddusw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3977                         paddusw->addArg(x.value);
3978                         paddusw->addArg(y.value);
3979                         ::basicBlock->appendInst(paddusw);
3980
3981                         return RValue<UShort4>(V(result));
3982                 }
3983         }
3984
3985         RValue<UShort4> SubSat(RValue<UShort4> x, RValue<UShort4> y)
3986         {
3987                 if(emulateIntrinsics)
3988                 {
3989                         UShort4 result;
3990                         result = Insert(result, SaturateUShort(Int(Extract(x, 0)) - Int(Extract(y, 0))), 0);
3991                         result = Insert(result, SaturateUShort(Int(Extract(x, 1)) - Int(Extract(y, 1))), 1);
3992                         result = Insert(result, SaturateUShort(Int(Extract(x, 2)) - Int(Extract(y, 2))), 2);
3993                         result = Insert(result, SaturateUShort(Int(Extract(x, 3)) - Int(Extract(y, 3))), 3);
3994
3995                         return result;
3996                 }
3997                 else
3998                 {
3999                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
4000                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SubtractSaturateUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
4001                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
4002                         auto psubusw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
4003                         psubusw->addArg(x.value);
4004                         psubusw->addArg(y.value);
4005                         ::basicBlock->appendInst(psubusw);
4006
4007                         return RValue<UShort4>(V(result));
4008                 }
4009         }
4010
4011         RValue<UShort4> MulHigh(RValue<UShort4> x, RValue<UShort4> y)
4012         {
4013                 if(emulateIntrinsics)
4014                 {
4015                         UShort4 result;
4016                         result = Insert(result, UShort((UInt(Extract(x, 0)) * UInt(Extract(y, 0))) >> 16), 0);
4017                         result = Insert(result, UShort((UInt(Extract(x, 1)) * UInt(Extract(y, 1))) >> 16), 1);
4018                         result = Insert(result, UShort((UInt(Extract(x, 2)) * UInt(Extract(y, 2))) >> 16), 2);
4019                         result = Insert(result, UShort((UInt(Extract(x, 3)) * UInt(Extract(y, 3))) >> 16), 3);
4020
4021                         return result;
4022                 }
4023                 else
4024                 {
4025                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
4026                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::MultiplyHighUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
4027                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
4028                         auto pmulhuw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
4029                         pmulhuw->addArg(x.value);
4030                         pmulhuw->addArg(y.value);
4031                         ::basicBlock->appendInst(pmulhuw);
4032
4033                         return RValue<UShort4>(V(result));
4034                 }
4035         }
4036
4037         RValue<UShort4> Average(RValue<UShort4> x, RValue<UShort4> y)
4038         {
4039                 assert(false && "UNIMPLEMENTED"); return RValue<UShort4>(V(nullptr));
4040         }
4041
4042         RValue<Byte8> Pack(RValue<UShort4> x, RValue<UShort4> y)
4043         {
4044                 if(emulateIntrinsics)
4045                 {
4046                         Byte8 result;
4047                         result = Insert(result, Saturate(Extract(x, 0)), 0);
4048                         result = Insert(result, Saturate(Extract(x, 1)), 1);
4049                         result = Insert(result, Saturate(Extract(x, 2)), 2);
4050                         result = Insert(result, Saturate(Extract(x, 3)), 3);
4051                         result = Insert(result, Saturate(Extract(y, 0)), 4);
4052                         result = Insert(result, Saturate(Extract(y, 1)), 5);
4053                         result = Insert(result, Saturate(Extract(y, 2)), 6);
4054                         result = Insert(result, Saturate(Extract(y, 3)), 7);
4055
4056                         return result;
4057                 }
4058                 else
4059                 {
4060                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
4061                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::VectorPackUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
4062                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
4063                         auto pack = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
4064                         pack->addArg(x.value);
4065                         pack->addArg(y.value);
4066                         ::basicBlock->appendInst(pack);
4067
4068                         return As<Byte8>(Swizzle(As<Int4>(V(result)), 0x88));
4069                 }
4070         }
4071
4072         Type *UShort4::getType()
4073         {
4074                 return T(Type_v4i16);
4075         }
4076
4077         Short8::Short8(short c)
4078         {
4079                 int64_t constantVector[8] = {c, c, c, c, c, c, c, c};
4080                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
4081         }
4082
4083         Short8::Short8(short c0, short c1, short c2, short c3, short c4, short c5, short c6, short c7)
4084         {
4085                 int64_t constantVector[8] = {c0, c1, c2, c3, c4, c5, c6, c7};
4086                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
4087         }
4088
4089         Short8::Short8(RValue<Short8> rhs)
4090         {
4091                 storeValue(rhs.value);
4092         }
4093
4094         Short8::Short8(const Reference<Short8> &rhs)
4095         {
4096                 Value *value = rhs.loadValue();
4097                 storeValue(value);
4098         }
4099
4100         Short8::Short8(RValue<Short4> lo, RValue<Short4> hi)
4101         {
4102                 int shuffle[8] = {0, 1, 2, 3, 8, 9, 10, 11};   // Real type is v8i16
4103                 Value *packed = Nucleus::createShuffleVector(lo.value, hi.value, shuffle);
4104
4105                 storeValue(packed);
4106         }
4107
4108         RValue<Short8> operator+(RValue<Short8> lhs, RValue<Short8> rhs)
4109         {
4110                 return RValue<Short8>(Nucleus::createAdd(lhs.value, rhs.value));
4111         }
4112
4113         RValue<Short8> operator&(RValue<Short8> lhs, RValue<Short8> rhs)
4114         {
4115                 return RValue<Short8>(Nucleus::createAnd(lhs.value, rhs.value));
4116         }
4117
4118         RValue<Short> Extract(RValue<Short8> val, int i)
4119         {
4120                 return RValue<Short>(Nucleus::createExtractElement(val.value, Short::getType(), i));
4121         }
4122
4123         RValue<Short8> Insert(RValue<Short8> val, RValue<Short> element, int i)
4124         {
4125                 return RValue<Short8>(Nucleus::createInsertElement(val.value, element.value, i));
4126         }
4127
4128         RValue<Short8> operator<<(RValue<Short8> lhs, unsigned char rhs)
4129         {
4130                 if(emulateIntrinsics)
4131                 {
4132                         Short8 result;
4133                         result = Insert(result, Extract(lhs, 0) << Short(rhs), 0);
4134                         result = Insert(result, Extract(lhs, 1) << Short(rhs), 1);
4135                         result = Insert(result, Extract(lhs, 2) << Short(rhs), 2);
4136                         result = Insert(result, Extract(lhs, 3) << Short(rhs), 3);
4137                         result = Insert(result, Extract(lhs, 4) << Short(rhs), 4);
4138                         result = Insert(result, Extract(lhs, 5) << Short(rhs), 5);
4139                         result = Insert(result, Extract(lhs, 6) << Short(rhs), 6);
4140                         result = Insert(result, Extract(lhs, 7) << Short(rhs), 7);
4141
4142                         return result;
4143                 }
4144                 else
4145                 {
4146                         return RValue<Short8>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
4147                 }
4148         }
4149
4150         RValue<Short8> operator>>(RValue<Short8> lhs, unsigned char rhs)
4151         {
4152                 if(emulateIntrinsics)
4153                 {
4154                         Short8 result;
4155                         result = Insert(result, Extract(lhs, 0) >> Short(rhs), 0);
4156                         result = Insert(result, Extract(lhs, 1) >> Short(rhs), 1);
4157                         result = Insert(result, Extract(lhs, 2) >> Short(rhs), 2);
4158                         result = Insert(result, Extract(lhs, 3) >> Short(rhs), 3);
4159                         result = Insert(result, Extract(lhs, 4) >> Short(rhs), 4);
4160                         result = Insert(result, Extract(lhs, 5) >> Short(rhs), 5);
4161                         result = Insert(result, Extract(lhs, 6) >> Short(rhs), 6);
4162                         result = Insert(result, Extract(lhs, 7) >> Short(rhs), 7);
4163
4164                         return result;
4165                 }
4166                 else
4167                 {
4168                         return RValue<Short8>(Nucleus::createAShr(lhs.value, V(::context->getConstantInt32(rhs))));
4169                 }
4170         }
4171
4172         RValue<Int4> MulAdd(RValue<Short8> x, RValue<Short8> y)
4173         {
4174                 assert(false && "UNIMPLEMENTED"); return RValue<Int4>(V(nullptr));
4175         }
4176
4177         RValue<Int4> Abs(RValue<Int4> x)
4178         {
4179                 auto negative = x >> 31;
4180                 return (x ^ negative) - negative;
4181         }
4182
4183         RValue<Short8> MulHigh(RValue<Short8> x, RValue<Short8> y)
4184         {
4185                 assert(false && "UNIMPLEMENTED"); return RValue<Short8>(V(nullptr));
4186         }
4187
4188         Type *Short8::getType()
4189         {
4190                 return T(Ice::IceType_v8i16);
4191         }
4192
4193         UShort8::UShort8(unsigned short c)
4194         {
4195                 int64_t constantVector[8] = {c, c, c, c, c, c, c, c};
4196                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
4197         }
4198
4199         UShort8::UShort8(unsigned short c0, unsigned short c1, unsigned short c2, unsigned short c3, unsigned short c4, unsigned short c5, unsigned short c6, unsigned short c7)
4200         {
4201                 int64_t constantVector[8] = {c0, c1, c2, c3, c4, c5, c6, c7};
4202                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
4203         }
4204
4205         UShort8::UShort8(RValue<UShort8> rhs)
4206         {
4207                 storeValue(rhs.value);
4208         }
4209
4210         UShort8::UShort8(const Reference<UShort8> &rhs)
4211         {
4212                 Value *value = rhs.loadValue();
4213                 storeValue(value);
4214         }
4215
4216         UShort8::UShort8(RValue<UShort4> lo, RValue<UShort4> hi)
4217         {
4218                 int shuffle[8] = {0, 1, 2, 3, 8, 9, 10, 11};   // Real type is v8i16
4219                 Value *packed = Nucleus::createShuffleVector(lo.value, hi.value, shuffle);
4220
4221                 storeValue(packed);
4222         }
4223
4224         RValue<UShort8> UShort8::operator=(RValue<UShort8> rhs)
4225         {
4226                 storeValue(rhs.value);
4227
4228                 return rhs;
4229         }
4230
4231         RValue<UShort8> UShort8::operator=(const UShort8 &rhs)
4232         {
4233                 Value *value = rhs.loadValue();
4234                 storeValue(value);
4235
4236                 return RValue<UShort8>(value);
4237         }
4238
4239         RValue<UShort8> UShort8::operator=(const Reference<UShort8> &rhs)
4240         {
4241                 Value *value = rhs.loadValue();
4242                 storeValue(value);
4243
4244                 return RValue<UShort8>(value);
4245         }
4246
4247         RValue<UShort8> operator&(RValue<UShort8> lhs, RValue<UShort8> rhs)
4248         {
4249                 return RValue<UShort8>(Nucleus::createAnd(lhs.value, rhs.value));
4250         }
4251
4252         RValue<UShort> Extract(RValue<UShort8> val, int i)
4253         {
4254                 return RValue<UShort>(Nucleus::createExtractElement(val.value, UShort::getType(), i));
4255         }
4256
4257         RValue<UShort8> Insert(RValue<UShort8> val, RValue<UShort> element, int i)
4258         {
4259                 return RValue<UShort8>(Nucleus::createInsertElement(val.value, element.value, i));
4260         }
4261
4262         RValue<UShort8> operator<<(RValue<UShort8> lhs, unsigned char rhs)
4263         {
4264                 if(emulateIntrinsics)
4265                 {
4266                         UShort8 result;
4267                         result = Insert(result, Extract(lhs, 0) << UShort(rhs), 0);
4268                         result = Insert(result, Extract(lhs, 1) << UShort(rhs), 1);
4269                         result = Insert(result, Extract(lhs, 2) << UShort(rhs), 2);
4270                         result = Insert(result, Extract(lhs, 3) << UShort(rhs), 3);
4271                         result = Insert(result, Extract(lhs, 4) << UShort(rhs), 4);
4272                         result = Insert(result, Extract(lhs, 5) << UShort(rhs), 5);
4273                         result = Insert(result, Extract(lhs, 6) << UShort(rhs), 6);
4274                         result = Insert(result, Extract(lhs, 7) << UShort(rhs), 7);
4275
4276                         return result;
4277                 }
4278                 else
4279                 {
4280                         return RValue<UShort8>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
4281                 }
4282         }
4283
4284         RValue<UShort8> operator>>(RValue<UShort8> lhs, unsigned char rhs)
4285         {
4286                 if(emulateIntrinsics)
4287                 {
4288                         UShort8 result;
4289                         result = Insert(result, Extract(lhs, 0) >> UShort(rhs), 0);
4290                         result = Insert(result, Extract(lhs, 1) >> UShort(rhs), 1);
4291                         result = Insert(result, Extract(lhs, 2) >> UShort(rhs), 2);
4292                         result = Insert(result, Extract(lhs, 3) >> UShort(rhs), 3);
4293                         result = Insert(result, Extract(lhs, 4) >> UShort(rhs), 4);
4294                         result = Insert(result, Extract(lhs, 5) >> UShort(rhs), 5);
4295                         result = Insert(result, Extract(lhs, 6) >> UShort(rhs), 6);
4296                         result = Insert(result, Extract(lhs, 7) >> UShort(rhs), 7);
4297
4298                         return result;
4299                 }
4300                 else
4301                 {
4302                         return RValue<UShort8>(Nucleus::createLShr(lhs.value, V(::context->getConstantInt32(rhs))));
4303                 }
4304         }
4305
4306         RValue<UShort8> operator+(RValue<UShort8> lhs, RValue<UShort8> rhs)
4307         {
4308                 return RValue<UShort8>(Nucleus::createAdd(lhs.value, rhs.value));
4309         }
4310
4311         RValue<UShort8> operator*(RValue<UShort8> lhs, RValue<UShort8> rhs)
4312         {
4313                 return RValue<UShort8>(Nucleus::createMul(lhs.value, rhs.value));
4314         }
4315
4316         RValue<UShort8> operator+=(UShort8 &lhs, RValue<UShort8> rhs)
4317         {
4318                 return lhs = lhs + rhs;
4319         }
4320
4321         RValue<UShort8> operator~(RValue<UShort8> val)
4322         {
4323                 return RValue<UShort8>(Nucleus::createNot(val.value));
4324         }
4325
4326         RValue<UShort8> Swizzle(RValue<UShort8> x, char select0, char select1, char select2, char select3, char select4, char select5, char select6, char select7)
4327         {
4328                 assert(false && "UNIMPLEMENTED"); return RValue<UShort8>(V(nullptr));
4329         }
4330
4331         RValue<UShort8> MulHigh(RValue<UShort8> x, RValue<UShort8> y)
4332         {
4333                 assert(false && "UNIMPLEMENTED"); return RValue<UShort8>(V(nullptr));
4334         }
4335
4336         // FIXME: Implement as Shuffle(x, y, Select(i0, ..., i16)) and Shuffle(x, y, SELECT_PACK_REPEAT(element))
4337 //      RValue<UShort8> PackRepeat(RValue<Byte16> x, RValue<Byte16> y, int element)
4338 //      {
4339 //              assert(false && "UNIMPLEMENTED"); return RValue<UShort8>(V(nullptr));
4340 //      }
4341
4342         Type *UShort8::getType()
4343         {
4344                 return T(Ice::IceType_v8i16);
4345         }
4346
4347         Int::Int(Argument<Int> argument)
4348         {
4349                 storeValue(argument.value);
4350         }
4351
4352         Int::Int(RValue<Byte> cast)
4353         {
4354                 Value *integer = Nucleus::createZExt(cast.value, Int::getType());
4355
4356                 storeValue(integer);
4357         }
4358
4359         Int::Int(RValue<SByte> cast)
4360         {
4361                 Value *integer = Nucleus::createSExt(cast.value, Int::getType());
4362
4363                 storeValue(integer);
4364         }
4365
4366         Int::Int(RValue<Short> cast)
4367         {
4368                 Value *integer = Nucleus::createSExt(cast.value, Int::getType());
4369
4370                 storeValue(integer);
4371         }
4372
4373         Int::Int(RValue<UShort> cast)
4374         {
4375                 Value *integer = Nucleus::createZExt(cast.value, Int::getType());
4376
4377                 storeValue(integer);
4378         }
4379
4380         Int::Int(RValue<Int2> cast)
4381         {
4382                 *this = Extract(cast, 0);
4383         }
4384
4385         Int::Int(RValue<Long> cast)
4386         {
4387                 Value *integer = Nucleus::createTrunc(cast.value, Int::getType());
4388
4389                 storeValue(integer);
4390         }
4391
4392         Int::Int(RValue<Float> cast)
4393         {
4394                 Value *integer = Nucleus::createFPToSI(cast.value, Int::getType());
4395
4396                 storeValue(integer);
4397         }
4398
4399         Int::Int(int x)
4400         {
4401                 storeValue(Nucleus::createConstantInt(x));
4402         }
4403
4404         Int::Int(RValue<Int> rhs)
4405         {
4406                 storeValue(rhs.value);
4407         }
4408
4409         Int::Int(RValue<UInt> rhs)
4410         {
4411                 storeValue(rhs.value);
4412         }
4413
4414         Int::Int(const Int &rhs)
4415         {
4416                 Value *value = rhs.loadValue();
4417                 storeValue(value);
4418         }
4419
4420         Int::Int(const Reference<Int> &rhs)
4421         {
4422                 Value *value = rhs.loadValue();
4423                 storeValue(value);
4424         }
4425
4426         Int::Int(const UInt &rhs)
4427         {
4428                 Value *value = rhs.loadValue();
4429                 storeValue(value);
4430         }
4431
4432         Int::Int(const Reference<UInt> &rhs)
4433         {
4434                 Value *value = rhs.loadValue();
4435                 storeValue(value);
4436         }
4437
4438         RValue<Int> Int::operator=(int rhs)
4439         {
4440                 return RValue<Int>(storeValue(Nucleus::createConstantInt(rhs)));
4441         }
4442
4443         RValue<Int> Int::operator=(RValue<Int> rhs)
4444         {
4445                 storeValue(rhs.value);
4446
4447                 return rhs;
4448         }
4449
4450         RValue<Int> Int::operator=(RValue<UInt> rhs)
4451         {
4452                 storeValue(rhs.value);
4453
4454                 return RValue<Int>(rhs);
4455         }
4456
4457         RValue<Int> Int::operator=(const Int &rhs)
4458         {
4459                 Value *value = rhs.loadValue();
4460                 storeValue(value);
4461
4462                 return RValue<Int>(value);
4463         }
4464
4465         RValue<Int> Int::operator=(const Reference<Int> &rhs)
4466         {
4467                 Value *value = rhs.loadValue();
4468                 storeValue(value);
4469
4470                 return RValue<Int>(value);
4471         }
4472
4473         RValue<Int> Int::operator=(const UInt &rhs)
4474         {
4475                 Value *value = rhs.loadValue();
4476                 storeValue(value);
4477
4478                 return RValue<Int>(value);
4479         }
4480
4481         RValue<Int> Int::operator=(const Reference<UInt> &rhs)
4482         {
4483                 Value *value = rhs.loadValue();
4484                 storeValue(value);
4485
4486                 return RValue<Int>(value);
4487         }
4488
4489         RValue<Int> operator+(RValue<Int> lhs, RValue<Int> rhs)
4490         {
4491                 return RValue<Int>(Nucleus::createAdd(lhs.value, rhs.value));
4492         }
4493
4494         RValue<Int> operator-(RValue<Int> lhs, RValue<Int> rhs)
4495         {
4496                 return RValue<Int>(Nucleus::createSub(lhs.value, rhs.value));
4497         }
4498
4499         RValue<Int> operator*(RValue<Int> lhs, RValue<Int> rhs)
4500         {
4501                 return RValue<Int>(Nucleus::createMul(lhs.value, rhs.value));
4502         }
4503
4504         RValue<Int> operator/(RValue<Int> lhs, RValue<Int> rhs)
4505         {
4506                 return RValue<Int>(Nucleus::createSDiv(lhs.value, rhs.value));
4507         }
4508
4509         RValue<Int> operator%(RValue<Int> lhs, RValue<Int> rhs)
4510         {
4511                 return RValue<Int>(Nucleus::createSRem(lhs.value, rhs.value));
4512         }
4513
4514         RValue<Int> operator&(RValue<Int> lhs, RValue<Int> rhs)
4515         {
4516                 return RValue<Int>(Nucleus::createAnd(lhs.value, rhs.value));
4517         }
4518
4519         RValue<Int> operator|(RValue<Int> lhs, RValue<Int> rhs)
4520         {
4521                 return RValue<Int>(Nucleus::createOr(lhs.value, rhs.value));
4522         }
4523
4524         RValue<Int> operator^(RValue<Int> lhs, RValue<Int> rhs)
4525         {
4526                 return RValue<Int>(Nucleus::createXor(lhs.value, rhs.value));
4527         }
4528
4529         RValue<Int> operator<<(RValue<Int> lhs, RValue<Int> rhs)
4530         {
4531                 return RValue<Int>(Nucleus::createShl(lhs.value, rhs.value));
4532         }
4533
4534         RValue<Int> operator>>(RValue<Int> lhs, RValue<Int> rhs)
4535         {
4536                 return RValue<Int>(Nucleus::createAShr(lhs.value, rhs.value));
4537         }
4538
4539         RValue<Int> operator+=(Int &lhs, RValue<Int> rhs)
4540         {
4541                 return lhs = lhs + rhs;
4542         }
4543
4544         RValue<Int> operator-=(Int &lhs, RValue<Int> rhs)
4545         {
4546                 return lhs = lhs - rhs;
4547         }
4548
4549         RValue<Int> operator*=(Int &lhs, RValue<Int> rhs)
4550         {
4551                 return lhs = lhs * rhs;
4552         }
4553
4554         RValue<Int> operator/=(Int &lhs, RValue<Int> rhs)
4555         {
4556                 return lhs = lhs / rhs;
4557         }
4558
4559         RValue<Int> operator%=(Int &lhs, RValue<Int> rhs)
4560         {
4561                 return lhs = lhs % rhs;
4562         }
4563
4564         RValue<Int> operator&=(Int &lhs, RValue<Int> rhs)
4565         {
4566                 return lhs = lhs & rhs;
4567         }
4568
4569         RValue<Int> operator|=(Int &lhs, RValue<Int> rhs)
4570         {
4571                 return lhs = lhs | rhs;
4572         }
4573
4574         RValue<Int> operator^=(Int &lhs, RValue<Int> rhs)
4575         {
4576                 return lhs = lhs ^ rhs;
4577         }
4578
4579         RValue<Int> operator<<=(Int &lhs, RValue<Int> rhs)
4580         {
4581                 return lhs = lhs << rhs;
4582         }
4583
4584         RValue<Int> operator>>=(Int &lhs, RValue<Int> rhs)
4585         {
4586                 return lhs = lhs >> rhs;
4587         }
4588
4589         RValue<Int> operator+(RValue<Int> val)
4590         {
4591                 return val;
4592         }
4593
4594         RValue<Int> operator-(RValue<Int> val)
4595         {
4596                 return RValue<Int>(Nucleus::createNeg(val.value));
4597         }
4598
4599         RValue<Int> operator~(RValue<Int> val)
4600         {
4601                 return RValue<Int>(Nucleus::createNot(val.value));
4602         }
4603
4604         RValue<Int> operator++(Int &val, int)   // Post-increment
4605         {
4606                 RValue<Int> res = val;
4607                 val += 1;
4608                 return res;
4609         }
4610
4611         const Int &operator++(Int &val)   // Pre-increment
4612         {
4613                 val += 1;
4614                 return val;
4615         }
4616
4617         RValue<Int> operator--(Int &val, int)   // Post-decrement
4618         {
4619                 RValue<Int> res = val;
4620                 val -= 1;
4621                 return res;
4622         }
4623
4624         const Int &operator--(Int &val)   // Pre-decrement
4625         {
4626                 val -= 1;
4627                 return val;
4628         }
4629
4630         RValue<Bool> operator<(RValue<Int> lhs, RValue<Int> rhs)
4631         {
4632                 return RValue<Bool>(Nucleus::createICmpSLT(lhs.value, rhs.value));
4633         }
4634
4635         RValue<Bool> operator<=(RValue<Int> lhs, RValue<Int> rhs)
4636         {
4637                 return RValue<Bool>(Nucleus::createICmpSLE(lhs.value, rhs.value));
4638         }
4639
4640         RValue<Bool> operator>(RValue<Int> lhs, RValue<Int> rhs)
4641         {
4642                 return RValue<Bool>(Nucleus::createICmpSGT(lhs.value, rhs.value));
4643         }
4644
4645         RValue<Bool> operator>=(RValue<Int> lhs, RValue<Int> rhs)
4646         {
4647                 return RValue<Bool>(Nucleus::createICmpSGE(lhs.value, rhs.value));
4648         }
4649
4650         RValue<Bool> operator!=(RValue<Int> lhs, RValue<Int> rhs)
4651         {
4652                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
4653         }
4654
4655         RValue<Bool> operator==(RValue<Int> lhs, RValue<Int> rhs)
4656         {
4657                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
4658         }
4659
4660         RValue<Int> Max(RValue<Int> x, RValue<Int> y)
4661         {
4662                 return IfThenElse(x > y, x, y);
4663         }
4664
4665         RValue<Int> Min(RValue<Int> x, RValue<Int> y)
4666         {
4667                 return IfThenElse(x < y, x, y);
4668         }
4669
4670         RValue<Int> Clamp(RValue<Int> x, RValue<Int> min, RValue<Int> max)
4671         {
4672                 return Min(Max(x, min), max);
4673         }
4674
4675         RValue<Int> RoundInt(RValue<Float> cast)
4676         {
4677                 if(emulateIntrinsics)
4678                 {
4679                         // Push the fractional part off the mantissa. Accurate up to +/-2^22.
4680                         return Int((cast + Float(0x00C00000)) - Float(0x00C00000));
4681                 }
4682                 else
4683                 {
4684                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_i32);
4685                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Nearbyint, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
4686                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
4687                         auto nearbyint = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
4688                         nearbyint->addArg(cast.value);
4689                         ::basicBlock->appendInst(nearbyint);
4690
4691                         return RValue<Int>(V(result));
4692                 }
4693         }
4694
4695         Type *Int::getType()
4696         {
4697                 return T(Ice::IceType_i32);
4698         }
4699
4700         Long::Long(RValue<Int> cast)
4701         {
4702                 Value *integer = Nucleus::createSExt(cast.value, Long::getType());
4703
4704                 storeValue(integer);
4705         }
4706
4707         Long::Long(RValue<UInt> cast)
4708         {
4709                 Value *integer = Nucleus::createZExt(cast.value, Long::getType());
4710
4711                 storeValue(integer);
4712         }
4713
4714         Long::Long(RValue<Long> rhs)
4715         {
4716                 storeValue(rhs.value);
4717         }
4718
4719         RValue<Long> Long::operator=(int64_t rhs)
4720         {
4721                 return RValue<Long>(storeValue(Nucleus::createConstantLong(rhs)));
4722         }
4723
4724         RValue<Long> Long::operator=(RValue<Long> rhs)
4725         {
4726                 storeValue(rhs.value);
4727
4728                 return rhs;
4729         }
4730
4731         RValue<Long> Long::operator=(const Long &rhs)
4732         {
4733                 Value *value = rhs.loadValue();
4734                 storeValue(value);
4735
4736                 return RValue<Long>(value);
4737         }
4738
4739         RValue<Long> Long::operator=(const Reference<Long> &rhs)
4740         {
4741                 Value *value = rhs.loadValue();
4742                 storeValue(value);
4743
4744                 return RValue<Long>(value);
4745         }
4746
4747         RValue<Long> operator+(RValue<Long> lhs, RValue<Long> rhs)
4748         {
4749                 return RValue<Long>(Nucleus::createAdd(lhs.value, rhs.value));
4750         }
4751
4752         RValue<Long> operator-(RValue<Long> lhs, RValue<Long> rhs)
4753         {
4754                 return RValue<Long>(Nucleus::createSub(lhs.value, rhs.value));
4755         }
4756
4757         RValue<Long> operator+=(Long &lhs, RValue<Long> rhs)
4758         {
4759                 return lhs = lhs + rhs;
4760         }
4761
4762         RValue<Long> operator-=(Long &lhs, RValue<Long> rhs)
4763         {
4764                 return lhs = lhs - rhs;
4765         }
4766
4767         RValue<Long> AddAtomic(RValue<Pointer<Long> > x, RValue<Long> y)
4768         {
4769                 return RValue<Long>(Nucleus::createAtomicAdd(x.value, y.value));
4770         }
4771
4772         Type *Long::getType()
4773         {
4774                 return T(Ice::IceType_i64);
4775         }
4776
4777         UInt::UInt(Argument<UInt> argument)
4778         {
4779                 storeValue(argument.value);
4780         }
4781
4782         UInt::UInt(RValue<UShort> cast)
4783         {
4784                 Value *integer = Nucleus::createZExt(cast.value, UInt::getType());
4785
4786                 storeValue(integer);
4787         }
4788
4789         UInt::UInt(RValue<Long> cast)
4790         {
4791                 Value *integer = Nucleus::createTrunc(cast.value, UInt::getType());
4792
4793                 storeValue(integer);
4794         }
4795
4796         UInt::UInt(RValue<Float> cast)
4797         {
4798                 // Smallest positive value representable in UInt, but not in Int
4799                 const unsigned int ustart = 0x80000000u;
4800                 const float ustartf = float(ustart);
4801
4802                 // If the value is negative, store 0, otherwise store the result of the conversion
4803                 storeValue((~(As<Int>(cast) >> 31) &
4804                 // Check if the value can be represented as an Int
4805                         IfThenElse(cast >= ustartf,
4806                 // If the value is too large, subtract ustart and re-add it after conversion.
4807                                 As<Int>(As<UInt>(Int(cast - Float(ustartf))) + UInt(ustart)),
4808                 // Otherwise, just convert normally
4809                                 Int(cast))).value);
4810         }
4811
4812         UInt::UInt(int x)
4813         {
4814                 storeValue(Nucleus::createConstantInt(x));
4815         }
4816
4817         UInt::UInt(unsigned int x)
4818         {
4819                 storeValue(Nucleus::createConstantInt(x));
4820         }
4821
4822         UInt::UInt(RValue<UInt> rhs)
4823         {
4824                 storeValue(rhs.value);
4825         }
4826
4827         UInt::UInt(RValue<Int> rhs)
4828         {
4829                 storeValue(rhs.value);
4830         }
4831
4832         UInt::UInt(const UInt &rhs)
4833         {
4834                 Value *value = rhs.loadValue();
4835                 storeValue(value);
4836         }
4837
4838         UInt::UInt(const Reference<UInt> &rhs)
4839         {
4840                 Value *value = rhs.loadValue();
4841                 storeValue(value);
4842         }
4843
4844         UInt::UInt(const Int &rhs)
4845         {
4846                 Value *value = rhs.loadValue();
4847                 storeValue(value);
4848         }
4849
4850         UInt::UInt(const Reference<Int> &rhs)
4851         {
4852                 Value *value = rhs.loadValue();
4853                 storeValue(value);
4854         }
4855
4856         RValue<UInt> UInt::operator=(unsigned int rhs)
4857         {
4858                 return RValue<UInt>(storeValue(Nucleus::createConstantInt(rhs)));
4859         }
4860
4861         RValue<UInt> UInt::operator=(RValue<UInt> rhs)
4862         {
4863                 storeValue(rhs.value);
4864
4865                 return rhs;
4866         }
4867
4868         RValue<UInt> UInt::operator=(RValue<Int> rhs)
4869         {
4870                 storeValue(rhs.value);
4871
4872                 return RValue<UInt>(rhs);
4873         }
4874
4875         RValue<UInt> UInt::operator=(const UInt &rhs)
4876         {
4877                 Value *value = rhs.loadValue();
4878                 storeValue(value);
4879
4880                 return RValue<UInt>(value);
4881         }
4882
4883         RValue<UInt> UInt::operator=(const Reference<UInt> &rhs)
4884         {
4885                 Value *value = rhs.loadValue();
4886                 storeValue(value);
4887
4888                 return RValue<UInt>(value);
4889         }
4890
4891         RValue<UInt> UInt::operator=(const Int &rhs)
4892         {
4893                 Value *value = rhs.loadValue();
4894                 storeValue(value);
4895
4896                 return RValue<UInt>(value);
4897         }
4898
4899         RValue<UInt> UInt::operator=(const Reference<Int> &rhs)
4900         {
4901                 Value *value = rhs.loadValue();
4902                 storeValue(value);
4903
4904                 return RValue<UInt>(value);
4905         }
4906
4907         RValue<UInt> operator+(RValue<UInt> lhs, RValue<UInt> rhs)
4908         {
4909                 return RValue<UInt>(Nucleus::createAdd(lhs.value, rhs.value));
4910         }
4911
4912         RValue<UInt> operator-(RValue<UInt> lhs, RValue<UInt> rhs)
4913         {
4914                 return RValue<UInt>(Nucleus::createSub(lhs.value, rhs.value));
4915         }
4916
4917         RValue<UInt> operator*(RValue<UInt> lhs, RValue<UInt> rhs)
4918         {
4919                 return RValue<UInt>(Nucleus::createMul(lhs.value, rhs.value));
4920         }
4921
4922         RValue<UInt> operator/(RValue<UInt> lhs, RValue<UInt> rhs)
4923         {
4924                 return RValue<UInt>(Nucleus::createUDiv(lhs.value, rhs.value));
4925         }
4926
4927         RValue<UInt> operator%(RValue<UInt> lhs, RValue<UInt> rhs)
4928         {
4929                 return RValue<UInt>(Nucleus::createURem(lhs.value, rhs.value));
4930         }
4931
4932         RValue<UInt> operator&(RValue<UInt> lhs, RValue<UInt> rhs)
4933         {
4934                 return RValue<UInt>(Nucleus::createAnd(lhs.value, rhs.value));
4935         }
4936
4937         RValue<UInt> operator|(RValue<UInt> lhs, RValue<UInt> rhs)
4938         {
4939                 return RValue<UInt>(Nucleus::createOr(lhs.value, rhs.value));
4940         }
4941
4942         RValue<UInt> operator^(RValue<UInt> lhs, RValue<UInt> rhs)
4943         {
4944                 return RValue<UInt>(Nucleus::createXor(lhs.value, rhs.value));
4945         }
4946
4947         RValue<UInt> operator<<(RValue<UInt> lhs, RValue<UInt> rhs)
4948         {
4949                 return RValue<UInt>(Nucleus::createShl(lhs.value, rhs.value));
4950         }
4951
4952         RValue<UInt> operator>>(RValue<UInt> lhs, RValue<UInt> rhs)
4953         {
4954                 return RValue<UInt>(Nucleus::createLShr(lhs.value, rhs.value));
4955         }
4956
4957         RValue<UInt> operator+=(UInt &lhs, RValue<UInt> rhs)
4958         {
4959                 return lhs = lhs + rhs;
4960         }
4961
4962         RValue<UInt> operator-=(UInt &lhs, RValue<UInt> rhs)
4963         {
4964                 return lhs = lhs - rhs;
4965         }
4966
4967         RValue<UInt> operator*=(UInt &lhs, RValue<UInt> rhs)
4968         {
4969                 return lhs = lhs * rhs;
4970         }
4971
4972         RValue<UInt> operator/=(UInt &lhs, RValue<UInt> rhs)
4973         {
4974                 return lhs = lhs / rhs;
4975         }
4976
4977         RValue<UInt> operator%=(UInt &lhs, RValue<UInt> rhs)
4978         {
4979                 return lhs = lhs % rhs;
4980         }
4981
4982         RValue<UInt> operator&=(UInt &lhs, RValue<UInt> rhs)
4983         {
4984                 return lhs = lhs & rhs;
4985         }
4986
4987         RValue<UInt> operator|=(UInt &lhs, RValue<UInt> rhs)
4988         {
4989                 return lhs = lhs | rhs;
4990         }
4991
4992         RValue<UInt> operator^=(UInt &lhs, RValue<UInt> rhs)
4993         {
4994                 return lhs = lhs ^ rhs;
4995         }
4996
4997         RValue<UInt> operator<<=(UInt &lhs, RValue<UInt> rhs)
4998         {
4999                 return lhs = lhs << rhs;
5000         }
5001
5002         RValue<UInt> operator>>=(UInt &lhs, RValue<UInt> rhs)
5003         {
5004                 return lhs = lhs >> rhs;
5005         }
5006
5007         RValue<UInt> operator+(RValue<UInt> val)
5008         {
5009                 return val;
5010         }
5011
5012         RValue<UInt> operator-(RValue<UInt> val)
5013         {
5014                 return RValue<UInt>(Nucleus::createNeg(val.value));
5015         }
5016
5017         RValue<UInt> operator~(RValue<UInt> val)
5018         {
5019                 return RValue<UInt>(Nucleus::createNot(val.value));
5020         }
5021
5022         RValue<UInt> operator++(UInt &val, int)   // Post-increment
5023         {
5024                 RValue<UInt> res = val;
5025                 val += 1;
5026                 return res;
5027         }
5028
5029         const UInt &operator++(UInt &val)   // Pre-increment
5030         {
5031                 val += 1;
5032                 return val;
5033         }
5034
5035         RValue<UInt> operator--(UInt &val, int)   // Post-decrement
5036         {
5037                 RValue<UInt> res = val;
5038                 val -= 1;
5039                 return res;
5040         }
5041
5042         const UInt &operator--(UInt &val)   // Pre-decrement
5043         {
5044                 val -= 1;
5045                 return val;
5046         }
5047
5048         RValue<UInt> Max(RValue<UInt> x, RValue<UInt> y)
5049         {
5050                 return IfThenElse(x > y, x, y);
5051         }
5052
5053         RValue<UInt> Min(RValue<UInt> x, RValue<UInt> y)
5054         {
5055                 return IfThenElse(x < y, x, y);
5056         }
5057
5058         RValue<UInt> Clamp(RValue<UInt> x, RValue<UInt> min, RValue<UInt> max)
5059         {
5060                 return Min(Max(x, min), max);
5061         }
5062
5063         RValue<Bool> operator<(RValue<UInt> lhs, RValue<UInt> rhs)
5064         {
5065                 return RValue<Bool>(Nucleus::createICmpULT(lhs.value, rhs.value));
5066         }
5067
5068         RValue<Bool> operator<=(RValue<UInt> lhs, RValue<UInt> rhs)
5069         {
5070                 return RValue<Bool>(Nucleus::createICmpULE(lhs.value, rhs.value));
5071         }
5072
5073         RValue<Bool> operator>(RValue<UInt> lhs, RValue<UInt> rhs)
5074         {
5075                 return RValue<Bool>(Nucleus::createICmpUGT(lhs.value, rhs.value));
5076         }
5077
5078         RValue<Bool> operator>=(RValue<UInt> lhs, RValue<UInt> rhs)
5079         {
5080                 return RValue<Bool>(Nucleus::createICmpUGE(lhs.value, rhs.value));
5081         }
5082
5083         RValue<Bool> operator!=(RValue<UInt> lhs, RValue<UInt> rhs)
5084         {
5085                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
5086         }
5087
5088         RValue<Bool> operator==(RValue<UInt> lhs, RValue<UInt> rhs)
5089         {
5090                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
5091         }
5092
5093 //      RValue<UInt> RoundUInt(RValue<Float> cast)
5094 //      {
5095 //              assert(false && "UNIMPLEMENTED"); return RValue<UInt>(V(nullptr));
5096 //      }
5097
5098         Type *UInt::getType()
5099         {
5100                 return T(Ice::IceType_i32);
5101         }
5102
5103 //      Int2::Int2(RValue<Int> cast)
5104 //      {
5105 //              Value *extend = Nucleus::createZExt(cast.value, Long::getType());
5106 //              Value *vector = Nucleus::createBitCast(extend, Int2::getType());
5107 //
5108 //              Constant *shuffle[2];
5109 //              shuffle[0] = Nucleus::createConstantInt(0);
5110 //              shuffle[1] = Nucleus::createConstantInt(0);
5111 //
5112 //              Value *replicate = Nucleus::createShuffleVector(vector, UndefValue::get(Int2::getType()), Nucleus::createConstantVector(shuffle, 2));
5113 //
5114 //              storeValue(replicate);
5115 //      }
5116
5117         Int2::Int2(RValue<Int4> cast)
5118         {
5119                 storeValue(Nucleus::createBitCast(cast.value, getType()));
5120         }
5121
5122         Int2::Int2(int x, int y)
5123         {
5124                 int64_t constantVector[2] = {x, y};
5125                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
5126         }
5127
5128         Int2::Int2(RValue<Int2> rhs)
5129         {
5130                 storeValue(rhs.value);
5131         }
5132
5133         Int2::Int2(const Int2 &rhs)
5134         {
5135                 Value *value = rhs.loadValue();
5136                 storeValue(value);
5137         }
5138
5139         Int2::Int2(const Reference<Int2> &rhs)
5140         {
5141                 Value *value = rhs.loadValue();
5142                 storeValue(value);
5143         }
5144
5145         Int2::Int2(RValue<Int> lo, RValue<Int> hi)
5146         {
5147                 int shuffle[4] = {0, 4, 1, 5};
5148                 Value *packed = Nucleus::createShuffleVector(Int4(lo).loadValue(), Int4(hi).loadValue(), shuffle);
5149
5150                 storeValue(Nucleus::createBitCast(packed, Int2::getType()));
5151         }
5152
5153         RValue<Int2> Int2::operator=(RValue<Int2> rhs)
5154         {
5155                 storeValue(rhs.value);
5156
5157                 return rhs;
5158         }
5159
5160         RValue<Int2> Int2::operator=(const Int2 &rhs)
5161         {
5162                 Value *value = rhs.loadValue();
5163                 storeValue(value);
5164
5165                 return RValue<Int2>(value);
5166         }
5167
5168         RValue<Int2> Int2::operator=(const Reference<Int2> &rhs)
5169         {
5170                 Value *value = rhs.loadValue();
5171                 storeValue(value);
5172
5173                 return RValue<Int2>(value);
5174         }
5175
5176         RValue<Int2> operator+(RValue<Int2> lhs, RValue<Int2> rhs)
5177         {
5178                 return RValue<Int2>(Nucleus::createAdd(lhs.value, rhs.value));
5179         }
5180
5181         RValue<Int2> operator-(RValue<Int2> lhs, RValue<Int2> rhs)
5182         {
5183                 return RValue<Int2>(Nucleus::createSub(lhs.value, rhs.value));
5184         }
5185
5186 //      RValue<Int2> operator*(RValue<Int2> lhs, RValue<Int2> rhs)
5187 //      {
5188 //              return RValue<Int2>(Nucleus::createMul(lhs.value, rhs.value));
5189 //      }
5190
5191 //      RValue<Int2> operator/(RValue<Int2> lhs, RValue<Int2> rhs)
5192 //      {
5193 //              return RValue<Int2>(Nucleus::createSDiv(lhs.value, rhs.value));
5194 //      }
5195
5196 //      RValue<Int2> operator%(RValue<Int2> lhs, RValue<Int2> rhs)
5197 //      {
5198 //              return RValue<Int2>(Nucleus::createSRem(lhs.value, rhs.value));
5199 //      }
5200
5201         RValue<Int2> operator&(RValue<Int2> lhs, RValue<Int2> rhs)
5202         {
5203                 return RValue<Int2>(Nucleus::createAnd(lhs.value, rhs.value));
5204         }
5205
5206         RValue<Int2> operator|(RValue<Int2> lhs, RValue<Int2> rhs)
5207         {
5208                 return RValue<Int2>(Nucleus::createOr(lhs.value, rhs.value));
5209         }
5210
5211         RValue<Int2> operator^(RValue<Int2> lhs, RValue<Int2> rhs)
5212         {
5213                 return RValue<Int2>(Nucleus::createXor(lhs.value, rhs.value));
5214         }
5215
5216         RValue<Int2> operator<<(RValue<Int2> lhs, unsigned char rhs)
5217         {
5218                 if(emulateIntrinsics)
5219                 {
5220                         Int2 result;
5221                         result = Insert(result, Extract(lhs, 0) << Int(rhs), 0);
5222                         result = Insert(result, Extract(lhs, 1) << Int(rhs), 1);
5223
5224                         return result;
5225                 }
5226                 else
5227                 {
5228                         return RValue<Int2>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
5229                 }
5230         }
5231
5232         RValue<Int2> operator>>(RValue<Int2> lhs, unsigned char rhs)
5233         {
5234                 if(emulateIntrinsics)
5235                 {
5236                         Int2 result;
5237                         result = Insert(result, Extract(lhs, 0) >> Int(rhs), 0);
5238                         result = Insert(result, Extract(lhs, 1) >> Int(rhs), 1);
5239
5240                         return result;
5241                 }
5242                 else
5243                 {
5244                         return RValue<Int2>(Nucleus::createAShr(lhs.value, V(::context->getConstantInt32(rhs))));
5245                 }
5246         }
5247
5248         RValue<Int2> operator+=(Int2 &lhs, RValue<Int2> rhs)
5249         {
5250                 return lhs = lhs + rhs;
5251         }
5252
5253         RValue<Int2> operator-=(Int2 &lhs, RValue<Int2> rhs)
5254         {
5255                 return lhs = lhs - rhs;
5256         }
5257
5258 //      RValue<Int2> operator*=(Int2 &lhs, RValue<Int2> rhs)
5259 //      {
5260 //              return lhs = lhs * rhs;
5261 //      }
5262
5263 //      RValue<Int2> operator/=(Int2 &lhs, RValue<Int2> rhs)
5264 //      {
5265 //              return lhs = lhs / rhs;
5266 //      }
5267
5268 //      RValue<Int2> operator%=(Int2 &lhs, RValue<Int2> rhs)
5269 //      {
5270 //              return lhs = lhs % rhs;
5271 //      }
5272
5273         RValue<Int2> operator&=(Int2 &lhs, RValue<Int2> rhs)
5274         {
5275                 return lhs = lhs & rhs;
5276         }
5277
5278         RValue<Int2> operator|=(Int2 &lhs, RValue<Int2> rhs)
5279         {
5280                 return lhs = lhs | rhs;
5281         }
5282
5283         RValue<Int2> operator^=(Int2 &lhs, RValue<Int2> rhs)
5284         {
5285                 return lhs = lhs ^ rhs;
5286         }
5287
5288         RValue<Int2> operator<<=(Int2 &lhs, unsigned char rhs)
5289         {
5290                 return lhs = lhs << rhs;
5291         }
5292
5293         RValue<Int2> operator>>=(Int2 &lhs, unsigned char rhs)
5294         {
5295                 return lhs = lhs >> rhs;
5296         }
5297
5298 //      RValue<Int2> operator+(RValue<Int2> val)
5299 //      {
5300 //              return val;
5301 //      }
5302
5303 //      RValue<Int2> operator-(RValue<Int2> val)
5304 //      {
5305 //              return RValue<Int2>(Nucleus::createNeg(val.value));
5306 //      }
5307
5308         RValue<Int2> operator~(RValue<Int2> val)
5309         {
5310                 return RValue<Int2>(Nucleus::createNot(val.value));
5311         }
5312
5313         RValue<Short4> UnpackLow(RValue<Int2> x, RValue<Int2> y)
5314         {
5315                 int shuffle[4] = {0, 4, 1, 5};   // Real type is v4i32
5316                 return As<Short4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
5317         }
5318
5319         RValue<Short4> UnpackHigh(RValue<Int2> x, RValue<Int2> y)
5320         {
5321                 int shuffle[4] = {0, 4, 1, 5};   // Real type is v4i32
5322                 auto lowHigh = RValue<Int4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
5323                 return As<Short4>(Swizzle(lowHigh, 0xEE));
5324         }
5325
5326         RValue<Int> Extract(RValue<Int2> val, int i)
5327         {
5328                 return RValue<Int>(Nucleus::createExtractElement(val.value, Int::getType(), i));
5329         }
5330
5331         RValue<Int2> Insert(RValue<Int2> val, RValue<Int> element, int i)
5332         {
5333                 return RValue<Int2>(Nucleus::createInsertElement(val.value, element.value, i));
5334         }
5335
5336         Type *Int2::getType()
5337         {
5338                 return T(Type_v2i32);
5339         }
5340
5341         UInt2::UInt2(unsigned int x, unsigned int y)
5342         {
5343                 int64_t constantVector[2] = {x, y};
5344                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
5345         }
5346
5347         UInt2::UInt2(RValue<UInt2> rhs)
5348         {
5349                 storeValue(rhs.value);
5350         }
5351
5352         UInt2::UInt2(const UInt2 &rhs)
5353         {
5354                 Value *value = rhs.loadValue();
5355                 storeValue(value);
5356         }
5357
5358         UInt2::UInt2(const Reference<UInt2> &rhs)
5359         {
5360                 Value *value = rhs.loadValue();
5361                 storeValue(value);
5362         }
5363
5364         RValue<UInt2> UInt2::operator=(RValue<UInt2> rhs)
5365         {
5366                 storeValue(rhs.value);
5367
5368                 return rhs;
5369         }
5370
5371         RValue<UInt2> UInt2::operator=(const UInt2 &rhs)
5372         {
5373                 Value *value = rhs.loadValue();
5374                 storeValue(value);
5375
5376                 return RValue<UInt2>(value);
5377         }
5378
5379         RValue<UInt2> UInt2::operator=(const Reference<UInt2> &rhs)
5380         {
5381                 Value *value = rhs.loadValue();
5382                 storeValue(value);
5383
5384                 return RValue<UInt2>(value);
5385         }
5386
5387         RValue<UInt2> operator+(RValue<UInt2> lhs, RValue<UInt2> rhs)
5388         {
5389                 return RValue<UInt2>(Nucleus::createAdd(lhs.value, rhs.value));
5390         }
5391
5392         RValue<UInt2> operator-(RValue<UInt2> lhs, RValue<UInt2> rhs)
5393         {
5394                 return RValue<UInt2>(Nucleus::createSub(lhs.value, rhs.value));
5395         }
5396
5397 //      RValue<UInt2> operator*(RValue<UInt2> lhs, RValue<UInt2> rhs)
5398 //      {
5399 //              return RValue<UInt2>(Nucleus::createMul(lhs.value, rhs.value));
5400 //      }
5401
5402 //      RValue<UInt2> operator/(RValue<UInt2> lhs, RValue<UInt2> rhs)
5403 //      {
5404 //              return RValue<UInt2>(Nucleus::createUDiv(lhs.value, rhs.value));
5405 //      }
5406
5407 //      RValue<UInt2> operator%(RValue<UInt2> lhs, RValue<UInt2> rhs)
5408 //      {
5409 //              return RValue<UInt2>(Nucleus::createURem(lhs.value, rhs.value));
5410 //      }
5411
5412         RValue<UInt2> operator&(RValue<UInt2> lhs, RValue<UInt2> rhs)
5413         {
5414                 return RValue<UInt2>(Nucleus::createAnd(lhs.value, rhs.value));
5415         }
5416
5417         RValue<UInt2> operator|(RValue<UInt2> lhs, RValue<UInt2> rhs)
5418         {
5419                 return RValue<UInt2>(Nucleus::createOr(lhs.value, rhs.value));
5420         }
5421
5422         RValue<UInt2> operator^(RValue<UInt2> lhs, RValue<UInt2> rhs)
5423         {
5424                 return RValue<UInt2>(Nucleus::createXor(lhs.value, rhs.value));
5425         }
5426
5427         RValue<UInt> Extract(RValue<UInt2> val, int i)
5428         {
5429                 return RValue<UInt>(Nucleus::createExtractElement(val.value, UInt::getType(), i));
5430         }
5431
5432         RValue<UInt2> Insert(RValue<UInt2> val, RValue<UInt> element, int i)
5433         {
5434                 return RValue<UInt2>(Nucleus::createInsertElement(val.value, element.value, i));
5435         }
5436
5437         RValue<UInt2> operator<<(RValue<UInt2> lhs, unsigned char rhs)
5438         {
5439                 if(emulateIntrinsics)
5440                 {
5441                         UInt2 result;
5442                         result = Insert(result, Extract(lhs, 0) << UInt(rhs), 0);
5443                         result = Insert(result, Extract(lhs, 1) << UInt(rhs), 1);
5444
5445                         return result;
5446                 }
5447                 else
5448                 {
5449                         return RValue<UInt2>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
5450                 }
5451         }
5452
5453         RValue<UInt2> operator>>(RValue<UInt2> lhs, unsigned char rhs)
5454         {
5455                 if(emulateIntrinsics)
5456                 {
5457                         UInt2 result;
5458                         result = Insert(result, Extract(lhs, 0) >> UInt(rhs), 0);
5459                         result = Insert(result, Extract(lhs, 1) >> UInt(rhs), 1);
5460
5461                         return result;
5462                 }
5463                 else
5464                 {
5465                         return RValue<UInt2>(Nucleus::createLShr(lhs.value, V(::context->getConstantInt32(rhs))));
5466                 }
5467         }
5468
5469         RValue<UInt2> operator+=(UInt2 &lhs, RValue<UInt2> rhs)
5470         {
5471                 return lhs = lhs + rhs;
5472         }
5473
5474         RValue<UInt2> operator-=(UInt2 &lhs, RValue<UInt2> rhs)
5475         {
5476                 return lhs = lhs - rhs;
5477         }
5478
5479 //      RValue<UInt2> operator*=(UInt2 &lhs, RValue<UInt2> rhs)
5480 //      {
5481 //              return lhs = lhs * rhs;
5482 //      }
5483
5484 //      RValue<UInt2> operator/=(UInt2 &lhs, RValue<UInt2> rhs)
5485 //      {
5486 //              return lhs = lhs / rhs;
5487 //      }
5488
5489 //      RValue<UInt2> operator%=(UInt2 &lhs, RValue<UInt2> rhs)
5490 //      {
5491 //              return lhs = lhs % rhs;
5492 //      }
5493
5494         RValue<UInt2> operator&=(UInt2 &lhs, RValue<UInt2> rhs)
5495         {
5496                 return lhs = lhs & rhs;
5497         }
5498
5499         RValue<UInt2> operator|=(UInt2 &lhs, RValue<UInt2> rhs)
5500         {
5501                 return lhs = lhs | rhs;
5502         }
5503
5504         RValue<UInt2> operator^=(UInt2 &lhs, RValue<UInt2> rhs)
5505         {
5506                 return lhs = lhs ^ rhs;
5507         }
5508
5509         RValue<UInt2> operator<<=(UInt2 &lhs, unsigned char rhs)
5510         {
5511                 return lhs = lhs << rhs;
5512         }
5513
5514         RValue<UInt2> operator>>=(UInt2 &lhs, unsigned char rhs)
5515         {
5516                 return lhs = lhs >> rhs;
5517         }
5518
5519 //      RValue<UInt2> operator+(RValue<UInt2> val)
5520 //      {
5521 //              return val;
5522 //      }
5523
5524 //      RValue<UInt2> operator-(RValue<UInt2> val)
5525 //      {
5526 //              return RValue<UInt2>(Nucleus::createNeg(val.value));
5527 //      }
5528
5529         RValue<UInt2> operator~(RValue<UInt2> val)
5530         {
5531                 return RValue<UInt2>(Nucleus::createNot(val.value));
5532         }
5533
5534         Type *UInt2::getType()
5535         {
5536                 return T(Type_v2i32);
5537         }
5538
5539         Int4::Int4(RValue<Byte4> cast)
5540         {
5541                 Value *x = Nucleus::createBitCast(cast.value, Int::getType());
5542                 Value *a = Nucleus::createInsertElement(loadValue(), x, 0);
5543
5544                 Value *e;
5545                 int swizzle[16] = {0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23};
5546                 Value *b = Nucleus::createBitCast(a, Byte16::getType());
5547                 Value *c = Nucleus::createShuffleVector(b, V(Nucleus::createNullValue(Byte16::getType())), swizzle);
5548
5549                 int swizzle2[8] = {0, 8, 1, 9, 2, 10, 3, 11};
5550                 Value *d = Nucleus::createBitCast(c, Short8::getType());
5551                 e = Nucleus::createShuffleVector(d, V(Nucleus::createNullValue(Short8::getType())), swizzle2);
5552
5553                 Value *f = Nucleus::createBitCast(e, Int4::getType());
5554                 storeValue(f);
5555         }
5556
5557         Int4::Int4(RValue<SByte4> cast)
5558         {
5559                 Value *x = Nucleus::createBitCast(cast.value, Int::getType());
5560                 Value *a = Nucleus::createInsertElement(loadValue(), x, 0);
5561
5562                 int swizzle[16] = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7};
5563                 Value *b = Nucleus::createBitCast(a, Byte16::getType());
5564                 Value *c = Nucleus::createShuffleVector(b, b, swizzle);
5565
5566                 int swizzle2[8] = {0, 0, 1, 1, 2, 2, 3, 3};
5567                 Value *d = Nucleus::createBitCast(c, Short8::getType());
5568                 Value *e = Nucleus::createShuffleVector(d, d, swizzle2);
5569
5570                 *this = As<Int4>(e) >> 24;
5571         }
5572
5573         Int4::Int4(RValue<Float4> cast)
5574         {
5575                 Value *xyzw = Nucleus::createFPToSI(cast.value, Int4::getType());
5576
5577                 storeValue(xyzw);
5578         }
5579
5580         Int4::Int4(RValue<Short4> cast)
5581         {
5582                 int swizzle[8] = {0, 0, 1, 1, 2, 2, 3, 3};
5583                 Value *c = Nucleus::createShuffleVector(cast.value, cast.value, swizzle);
5584
5585                 *this = As<Int4>(c) >> 16;
5586         }
5587
5588         Int4::Int4(RValue<UShort4> cast)
5589         {
5590                 int swizzle[8] = {0, 8, 1, 9, 2, 10, 3, 11};
5591                 Value *c = Nucleus::createShuffleVector(cast.value, Short8(0, 0, 0, 0, 0, 0, 0, 0).loadValue(), swizzle);
5592                 Value *d = Nucleus::createBitCast(c, Int4::getType());
5593                 storeValue(d);
5594         }
5595
5596         Int4::Int4(int xyzw)
5597         {
5598                 constant(xyzw, xyzw, xyzw, xyzw);
5599         }
5600
5601         Int4::Int4(int x, int yzw)
5602         {
5603                 constant(x, yzw, yzw, yzw);
5604         }
5605
5606         Int4::Int4(int x, int y, int zw)
5607         {
5608                 constant(x, y, zw, zw);
5609         }
5610
5611         Int4::Int4(int x, int y, int z, int w)
5612         {
5613                 constant(x, y, z, w);
5614         }
5615
5616         void Int4::constant(int x, int y, int z, int w)
5617         {
5618                 int64_t constantVector[4] = {x, y, z, w};
5619                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
5620         }
5621
5622         Int4::Int4(RValue<Int4> rhs)
5623         {
5624                 storeValue(rhs.value);
5625         }
5626
5627         Int4::Int4(const Int4 &rhs)
5628         {
5629                 Value *value = rhs.loadValue();
5630                 storeValue(value);
5631         }
5632
5633         Int4::Int4(const Reference<Int4> &rhs)
5634         {
5635                 Value *value = rhs.loadValue();
5636                 storeValue(value);
5637         }
5638
5639         Int4::Int4(RValue<UInt4> rhs)
5640         {
5641                 storeValue(rhs.value);
5642         }
5643
5644         Int4::Int4(const UInt4 &rhs)
5645         {
5646                 Value *value = rhs.loadValue();
5647                 storeValue(value);
5648         }
5649
5650         Int4::Int4(const Reference<UInt4> &rhs)
5651         {
5652                 Value *value = rhs.loadValue();
5653                 storeValue(value);
5654         }
5655
5656         Int4::Int4(RValue<Int2> lo, RValue<Int2> hi)
5657         {
5658                 int shuffle[4] = {0, 1, 4, 5};   // Real type is v4i32
5659                 Value *packed = Nucleus::createShuffleVector(lo.value, hi.value, shuffle);
5660
5661                 storeValue(packed);
5662         }
5663
5664         Int4::Int4(RValue<Int> rhs)
5665         {
5666                 Value *vector = Nucleus::createBitCast(rhs.value, Int4::getType());
5667
5668                 int swizzle[4] = {0, 0, 0, 0};
5669                 Value *replicate = Nucleus::createShuffleVector(vector, vector, swizzle);
5670
5671                 storeValue(replicate);
5672         }
5673
5674         Int4::Int4(const Int &rhs)
5675         {
5676                 *this = RValue<Int>(rhs.loadValue());
5677         }
5678
5679         Int4::Int4(const Reference<Int> &rhs)
5680         {
5681                 *this = RValue<Int>(rhs.loadValue());
5682         }
5683
5684         RValue<Int4> Int4::operator=(RValue<Int4> rhs)
5685         {
5686                 storeValue(rhs.value);
5687
5688                 return rhs;
5689         }
5690
5691         RValue<Int4> Int4::operator=(const Int4 &rhs)
5692         {
5693                 Value *value = rhs.loadValue();
5694                 storeValue(value);
5695
5696                 return RValue<Int4>(value);
5697         }
5698
5699         RValue<Int4> Int4::operator=(const Reference<Int4> &rhs)
5700         {
5701                 Value *value = rhs.loadValue();
5702                 storeValue(value);
5703
5704                 return RValue<Int4>(value);
5705         }
5706
5707         RValue<Int4> operator+(RValue<Int4> lhs, RValue<Int4> rhs)
5708         {
5709                 return RValue<Int4>(Nucleus::createAdd(lhs.value, rhs.value));
5710         }
5711
5712         RValue<Int4> operator-(RValue<Int4> lhs, RValue<Int4> rhs)
5713         {
5714                 return RValue<Int4>(Nucleus::createSub(lhs.value, rhs.value));
5715         }
5716
5717         RValue<Int4> operator*(RValue<Int4> lhs, RValue<Int4> rhs)
5718         {
5719                 return RValue<Int4>(Nucleus::createMul(lhs.value, rhs.value));
5720         }
5721
5722         RValue<Int4> operator/(RValue<Int4> lhs, RValue<Int4> rhs)
5723         {
5724                 return RValue<Int4>(Nucleus::createSDiv(lhs.value, rhs.value));
5725         }
5726
5727         RValue<Int4> operator%(RValue<Int4> lhs, RValue<Int4> rhs)
5728         {
5729                 return RValue<Int4>(Nucleus::createSRem(lhs.value, rhs.value));
5730         }
5731
5732         RValue<Int4> operator&(RValue<Int4> lhs, RValue<Int4> rhs)
5733         {
5734                 return RValue<Int4>(Nucleus::createAnd(lhs.value, rhs.value));
5735         }
5736
5737         RValue<Int4> operator|(RValue<Int4> lhs, RValue<Int4> rhs)
5738         {
5739                 return RValue<Int4>(Nucleus::createOr(lhs.value, rhs.value));
5740         }
5741
5742         RValue<Int4> operator^(RValue<Int4> lhs, RValue<Int4> rhs)
5743         {
5744                 return RValue<Int4>(Nucleus::createXor(lhs.value, rhs.value));
5745         }
5746
5747         RValue<Int4> operator<<(RValue<Int4> lhs, unsigned char rhs)
5748         {
5749                 if(emulateIntrinsics)
5750                 {
5751                         Int4 result;
5752                         result = Insert(result, Extract(lhs, 0) << Int(rhs), 0);
5753                         result = Insert(result, Extract(lhs, 1) << Int(rhs), 1);
5754                         result = Insert(result, Extract(lhs, 2) << Int(rhs), 2);
5755                         result = Insert(result, Extract(lhs, 3) << Int(rhs), 3);
5756
5757                         return result;
5758                 }
5759                 else
5760                 {
5761                         return RValue<Int4>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
5762                 }
5763         }
5764
5765         RValue<Int4> operator>>(RValue<Int4> lhs, unsigned char rhs)
5766         {
5767                 if(emulateIntrinsics)
5768                 {
5769                         Int4 result;
5770                         result = Insert(result, Extract(lhs, 0) >> Int(rhs), 0);
5771                         result = Insert(result, Extract(lhs, 1) >> Int(rhs), 1);
5772                         result = Insert(result, Extract(lhs, 2) >> Int(rhs), 2);
5773                         result = Insert(result, Extract(lhs, 3) >> Int(rhs), 3);
5774
5775                         return result;
5776                 }
5777                 else
5778                 {
5779                         return RValue<Int4>(Nucleus::createAShr(lhs.value, V(::context->getConstantInt32(rhs))));
5780                 }
5781         }
5782
5783         RValue<Int4> operator<<(RValue<Int4> lhs, RValue<Int4> rhs)
5784         {
5785                 return RValue<Int4>(Nucleus::createShl(lhs.value, rhs.value));
5786         }
5787
5788         RValue<Int4> operator>>(RValue<Int4> lhs, RValue<Int4> rhs)
5789         {
5790                 return RValue<Int4>(Nucleus::createAShr(lhs.value, rhs.value));
5791         }
5792
5793         RValue<Int4> operator+=(Int4 &lhs, RValue<Int4> rhs)
5794         {
5795                 return lhs = lhs + rhs;
5796         }
5797
5798         RValue<Int4> operator-=(Int4 &lhs, RValue<Int4> rhs)
5799         {
5800                 return lhs = lhs - rhs;
5801         }
5802
5803         RValue<Int4> operator*=(Int4 &lhs, RValue<Int4> rhs)
5804         {
5805                 return lhs = lhs * rhs;
5806         }
5807
5808 //      RValue<Int4> operator/=(Int4 &lhs, RValue<Int4> rhs)
5809 //      {
5810 //              return lhs = lhs / rhs;
5811 //      }
5812
5813 //      RValue<Int4> operator%=(Int4 &lhs, RValue<Int4> rhs)
5814 //      {
5815 //              return lhs = lhs % rhs;
5816 //      }
5817
5818         RValue<Int4> operator&=(Int4 &lhs, RValue<Int4> rhs)
5819         {
5820                 return lhs = lhs & rhs;
5821         }
5822
5823         RValue<Int4> operator|=(Int4 &lhs, RValue<Int4> rhs)
5824         {
5825                 return lhs = lhs | rhs;
5826         }
5827
5828         RValue<Int4> operator^=(Int4 &lhs, RValue<Int4> rhs)
5829         {
5830                 return lhs = lhs ^ rhs;
5831         }
5832
5833         RValue<Int4> operator<<=(Int4 &lhs, unsigned char rhs)
5834         {
5835                 return lhs = lhs << rhs;
5836         }
5837
5838         RValue<Int4> operator>>=(Int4 &lhs, unsigned char rhs)
5839         {
5840                 return lhs = lhs >> rhs;
5841         }
5842
5843         RValue<Int4> operator+(RValue<Int4> val)
5844         {
5845                 return val;
5846         }
5847
5848         RValue<Int4> operator-(RValue<Int4> val)
5849         {
5850                 return RValue<Int4>(Nucleus::createNeg(val.value));
5851         }
5852
5853         RValue<Int4> operator~(RValue<Int4> val)
5854         {
5855                 return RValue<Int4>(Nucleus::createNot(val.value));
5856         }
5857
5858         RValue<Int4> CmpEQ(RValue<Int4> x, RValue<Int4> y)
5859         {
5860                 return RValue<Int4>(Nucleus::createICmpEQ(x.value, y.value));
5861         }
5862
5863         RValue<Int4> CmpLT(RValue<Int4> x, RValue<Int4> y)
5864         {
5865                 return RValue<Int4>(Nucleus::createICmpSLT(x.value, y.value));
5866         }
5867
5868         RValue<Int4> CmpLE(RValue<Int4> x, RValue<Int4> y)
5869         {
5870                 return RValue<Int4>(Nucleus::createICmpSLE(x.value, y.value));
5871         }
5872
5873         RValue<Int4> CmpNEQ(RValue<Int4> x, RValue<Int4> y)
5874         {
5875                 return RValue<Int4>(Nucleus::createICmpNE(x.value, y.value));
5876         }
5877
5878         RValue<Int4> CmpNLT(RValue<Int4> x, RValue<Int4> y)
5879         {
5880                 return RValue<Int4>(Nucleus::createICmpSGE(x.value, y.value));
5881         }
5882
5883         RValue<Int4> CmpNLE(RValue<Int4> x, RValue<Int4> y)
5884         {
5885                 return RValue<Int4>(Nucleus::createICmpSGT(x.value, y.value));
5886         }
5887
5888         RValue<Int4> Max(RValue<Int4> x, RValue<Int4> y)
5889         {
5890                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v4i1);
5891                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Sle, condition, x.value, y.value);
5892                 ::basicBlock->appendInst(cmp);
5893
5894                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4i32);
5895                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
5896                 ::basicBlock->appendInst(select);
5897
5898                 return RValue<Int4>(V(result));
5899         }
5900
5901         RValue<Int4> Min(RValue<Int4> x, RValue<Int4> y)
5902         {
5903                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v4i1);
5904                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Sgt, condition, x.value, y.value);
5905                 ::basicBlock->appendInst(cmp);
5906
5907                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4i32);
5908                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
5909                 ::basicBlock->appendInst(select);
5910
5911                 return RValue<Int4>(V(result));
5912         }
5913
5914         RValue<Int4> RoundInt(RValue<Float4> cast)
5915         {
5916                 if(emulateIntrinsics)
5917                 {
5918                         // Push the fractional part off the mantissa. Accurate up to +/-2^22.
5919                         return Int4((cast + Float4(0x00C00000)) - Float4(0x00C00000));
5920                 }
5921                 else
5922                 {
5923                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4i32);
5924                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Nearbyint, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
5925                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
5926                         auto nearbyint = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
5927                         nearbyint->addArg(cast.value);
5928                         ::basicBlock->appendInst(nearbyint);
5929
5930                         return RValue<Int4>(V(result));
5931                 }
5932         }
5933
5934         RValue<Short8> Pack(RValue<Int4> x, RValue<Int4> y)
5935         {
5936                 if(emulateIntrinsics)
5937                 {
5938                         Short8 result;
5939                         result = Insert(result, Saturate(Extract(x, 0)), 0);
5940                         result = Insert(result, Saturate(Extract(x, 1)), 1);
5941                         result = Insert(result, Saturate(Extract(x, 2)), 2);
5942                         result = Insert(result, Saturate(Extract(x, 3)), 3);
5943                         result = Insert(result, Saturate(Extract(y, 0)), 4);
5944                         result = Insert(result, Saturate(Extract(y, 1)), 5);
5945                         result = Insert(result, Saturate(Extract(y, 2)), 6);
5946                         result = Insert(result, Saturate(Extract(y, 3)), 7);
5947
5948                         return result;
5949                 }
5950                 else
5951                 {
5952                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
5953                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::VectorPackSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
5954                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
5955                         auto pack = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
5956                         pack->addArg(x.value);
5957                         pack->addArg(y.value);
5958                         ::basicBlock->appendInst(pack);
5959
5960                         return RValue<Short8>(V(result));
5961                 }
5962         }
5963
5964         RValue<Int> Extract(RValue<Int4> x, int i)
5965         {
5966                 return RValue<Int>(Nucleus::createExtractElement(x.value, Int::getType(), i));
5967         }
5968
5969         RValue<Int4> Insert(RValue<Int4> x, RValue<Int> element, int i)
5970         {
5971                 return RValue<Int4>(Nucleus::createInsertElement(x.value, element.value, i));
5972         }
5973
5974         RValue<Int> SignMask(RValue<Int4> x)
5975         {
5976                 if(emulateIntrinsics)
5977                 {
5978                         Int4 xx = (x >> 31) & Int4(0x00000001, 0x00000002, 0x00000004, 0x00000008);
5979                         return Extract(xx, 0) | Extract(xx, 1) | Extract(xx, 2) | Extract(xx, 3);
5980                 }
5981                 else
5982                 {
5983                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_i32);
5984                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SignMask, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
5985                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
5986                         auto movmsk = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
5987                         movmsk->addArg(x.value);
5988                         ::basicBlock->appendInst(movmsk);
5989
5990                         return RValue<Int>(V(result));
5991                 }
5992         }
5993
5994         RValue<Int4> Swizzle(RValue<Int4> x, unsigned char select)
5995         {
5996                 return RValue<Int4>(createSwizzle4(x.value, select));
5997         }
5998
5999         Type *Int4::getType()
6000         {
6001                 return T(Ice::IceType_v4i32);
6002         }
6003
6004         UInt4::UInt4(RValue<Float4> cast)
6005         {
6006                 // Smallest positive value representable in UInt, but not in Int
6007                 const unsigned int ustart = 0x80000000u;
6008                 const float ustartf = float(ustart);
6009
6010                 // Check if the value can be represented as an Int
6011                 Int4 uiValue = CmpNLT(cast, Float4(ustartf));
6012                 // If the value is too large, subtract ustart and re-add it after conversion.
6013                 uiValue = (uiValue & As<Int4>(As<UInt4>(Int4(cast - Float4(ustartf))) + UInt4(ustart))) |
6014                 // Otherwise, just convert normally
6015                           (~uiValue & Int4(cast));
6016                 // If the value is negative, store 0, otherwise store the result of the conversion
6017                 storeValue((~(As<Int4>(cast) >> 31) & uiValue).value);
6018         }
6019
6020         UInt4::UInt4(int xyzw)
6021         {
6022                 constant(xyzw, xyzw, xyzw, xyzw);
6023         }
6024
6025         UInt4::UInt4(int x, int yzw)
6026         {
6027                 constant(x, yzw, yzw, yzw);
6028         }
6029
6030         UInt4::UInt4(int x, int y, int zw)
6031         {
6032                 constant(x, y, zw, zw);
6033         }
6034
6035         UInt4::UInt4(int x, int y, int z, int w)
6036         {
6037                 constant(x, y, z, w);
6038         }
6039
6040         void UInt4::constant(int x, int y, int z, int w)
6041         {
6042                 int64_t constantVector[4] = {x, y, z, w};
6043                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
6044         }
6045
6046         UInt4::UInt4(RValue<UInt4> rhs)
6047         {
6048                 storeValue(rhs.value);
6049         }
6050
6051         UInt4::UInt4(const UInt4 &rhs)
6052         {
6053                 Value *value = rhs.loadValue();
6054                 storeValue(value);
6055         }
6056
6057         UInt4::UInt4(const Reference<UInt4> &rhs)
6058         {
6059                 Value *value = rhs.loadValue();
6060                 storeValue(value);
6061         }
6062
6063         UInt4::UInt4(RValue<Int4> rhs)
6064         {
6065                 storeValue(rhs.value);
6066         }
6067
6068         UInt4::UInt4(const Int4 &rhs)
6069         {
6070                 Value *value = rhs.loadValue();
6071                 storeValue(value);
6072         }
6073
6074         UInt4::UInt4(const Reference<Int4> &rhs)
6075         {
6076                 Value *value = rhs.loadValue();
6077                 storeValue(value);
6078         }
6079
6080         UInt4::UInt4(RValue<UInt2> lo, RValue<UInt2> hi)
6081         {
6082                 int shuffle[4] = {0, 1, 4, 5};   // Real type is v4i32
6083                 Value *packed = Nucleus::createShuffleVector(lo.value, hi.value, shuffle);
6084
6085                 storeValue(packed);
6086         }
6087
6088         RValue<UInt4> UInt4::operator=(RValue<UInt4> rhs)
6089         {
6090                 storeValue(rhs.value);
6091
6092                 return rhs;
6093         }
6094
6095         RValue<UInt4> UInt4::operator=(const UInt4 &rhs)
6096         {
6097                 Value *value = rhs.loadValue();
6098                 storeValue(value);
6099
6100                 return RValue<UInt4>(value);
6101         }
6102
6103         RValue<UInt4> UInt4::operator=(const Reference<UInt4> &rhs)
6104         {
6105                 Value *value = rhs.loadValue();
6106                 storeValue(value);
6107
6108                 return RValue<UInt4>(value);
6109         }
6110
6111         RValue<UInt4> operator+(RValue<UInt4> lhs, RValue<UInt4> rhs)
6112         {
6113                 return RValue<UInt4>(Nucleus::createAdd(lhs.value, rhs.value));
6114         }
6115
6116         RValue<UInt4> operator-(RValue<UInt4> lhs, RValue<UInt4> rhs)
6117         {
6118                 return RValue<UInt4>(Nucleus::createSub(lhs.value, rhs.value));
6119         }
6120
6121         RValue<UInt4> operator*(RValue<UInt4> lhs, RValue<UInt4> rhs)
6122         {
6123                 return RValue<UInt4>(Nucleus::createMul(lhs.value, rhs.value));
6124         }
6125
6126         RValue<UInt4> operator/(RValue<UInt4> lhs, RValue<UInt4> rhs)
6127         {
6128                 return RValue<UInt4>(Nucleus::createUDiv(lhs.value, rhs.value));
6129         }
6130
6131         RValue<UInt4> operator%(RValue<UInt4> lhs, RValue<UInt4> rhs)
6132         {
6133                 return RValue<UInt4>(Nucleus::createURem(lhs.value, rhs.value));
6134         }
6135
6136         RValue<UInt4> operator&(RValue<UInt4> lhs, RValue<UInt4> rhs)
6137         {
6138                 return RValue<UInt4>(Nucleus::createAnd(lhs.value, rhs.value));
6139         }
6140
6141         RValue<UInt4> operator|(RValue<UInt4> lhs, RValue<UInt4> rhs)
6142         {
6143                 return RValue<UInt4>(Nucleus::createOr(lhs.value, rhs.value));
6144         }
6145
6146         RValue<UInt4> operator^(RValue<UInt4> lhs, RValue<UInt4> rhs)
6147         {
6148                 return RValue<UInt4>(Nucleus::createXor(lhs.value, rhs.value));
6149         }
6150
6151         RValue<UInt> Extract(RValue<UInt4> x, int i)
6152         {
6153                 return RValue<UInt>(Nucleus::createExtractElement(x.value, UInt::getType(), i));
6154         }
6155
6156         RValue<UInt4> Insert(RValue<UInt4> x, RValue<UInt> element, int i)
6157         {
6158                 return RValue<UInt4>(Nucleus::createInsertElement(x.value, element.value, i));
6159         }
6160
6161         RValue<UInt4> operator<<(RValue<UInt4> lhs, unsigned char rhs)
6162         {
6163                 if(emulateIntrinsics)
6164                 {
6165                         UInt4 result;
6166                         result = Insert(result, Extract(lhs, 0) << UInt(rhs), 0);
6167                         result = Insert(result, Extract(lhs, 1) << UInt(rhs), 1);
6168                         result = Insert(result, Extract(lhs, 2) << UInt(rhs), 2);
6169                         result = Insert(result, Extract(lhs, 3) << UInt(rhs), 3);
6170
6171                         return result;
6172                 }
6173                 else
6174                 {
6175                         return RValue<UInt4>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
6176                 }
6177         }
6178
6179         RValue<UInt4> operator>>(RValue<UInt4> lhs, unsigned char rhs)
6180         {
6181                 if(emulateIntrinsics)
6182                 {
6183                         UInt4 result;
6184                         result = Insert(result, Extract(lhs, 0) >> UInt(rhs), 0);
6185                         result = Insert(result, Extract(lhs, 1) >> UInt(rhs), 1);
6186                         result = Insert(result, Extract(lhs, 2) >> UInt(rhs), 2);
6187                         result = Insert(result, Extract(lhs, 3) >> UInt(rhs), 3);
6188
6189                         return result;
6190                 }
6191                 else
6192                 {
6193                         return RValue<UInt4>(Nucleus::createLShr(lhs.value, V(::context->getConstantInt32(rhs))));
6194                 }
6195         }
6196
6197         RValue<UInt4> operator<<(RValue<UInt4> lhs, RValue<UInt4> rhs)
6198         {
6199                 return RValue<UInt4>(Nucleus::createShl(lhs.value, rhs.value));
6200         }
6201
6202         RValue<UInt4> operator>>(RValue<UInt4> lhs, RValue<UInt4> rhs)
6203         {
6204                 return RValue<UInt4>(Nucleus::createLShr(lhs.value, rhs.value));
6205         }
6206
6207         RValue<UInt4> operator+=(UInt4 &lhs, RValue<UInt4> rhs)
6208         {
6209                 return lhs = lhs + rhs;
6210         }
6211
6212         RValue<UInt4> operator-=(UInt4 &lhs, RValue<UInt4> rhs)
6213         {
6214                 return lhs = lhs - rhs;
6215         }
6216
6217         RValue<UInt4> operator*=(UInt4 &lhs, RValue<UInt4> rhs)
6218         {
6219                 return lhs = lhs * rhs;
6220         }
6221
6222 //      RValue<UInt4> operator/=(UInt4 &lhs, RValue<UInt4> rhs)
6223 //      {
6224 //              return lhs = lhs / rhs;
6225 //      }
6226
6227 //      RValue<UInt4> operator%=(UInt4 &lhs, RValue<UInt4> rhs)
6228 //      {
6229 //              return lhs = lhs % rhs;
6230 //      }
6231
6232         RValue<UInt4> operator&=(UInt4 &lhs, RValue<UInt4> rhs)
6233         {
6234                 return lhs = lhs & rhs;
6235         }
6236
6237         RValue<UInt4> operator|=(UInt4 &lhs, RValue<UInt4> rhs)
6238         {
6239                 return lhs = lhs | rhs;
6240         }
6241
6242         RValue<UInt4> operator^=(UInt4 &lhs, RValue<UInt4> rhs)
6243         {
6244                 return lhs = lhs ^ rhs;
6245         }
6246
6247         RValue<UInt4> operator<<=(UInt4 &lhs, unsigned char rhs)
6248         {
6249                 return lhs = lhs << rhs;
6250         }
6251
6252         RValue<UInt4> operator>>=(UInt4 &lhs, unsigned char rhs)
6253         {
6254                 return lhs = lhs >> rhs;
6255         }
6256
6257         RValue<UInt4> operator+(RValue<UInt4> val)
6258         {
6259                 return val;
6260         }
6261
6262         RValue<UInt4> operator-(RValue<UInt4> val)
6263         {
6264                 return RValue<UInt4>(Nucleus::createNeg(val.value));
6265         }
6266
6267         RValue<UInt4> operator~(RValue<UInt4> val)
6268         {
6269                 return RValue<UInt4>(Nucleus::createNot(val.value));
6270         }
6271
6272         RValue<UInt4> CmpEQ(RValue<UInt4> x, RValue<UInt4> y)
6273         {
6274                 return RValue<UInt4>(Nucleus::createICmpEQ(x.value, y.value));
6275         }
6276
6277         RValue<UInt4> CmpLT(RValue<UInt4> x, RValue<UInt4> y)
6278         {
6279                 return RValue<UInt4>(Nucleus::createICmpULT(x.value, y.value));
6280         }
6281
6282         RValue<UInt4> CmpLE(RValue<UInt4> x, RValue<UInt4> y)
6283         {
6284                 return RValue<UInt4>(Nucleus::createICmpULE(x.value, y.value));
6285         }
6286
6287         RValue<UInt4> CmpNEQ(RValue<UInt4> x, RValue<UInt4> y)
6288         {
6289                 return RValue<UInt4>(Nucleus::createICmpNE(x.value, y.value));
6290         }
6291
6292         RValue<UInt4> CmpNLT(RValue<UInt4> x, RValue<UInt4> y)
6293         {
6294                 return RValue<UInt4>(Nucleus::createICmpUGE(x.value, y.value));
6295         }
6296
6297         RValue<UInt4> CmpNLE(RValue<UInt4> x, RValue<UInt4> y)
6298         {
6299                 return RValue<UInt4>(Nucleus::createICmpUGT(x.value, y.value));
6300         }
6301
6302         RValue<UInt4> Max(RValue<UInt4> x, RValue<UInt4> y)
6303         {
6304                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v4i1);
6305                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Ule, condition, x.value, y.value);
6306                 ::basicBlock->appendInst(cmp);
6307
6308                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4i32);
6309                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
6310                 ::basicBlock->appendInst(select);
6311
6312                 return RValue<UInt4>(V(result));
6313         }
6314
6315         RValue<UInt4> Min(RValue<UInt4> x, RValue<UInt4> y)
6316         {
6317                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v4i1);
6318                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Ugt, condition, x.value, y.value);
6319                 ::basicBlock->appendInst(cmp);
6320
6321                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4i32);
6322                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
6323                 ::basicBlock->appendInst(select);
6324
6325                 return RValue<UInt4>(V(result));
6326         }
6327
6328         RValue<UShort8> Pack(RValue<UInt4> x, RValue<UInt4> y)
6329         {
6330                 if(CPUID::SSE4_1)
6331                 {
6332                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
6333                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::VectorPackUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
6334                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
6335                         auto pack = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
6336                         pack->addArg(x.value);
6337                         pack->addArg(y.value);
6338                         ::basicBlock->appendInst(pack);
6339
6340                         return RValue<UShort8>(V(result));
6341                 }
6342                 else
6343                 {
6344                         RValue<Int4> sx = As<Int4>(x);
6345                         RValue<Int4> bx = (sx & ~(sx >> 31)) - Int4(0x8000);
6346
6347                         RValue<Int4> sy = As<Int4>(y);
6348                         RValue<Int4> by = (sy & ~(sy >> 31)) - Int4(0x8000);
6349
6350                         return As<UShort8>(Pack(bx, by) + Short8(0x8000u));
6351                 }
6352         }
6353
6354         Type *UInt4::getType()
6355         {
6356                 return T(Ice::IceType_v4i32);
6357         }
6358
6359         Float::Float(RValue<Int> cast)
6360         {
6361                 Value *integer = Nucleus::createSIToFP(cast.value, Float::getType());
6362
6363                 storeValue(integer);
6364         }
6365
6366         Float::Float(RValue<UInt> cast)
6367         {
6368                 RValue<Float> result = Float(Int(cast & UInt(0x7FFFFFFF))) +
6369                                        As<Float>((As<Int>(cast) >> 31) & As<Int>(Float(0x80000000u)));
6370
6371                 storeValue(result.value);
6372         }
6373
6374         Float::Float(float x)
6375         {
6376                 storeValue(Nucleus::createConstantFloat(x));
6377         }
6378
6379         Float::Float(RValue<Float> rhs)
6380         {
6381                 storeValue(rhs.value);
6382         }
6383
6384         Float::Float(const Float &rhs)
6385         {
6386                 Value *value = rhs.loadValue();
6387                 storeValue(value);
6388         }
6389
6390         Float::Float(const Reference<Float> &rhs)
6391         {
6392                 Value *value = rhs.loadValue();
6393                 storeValue(value);
6394         }
6395
6396         RValue<Float> Float::operator=(RValue<Float> rhs)
6397         {
6398                 storeValue(rhs.value);
6399
6400                 return rhs;
6401         }
6402
6403         RValue<Float> Float::operator=(const Float &rhs)
6404         {
6405                 Value *value = rhs.loadValue();
6406                 storeValue(value);
6407
6408                 return RValue<Float>(value);
6409         }
6410
6411         RValue<Float> Float::operator=(const Reference<Float> &rhs)
6412         {
6413                 Value *value = rhs.loadValue();
6414                 storeValue(value);
6415
6416                 return RValue<Float>(value);
6417         }
6418
6419         RValue<Float> operator+(RValue<Float> lhs, RValue<Float> rhs)
6420         {
6421                 return RValue<Float>(Nucleus::createFAdd(lhs.value, rhs.value));
6422         }
6423
6424         RValue<Float> operator-(RValue<Float> lhs, RValue<Float> rhs)
6425         {
6426                 return RValue<Float>(Nucleus::createFSub(lhs.value, rhs.value));
6427         }
6428
6429         RValue<Float> operator*(RValue<Float> lhs, RValue<Float> rhs)
6430         {
6431                 return RValue<Float>(Nucleus::createFMul(lhs.value, rhs.value));
6432         }
6433
6434         RValue<Float> operator/(RValue<Float> lhs, RValue<Float> rhs)
6435         {
6436                 return RValue<Float>(Nucleus::createFDiv(lhs.value, rhs.value));
6437         }
6438
6439         RValue<Float> operator+=(Float &lhs, RValue<Float> rhs)
6440         {
6441                 return lhs = lhs + rhs;
6442         }
6443
6444         RValue<Float> operator-=(Float &lhs, RValue<Float> rhs)
6445         {
6446                 return lhs = lhs - rhs;
6447         }
6448
6449         RValue<Float> operator*=(Float &lhs, RValue<Float> rhs)
6450         {
6451                 return lhs = lhs * rhs;
6452         }
6453
6454         RValue<Float> operator/=(Float &lhs, RValue<Float> rhs)
6455         {
6456                 return lhs = lhs / rhs;
6457         }
6458
6459         RValue<Float> operator+(RValue<Float> val)
6460         {
6461                 return val;
6462         }
6463
6464         RValue<Float> operator-(RValue<Float> val)
6465         {
6466                 return RValue<Float>(Nucleus::createFNeg(val.value));
6467         }
6468
6469         RValue<Bool> operator<(RValue<Float> lhs, RValue<Float> rhs)
6470         {
6471                 return RValue<Bool>(Nucleus::createFCmpOLT(lhs.value, rhs.value));
6472         }
6473
6474         RValue<Bool> operator<=(RValue<Float> lhs, RValue<Float> rhs)
6475         {
6476                 return RValue<Bool>(Nucleus::createFCmpOLE(lhs.value, rhs.value));
6477         }
6478
6479         RValue<Bool> operator>(RValue<Float> lhs, RValue<Float> rhs)
6480         {
6481                 return RValue<Bool>(Nucleus::createFCmpOGT(lhs.value, rhs.value));
6482         }
6483
6484         RValue<Bool> operator>=(RValue<Float> lhs, RValue<Float> rhs)
6485         {
6486                 return RValue<Bool>(Nucleus::createFCmpOGE(lhs.value, rhs.value));
6487         }
6488
6489         RValue<Bool> operator!=(RValue<Float> lhs, RValue<Float> rhs)
6490         {
6491                 return RValue<Bool>(Nucleus::createFCmpONE(lhs.value, rhs.value));
6492         }
6493
6494         RValue<Bool> operator==(RValue<Float> lhs, RValue<Float> rhs)
6495         {
6496                 return RValue<Bool>(Nucleus::createFCmpOEQ(lhs.value, rhs.value));
6497         }
6498
6499         RValue<Float> Abs(RValue<Float> x)
6500         {
6501                 return IfThenElse(x > 0.0f, x, -x);
6502         }
6503
6504         RValue<Float> Max(RValue<Float> x, RValue<Float> y)
6505         {
6506                 return IfThenElse(x > y, x, y);
6507         }
6508
6509         RValue<Float> Min(RValue<Float> x, RValue<Float> y)
6510         {
6511                 return IfThenElse(x < y, x, y);
6512         }
6513
6514         RValue<Float> Rcp_pp(RValue<Float> x, bool exactAtPow2)
6515         {
6516                 return 1.0f / x;
6517         }
6518
6519         RValue<Float> RcpSqrt_pp(RValue<Float> x)
6520         {
6521                 return Rcp_pp(Sqrt(x));
6522         }
6523
6524         RValue<Float> Sqrt(RValue<Float> x)
6525         {
6526                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_f32);
6527                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Sqrt, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
6528                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
6529                 auto sqrt = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
6530                 sqrt->addArg(x.value);
6531                 ::basicBlock->appendInst(sqrt);
6532
6533                 return RValue<Float>(V(result));
6534         }
6535
6536         RValue<Float> Round(RValue<Float> x)
6537         {
6538                 return Float4(Round(Float4(x))).x;
6539         }
6540
6541         RValue<Float> Trunc(RValue<Float> x)
6542         {
6543                 return Float4(Trunc(Float4(x))).x;
6544         }
6545
6546         RValue<Float> Frac(RValue<Float> x)
6547         {
6548                 return Float4(Frac(Float4(x))).x;
6549         }
6550
6551         RValue<Float> Floor(RValue<Float> x)
6552         {
6553                 return Float4(Floor(Float4(x))).x;
6554         }
6555
6556         RValue<Float> Ceil(RValue<Float> x)
6557         {
6558                 return Float4(Ceil(Float4(x))).x;
6559         }
6560
6561         Type *Float::getType()
6562         {
6563                 return T(Ice::IceType_f32);
6564         }
6565
6566         Float2::Float2(RValue<Float4> cast)
6567         {
6568                 storeValue(Nucleus::createBitCast(cast.value, getType()));
6569         }
6570
6571         Type *Float2::getType()
6572         {
6573                 return T(Type_v2f32);
6574         }
6575
6576         Float4::Float4(RValue<Byte4> cast) : FloatXYZW(this)
6577         {
6578                 Value *a = Int4(cast).loadValue();
6579                 Value *xyzw = Nucleus::createSIToFP(a, Float4::getType());
6580
6581                 storeValue(xyzw);
6582         }
6583
6584         Float4::Float4(RValue<SByte4> cast) : FloatXYZW(this)
6585         {
6586                 Value *a = Int4(cast).loadValue();
6587                 Value *xyzw = Nucleus::createSIToFP(a, Float4::getType());
6588
6589                 storeValue(xyzw);
6590         }
6591
6592         Float4::Float4(RValue<Short4> cast) : FloatXYZW(this)
6593         {
6594                 Int4 c(cast);
6595                 storeValue(Nucleus::createSIToFP(RValue<Int4>(c).value, Float4::getType()));
6596         }
6597
6598         Float4::Float4(RValue<UShort4> cast) : FloatXYZW(this)
6599         {
6600                 Int4 c(cast);
6601                 storeValue(Nucleus::createSIToFP(RValue<Int4>(c).value, Float4::getType()));
6602         }
6603
6604         Float4::Float4(RValue<Int4> cast) : FloatXYZW(this)
6605         {
6606                 Value *xyzw = Nucleus::createSIToFP(cast.value, Float4::getType());
6607
6608                 storeValue(xyzw);
6609         }
6610
6611         Float4::Float4(RValue<UInt4> cast) : FloatXYZW(this)
6612         {
6613                 RValue<Float4> result = Float4(Int4(cast & UInt4(0x7FFFFFFF))) +
6614                                         As<Float4>((As<Int4>(cast) >> 31) & As<Int4>(Float4(0x80000000u)));
6615
6616                 storeValue(result.value);
6617         }
6618
6619         Float4::Float4() : FloatXYZW(this)
6620         {
6621         }
6622
6623         Float4::Float4(float xyzw) : FloatXYZW(this)
6624         {
6625                 constant(xyzw, xyzw, xyzw, xyzw);
6626         }
6627
6628         Float4::Float4(float x, float yzw) : FloatXYZW(this)
6629         {
6630                 constant(x, yzw, yzw, yzw);
6631         }
6632
6633         Float4::Float4(float x, float y, float zw) : FloatXYZW(this)
6634         {
6635                 constant(x, y, zw, zw);
6636         }
6637
6638         Float4::Float4(float x, float y, float z, float w) : FloatXYZW(this)
6639         {
6640                 constant(x, y, z, w);
6641         }
6642
6643         void Float4::constant(float x, float y, float z, float w)
6644         {
6645                 double constantVector[4] = {x, y, z, w};
6646                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
6647         }
6648
6649         Float4::Float4(RValue<Float4> rhs) : FloatXYZW(this)
6650         {
6651                 storeValue(rhs.value);
6652         }
6653
6654         Float4::Float4(const Float4 &rhs) : FloatXYZW(this)
6655         {
6656                 Value *value = rhs.loadValue();
6657                 storeValue(value);
6658         }
6659
6660         Float4::Float4(const Reference<Float4> &rhs) : FloatXYZW(this)
6661         {
6662                 Value *value = rhs.loadValue();
6663                 storeValue(value);
6664         }
6665
6666         Float4::Float4(RValue<Float> rhs) : FloatXYZW(this)
6667         {
6668                 Value *vector = Nucleus::createBitCast(rhs.value, Float4::getType());
6669
6670                 int swizzle[4] = {0, 0, 0, 0};
6671                 Value *replicate = Nucleus::createShuffleVector(vector, vector, swizzle);
6672
6673                 storeValue(replicate);
6674         }
6675
6676         Float4::Float4(const Float &rhs) : FloatXYZW(this)
6677         {
6678                 *this = RValue<Float>(rhs.loadValue());
6679         }
6680
6681         Float4::Float4(const Reference<Float> &rhs) : FloatXYZW(this)
6682         {
6683                 *this = RValue<Float>(rhs.loadValue());
6684         }
6685
6686         RValue<Float4> Float4::operator=(float x)
6687         {
6688                 return *this = Float4(x, x, x, x);
6689         }
6690
6691         RValue<Float4> Float4::operator=(RValue<Float4> rhs)
6692         {
6693                 storeValue(rhs.value);
6694
6695                 return rhs;
6696         }
6697
6698         RValue<Float4> Float4::operator=(const Float4 &rhs)
6699         {
6700                 Value *value = rhs.loadValue();
6701                 storeValue(value);
6702
6703                 return RValue<Float4>(value);
6704         }
6705
6706         RValue<Float4> Float4::operator=(const Reference<Float4> &rhs)
6707         {
6708                 Value *value = rhs.loadValue();
6709                 storeValue(value);
6710
6711                 return RValue<Float4>(value);
6712         }
6713
6714         RValue<Float4> Float4::operator=(RValue<Float> rhs)
6715         {
6716                 return *this = Float4(rhs);
6717         }
6718
6719         RValue<Float4> Float4::operator=(const Float &rhs)
6720         {
6721                 return *this = Float4(rhs);
6722         }
6723
6724         RValue<Float4> Float4::operator=(const Reference<Float> &rhs)
6725         {
6726                 return *this = Float4(rhs);
6727         }
6728
6729         RValue<Float4> operator+(RValue<Float4> lhs, RValue<Float4> rhs)
6730         {
6731                 return RValue<Float4>(Nucleus::createFAdd(lhs.value, rhs.value));
6732         }
6733
6734         RValue<Float4> operator-(RValue<Float4> lhs, RValue<Float4> rhs)
6735         {
6736                 return RValue<Float4>(Nucleus::createFSub(lhs.value, rhs.value));
6737         }
6738
6739         RValue<Float4> operator*(RValue<Float4> lhs, RValue<Float4> rhs)
6740         {
6741                 return RValue<Float4>(Nucleus::createFMul(lhs.value, rhs.value));
6742         }
6743
6744         RValue<Float4> operator/(RValue<Float4> lhs, RValue<Float4> rhs)
6745         {
6746                 return RValue<Float4>(Nucleus::createFDiv(lhs.value, rhs.value));
6747         }
6748
6749         RValue<Float4> operator%(RValue<Float4> lhs, RValue<Float4> rhs)
6750         {
6751                 return RValue<Float4>(Nucleus::createFRem(lhs.value, rhs.value));
6752         }
6753
6754         RValue<Float4> operator+=(Float4 &lhs, RValue<Float4> rhs)
6755         {
6756                 return lhs = lhs + rhs;
6757         }
6758
6759         RValue<Float4> operator-=(Float4 &lhs, RValue<Float4> rhs)
6760         {
6761                 return lhs = lhs - rhs;
6762         }
6763
6764         RValue<Float4> operator*=(Float4 &lhs, RValue<Float4> rhs)
6765         {
6766                 return lhs = lhs * rhs;
6767         }
6768
6769         RValue<Float4> operator/=(Float4 &lhs, RValue<Float4> rhs)
6770         {
6771                 return lhs = lhs / rhs;
6772         }
6773
6774         RValue<Float4> operator%=(Float4 &lhs, RValue<Float4> rhs)
6775         {
6776                 return lhs = lhs % rhs;
6777         }
6778
6779         RValue<Float4> operator+(RValue<Float4> val)
6780         {
6781                 return val;
6782         }
6783
6784         RValue<Float4> operator-(RValue<Float4> val)
6785         {
6786                 return RValue<Float4>(Nucleus::createFNeg(val.value));
6787         }
6788
6789         RValue<Float4> Abs(RValue<Float4> x)
6790         {
6791                 Value *vector = Nucleus::createBitCast(x.value, Int4::getType());
6792                 int64_t constantVector[4] = {0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF};
6793                 Value *result = Nucleus::createAnd(vector, V(Nucleus::createConstantVector(constantVector, Int4::getType())));
6794
6795                 return As<Float4>(result);
6796         }
6797
6798         RValue<Float4> Max(RValue<Float4> x, RValue<Float4> y)
6799         {
6800                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v4i1);
6801                 auto cmp = Ice::InstFcmp::create(::function, Ice::InstFcmp::Ogt, condition, x.value, y.value);
6802                 ::basicBlock->appendInst(cmp);
6803
6804                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
6805                 auto select = Ice::InstSelect::create(::function, result, condition, x.value, y.value);
6806                 ::basicBlock->appendInst(select);
6807
6808                 return RValue<Float4>(V(result));
6809         }
6810
6811         RValue<Float4> Min(RValue<Float4> x, RValue<Float4> y)
6812         {
6813                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v4i1);
6814                 auto cmp = Ice::InstFcmp::create(::function, Ice::InstFcmp::Olt, condition, x.value, y.value);
6815                 ::basicBlock->appendInst(cmp);
6816
6817                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
6818                 auto select = Ice::InstSelect::create(::function, result, condition, x.value, y.value);
6819                 ::basicBlock->appendInst(select);
6820
6821                 return RValue<Float4>(V(result));
6822         }
6823
6824         RValue<Float4> Rcp_pp(RValue<Float4> x, bool exactAtPow2)
6825         {
6826                 return Float4(1.0f) / x;
6827         }
6828
6829         RValue<Float4> RcpSqrt_pp(RValue<Float4> x)
6830         {
6831                 return Rcp_pp(Sqrt(x));
6832         }
6833
6834         RValue<Float4> Sqrt(RValue<Float4> x)
6835         {
6836                 if(emulateIntrinsics)
6837                 {
6838                         Float4 result;
6839                         result.x = Sqrt(Float(Float4(x).x));
6840                         result.y = Sqrt(Float(Float4(x).y));
6841                         result.z = Sqrt(Float(Float4(x).z));
6842                         result.w = Sqrt(Float(Float4(x).w));
6843
6844                         return result;
6845                 }
6846                 else
6847                 {
6848                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
6849                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Sqrt, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
6850                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
6851                         auto sqrt = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
6852                         sqrt->addArg(x.value);
6853                         ::basicBlock->appendInst(sqrt);
6854
6855                         return RValue<Float4>(V(result));
6856                 }
6857         }
6858
6859         RValue<Float4> Insert(RValue<Float4> x, RValue<Float> element, int i)
6860         {
6861                 return RValue<Float4>(Nucleus::createInsertElement(x.value, element.value, i));
6862         }
6863
6864         RValue<Float> Extract(RValue<Float4> x, int i)
6865         {
6866                 return RValue<Float>(Nucleus::createExtractElement(x.value, Float::getType(), i));
6867         }
6868
6869         RValue<Float4> Swizzle(RValue<Float4> x, unsigned char select)
6870         {
6871                 return RValue<Float4>(createSwizzle4(x.value, select));
6872         }
6873
6874         RValue<Float4> ShuffleLowHigh(RValue<Float4> x, RValue<Float4> y, unsigned char imm)
6875         {
6876                 int shuffle[4] =
6877                 {
6878                         ((imm >> 0) & 0x03) + 0,
6879                         ((imm >> 2) & 0x03) + 0,
6880                         ((imm >> 4) & 0x03) + 4,
6881                         ((imm >> 6) & 0x03) + 4,
6882                 };
6883
6884                 return RValue<Float4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
6885         }
6886
6887         RValue<Float4> UnpackLow(RValue<Float4> x, RValue<Float4> y)
6888         {
6889                 int shuffle[4] = {0, 4, 1, 5};
6890                 return RValue<Float4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
6891         }
6892
6893         RValue<Float4> UnpackHigh(RValue<Float4> x, RValue<Float4> y)
6894         {
6895                 int shuffle[4] = {2, 6, 3, 7};
6896                 return RValue<Float4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
6897         }
6898
6899         RValue<Float4> Mask(Float4 &lhs, RValue<Float4> rhs, unsigned char select)
6900         {
6901                 Value *vector = lhs.loadValue();
6902                 Value *result = createMask4(vector, rhs.value, select);
6903                 lhs.storeValue(result);
6904
6905                 return RValue<Float4>(result);
6906         }
6907
6908         RValue<Int> SignMask(RValue<Float4> x)
6909         {
6910                 if(emulateIntrinsics)
6911                 {
6912                         Int4 xx = (As<Int4>(x) >> 31) & Int4(0x00000001, 0x00000002, 0x00000004, 0x00000008);
6913                         return Extract(xx, 0) | Extract(xx, 1) | Extract(xx, 2) | Extract(xx, 3);
6914                 }
6915                 else
6916                 {
6917                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_i32);
6918                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SignMask, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
6919                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
6920                         auto movmsk = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
6921                         movmsk->addArg(x.value);
6922                         ::basicBlock->appendInst(movmsk);
6923
6924                         return RValue<Int>(V(result));
6925                 }
6926         }
6927
6928         RValue<Int4> CmpEQ(RValue<Float4> x, RValue<Float4> y)
6929         {
6930                 return RValue<Int4>(Nucleus::createFCmpOEQ(x.value, y.value));
6931         }
6932
6933         RValue<Int4> CmpLT(RValue<Float4> x, RValue<Float4> y)
6934         {
6935                 return RValue<Int4>(Nucleus::createFCmpOLT(x.value, y.value));
6936         }
6937
6938         RValue<Int4> CmpLE(RValue<Float4> x, RValue<Float4> y)
6939         {
6940                 return RValue<Int4>(Nucleus::createFCmpOLE(x.value, y.value));
6941         }
6942
6943         RValue<Int4> CmpNEQ(RValue<Float4> x, RValue<Float4> y)
6944         {
6945                 return RValue<Int4>(Nucleus::createFCmpONE(x.value, y.value));
6946         }
6947
6948         RValue<Int4> CmpNLT(RValue<Float4> x, RValue<Float4> y)
6949         {
6950                 return RValue<Int4>(Nucleus::createFCmpOGE(x.value, y.value));
6951         }
6952
6953         RValue<Int4> CmpNLE(RValue<Float4> x, RValue<Float4> y)
6954         {
6955                 return RValue<Int4>(Nucleus::createFCmpOGT(x.value, y.value));
6956         }
6957
6958         RValue<Float4> Round(RValue<Float4> x)
6959         {
6960                 if(emulateIntrinsics)
6961                 {
6962                         // Push the fractional part off the mantissa. Accurate up to +/-2^22.
6963                         return (x + Float4(0x00C00000)) - Float4(0x00C00000);
6964                 }
6965                 else if(CPUID::SSE4_1)
6966                 {
6967                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
6968                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Round, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
6969                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
6970                         auto round = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
6971                         round->addArg(x.value);
6972                         round->addArg(::context->getConstantInt32(0));
6973                         ::basicBlock->appendInst(round);
6974
6975                         return RValue<Float4>(V(result));
6976                 }
6977                 else
6978                 {
6979                         return Float4(RoundInt(x));
6980                 }
6981         }
6982
6983         RValue<Float4> Trunc(RValue<Float4> x)
6984         {
6985                 if(CPUID::SSE4_1)
6986                 {
6987                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
6988                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Round, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
6989                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
6990                         auto round = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
6991                         round->addArg(x.value);
6992                         round->addArg(::context->getConstantInt32(3));
6993                         ::basicBlock->appendInst(round);
6994
6995                         return RValue<Float4>(V(result));
6996                 }
6997                 else
6998                 {
6999                         return Float4(Int4(x));
7000                 }
7001         }
7002
7003         RValue<Float4> Frac(RValue<Float4> x)
7004         {
7005                 Float4 frc;
7006
7007                 if(CPUID::SSE4_1)
7008                 {
7009                         frc = x - Floor(x);
7010                 }
7011                 else
7012                 {
7013                         frc = x - Float4(Int4(x));   // Signed fractional part.
7014
7015                         frc += As<Float4>(As<Int4>(CmpNLE(Float4(0.0f), frc)) & As<Int4>(Float4(1, 1, 1, 1)));   // Add 1.0 if negative.
7016                 }
7017
7018                 // x - floor(x) can be 1.0 for very small negative x.
7019                 // Clamp against the value just below 1.0.
7020                 return Min(frc, As<Float4>(Int4(0x3F7FFFFF)));
7021         }
7022
7023         RValue<Float4> Floor(RValue<Float4> x)
7024         {
7025                 if(CPUID::SSE4_1)
7026                 {
7027                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
7028                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Round, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
7029                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
7030                         auto round = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
7031                         round->addArg(x.value);
7032                         round->addArg(::context->getConstantInt32(1));
7033                         ::basicBlock->appendInst(round);
7034
7035                         return RValue<Float4>(V(result));
7036                 }
7037                 else
7038                 {
7039                         return x - Frac(x);
7040                 }
7041         }
7042
7043         RValue<Float4> Ceil(RValue<Float4> x)
7044         {
7045                 if(CPUID::SSE4_1)
7046                 {
7047                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
7048                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Round, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
7049                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
7050                         auto round = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
7051                         round->addArg(x.value);
7052                         round->addArg(::context->getConstantInt32(2));
7053                         ::basicBlock->appendInst(round);
7054
7055                         return RValue<Float4>(V(result));
7056                 }
7057                 else
7058                 {
7059                         return -Floor(-x);
7060                 }
7061         }
7062
7063         Type *Float4::getType()
7064         {
7065                 return T(Ice::IceType_v4f32);
7066         }
7067
7068         RValue<Pointer<Byte>> operator+(RValue<Pointer<Byte>> lhs, int offset)
7069         {
7070                 return lhs + RValue<Int>(Nucleus::createConstantInt(offset));
7071         }
7072
7073         RValue<Pointer<Byte>> operator+(RValue<Pointer<Byte>> lhs, RValue<Int> offset)
7074         {
7075                 return RValue<Pointer<Byte>>(Nucleus::createGEP(lhs.value, Byte::getType(), offset.value, false));
7076         }
7077
7078         RValue<Pointer<Byte>> operator+(RValue<Pointer<Byte>> lhs, RValue<UInt> offset)
7079         {
7080                 return RValue<Pointer<Byte>>(Nucleus::createGEP(lhs.value, Byte::getType(), offset.value, true));
7081         }
7082
7083         RValue<Pointer<Byte>> operator+=(Pointer<Byte> &lhs, int offset)
7084         {
7085                 return lhs = lhs + offset;
7086         }
7087
7088         RValue<Pointer<Byte>> operator+=(Pointer<Byte> &lhs, RValue<Int> offset)
7089         {
7090                 return lhs = lhs + offset;
7091         }
7092
7093         RValue<Pointer<Byte>> operator+=(Pointer<Byte> &lhs, RValue<UInt> offset)
7094         {
7095                 return lhs = lhs + offset;
7096         }
7097
7098         RValue<Pointer<Byte>> operator-(RValue<Pointer<Byte>> lhs, int offset)
7099         {
7100                 return lhs + -offset;
7101         }
7102
7103         RValue<Pointer<Byte>> operator-(RValue<Pointer<Byte>> lhs, RValue<Int> offset)
7104         {
7105                 return lhs + -offset;
7106         }
7107
7108         RValue<Pointer<Byte>> operator-(RValue<Pointer<Byte>> lhs, RValue<UInt> offset)
7109         {
7110                 return lhs + -offset;
7111         }
7112
7113         RValue<Pointer<Byte>> operator-=(Pointer<Byte> &lhs, int offset)
7114         {
7115                 return lhs = lhs - offset;
7116         }
7117
7118         RValue<Pointer<Byte>> operator-=(Pointer<Byte> &lhs, RValue<Int> offset)
7119         {
7120                 return lhs = lhs - offset;
7121         }
7122
7123         RValue<Pointer<Byte>> operator-=(Pointer<Byte> &lhs, RValue<UInt> offset)
7124         {
7125                 return lhs = lhs - offset;
7126         }
7127
7128         void Return()
7129         {
7130                 Nucleus::createRetVoid();
7131                 Nucleus::setInsertBlock(Nucleus::createBasicBlock());
7132                 Nucleus::createUnreachable();
7133         }
7134
7135         void Return(RValue<Int> ret)
7136         {
7137                 Nucleus::createRet(ret.value);
7138                 Nucleus::setInsertBlock(Nucleus::createBasicBlock());
7139                 Nucleus::createUnreachable();
7140         }
7141
7142         void branch(RValue<Bool> cmp, BasicBlock *bodyBB, BasicBlock *endBB)
7143         {
7144                 Nucleus::createCondBr(cmp.value, bodyBB, endBB);
7145                 Nucleus::setInsertBlock(bodyBB);
7146         }
7147
7148         RValue<Long> Ticks()
7149         {
7150                 assert(false && "UNIMPLEMENTED"); return RValue<Long>(V(nullptr));
7151         }
7152 }