src/Reactor/SubzeroReactor.cpp

   1 // Copyright 2016 The SwiftShader Authors. All Rights Reserved.
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License");
   4 // you may not use this file except in compliance with the License.
   5 // You may obtain a copy of the License at
   6 //
   7 //    http://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS,
  11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 // See the License for the specific language governing permissions and
  13 // limitations under the License.
  14
  15 #include "Nucleus.hpp"
  16
  17 #include "Reactor.hpp"
  18 #include "Routine.hpp"
  19
  20 #include "Optimizer.hpp"
  21
  22 #include "src/IceTypes.h"
  23 #include "src/IceCfg.h"
  24 #include "src/IceELFStreamer.h"
  25 #include "src/IceGlobalContext.h"
  26 #include "src/IceCfgNode.h"
  27 #include "src/IceELFObjectWriter.h"
  28 #include "src/IceGlobalInits.h"
  29
  30 #include "llvm/Support/FileSystem.h"
  31 #include "llvm/Support/raw_os_ostream.h"
  32
  33 #if defined(_WIN32)
  34 #ifndef WIN32_LEAN_AND_MEAN
  35 #define WIN32_LEAN_AND_MEAN
  36 #endif // !WIN32_LEAN_AND_MEAN
  37 #ifndef NOMINMAX
  38 #define NOMINMAX
  39 #endif // !NOMINMAX
  40 #include <Windows.h>
  41 #else
  42 #include <sys/mman.h>
  43 #if !defined(MAP_ANONYMOUS)
  44 #define MAP_ANONYMOUS MAP_ANON
  45 #endif
  46 #endif
  47
  48 #include <mutex>
  49 #include <limits>
  50 #include <iostream>
  51 #include <cassert>
  52
  53 namespace
  54 {
  55         Ice::GlobalContext *context = nullptr;
  56         Ice::Cfg *function = nullptr;
  57         Ice::CfgNode *basicBlock = nullptr;
  58         Ice::CfgLocalAllocatorScope *allocator = nullptr;
  59         sw::Routine *routine = nullptr;
  60
  61         std::mutex codegenMutex;
  62
  63         Ice::ELFFileStreamer *elfFile = nullptr;
  64         Ice::Fdstream *out = nullptr;
  65 }
  66
  67 namespace
  68 {
  69         #if !defined(__i386__) && defined(_M_IX86)
  70                 #define __i386__ 1
  71         #endif
  72
  73         #if !defined(__x86_64__) && (defined(_M_AMD64) || defined (_M_X64))
  74                 #define __x86_64__ 1
  75         #endif
  76
  77         class CPUID
  78         {
  79         public:
  80                 const static bool ARM;
  81                 const static bool SSE4_1;
  82
  83         private:
  84                 static void cpuid(int registers[4], int info)
  85                 {
  86                         #if defined(__i386__) || defined(__x86_64__)
  87                                 #if defined(_WIN32)
  88                                         __cpuid(registers, info);
  89                                 #else
  90                                         __asm volatile("cpuid": "=a" (registers[0]), "=b" (registers[1]), "=c" (registers[2]), "=d" (registers[3]): "a" (info));
  91                                 #endif
  92                         #else
  93                                 registers[0] = 0;
  94                                 registers[1] = 0;
  95                                 registers[2] = 0;
  96                                 registers[3] = 0;
  97                         #endif
  98                 }
  99
 100                 static bool detectARM()
 101                 {
 102                         #if defined(__arm__)
 103                                 return true;
 104                         #elif defined(__i386__) || defined(__x86_64__)
 105                                 return false;
 106                         #else
 107                                 #error "Unknown architecture"
 108                         #endif
 109                 }
 110
 111                 static bool detectSSE4_1()
 112                 {
 113                         #if defined(__i386__) || defined(__x86_64__)
 114                                 int registers[4];
 115                                 cpuid(registers, 1);
 116                                 return (registers[2] & 0x00080000) != 0;
 117                         #else
 118                                 return false;
 119                         #endif
 120                 }
 121         };
 122
 123         const bool CPUID::ARM = CPUID::detectARM();
 124         const bool CPUID::SSE4_1 = CPUID::detectSSE4_1();
 125         const bool emulateIntrinsics = CPUID::ARM;
 126 }
 127
 128 namespace sw
 129 {
 130         enum EmulatedType
 131         {
 132                 EmulatedShift = 16,
 133                 EmulatedV2 = 2 << EmulatedShift,
 134                 EmulatedV4 = 4 << EmulatedShift,
 135                 EmulatedV8 = 8 << EmulatedShift,
 136                 EmulatedBits = EmulatedV2 | EmulatedV4 | EmulatedV8,
 137
 138                 Type_v2i32 = Ice::IceType_v4i32 | EmulatedV2,
 139                 Type_v4i16 = Ice::IceType_v8i16 | EmulatedV4,
 140                 Type_v2i16 = Ice::IceType_v8i16 | EmulatedV2,
 141                 Type_v8i8 =  Ice::IceType_v16i8 | EmulatedV8,
 142                 Type_v4i8 =  Ice::IceType_v16i8 | EmulatedV4,
 143                 Type_v2f32 = Ice::IceType_v4f32 | EmulatedV2,
 144         };
 145
 146         class Value : public Ice::Operand {};
 147         class SwitchCases : public Ice::InstSwitch {};
 148         class BasicBlock : public Ice::CfgNode {};
 149
 150         Ice::Type T(Type *t)
 151         {
 152                 static_assert(static_cast<unsigned int>(Ice::IceType_NUM) < static_cast<unsigned int>(EmulatedBits), "Ice::Type overlaps with our emulated types!");
 153                 return (Ice::Type)(reinterpret_cast<std::intptr_t>(t) & ~EmulatedBits);
 154         }
 155
 156         Type *T(Ice::Type t)
 157         {
 158                 return reinterpret_cast<Type*>(t);
 159         }
 160
 161         Type *T(EmulatedType t)
 162         {
 163                 return reinterpret_cast<Type*>(t);
 164         }
 165
 166         Value *V(Ice::Operand *v)
 167         {
 168                 return reinterpret_cast<Value*>(v);
 169         }
 170
 171         BasicBlock *B(Ice::CfgNode *b)
 172         {
 173                 return reinterpret_cast<BasicBlock*>(b);
 174         }
 175
 176         static size_t typeSize(Type *type)
 177         {
 178                 if(reinterpret_cast<std::intptr_t>(type) & EmulatedBits)
 179                 {
 180                         switch(reinterpret_cast<std::intptr_t>(type))
 181                         {
 182                         case Type_v2i32: return 8;
 183                         case Type_v4i16: return 8;
 184                         case Type_v2i16: return 4;
 185                         case Type_v8i8:  return 8;
 186                         case Type_v4i8:  return 4;
 187                         case Type_v2f32: return 8;
 188                         default: assert(false);
 189                         }
 190                 }
 191
 192                 return Ice::typeWidthInBytes(T(type));
 193         }
 194
 195         Optimization optimization[10] = {InstructionCombining, Disabled};
 196
 197         using ElfHeader = std::conditional<sizeof(void*) == 8, Elf64_Ehdr, Elf32_Ehdr>::type;
 198         using SectionHeader = std::conditional<sizeof(void*) == 8, Elf64_Shdr, Elf32_Shdr>::type;
 199
 200         inline const SectionHeader *sectionHeader(const ElfHeader *elfHeader)
 201         {
 202                 return reinterpret_cast<const SectionHeader*>((intptr_t)elfHeader + elfHeader->e_shoff);
 203         }
 204
 205         inline const SectionHeader *elfSection(const ElfHeader *elfHeader, int index)
 206         {
 207                 return &sectionHeader(elfHeader)[index];
 208         }
 209
 210         static void *relocateSymbol(const ElfHeader *elfHeader, const Elf32_Rel &relocation, const SectionHeader &relocationTable)
 211         {
 212                 const SectionHeader *target = elfSection(elfHeader, relocationTable.sh_info);
 213
 214                 intptr_t address = (intptr_t)elfHeader + target->sh_offset;
 215                 int32_t *patchSite = (int*)(address + relocation.r_offset);
 216                 uint32_t index = relocation.getSymbol();
 217                 int table = relocationTable.sh_link;
 218                 void *symbolValue = nullptr;
 219
 220                 if(index != SHN_UNDEF)
 221                 {
 222                         if(table == SHN_UNDEF) return nullptr;
 223                         const SectionHeader *symbolTable = elfSection(elfHeader, table);
 224
 225                         uint32_t symtab_entries = symbolTable->sh_size / symbolTable->sh_entsize;
 226                         if(index >= symtab_entries)
 227                         {
 228                                 assert(index < symtab_entries && "Symbol Index out of range");
 229                                 return nullptr;
 230                         }
 231
 232                         intptr_t symbolAddress = (intptr_t)elfHeader + symbolTable->sh_offset;
 233                         Elf32_Sym &symbol = ((Elf32_Sym*)symbolAddress)[index];
 234                         uint16_t section = symbol.st_shndx;
 235
 236                         if(section != SHN_UNDEF && section < SHN_LORESERVE)
 237                         {
 238                                 const SectionHeader *target = elfSection(elfHeader, symbol.st_shndx);
 239                                 symbolValue = reinterpret_cast<void*>((intptr_t)elfHeader + symbol.st_value + target->sh_offset);
 240                         }
 241                         else
 242                         {
 243                                 return nullptr;
 244                         }
 245                 }
 246
 247                 if(CPUID::ARM)
 248                 {
 249                         switch(relocation.getType())
 250                         {
 251                         case R_ARM_NONE:
 252                                 // No relocation
 253                                 break;
 254                         case R_ARM_MOVW_ABS_NC:
 255                                 {
 256                                         uint32_t thumb = 0;   // Calls to Thumb code not supported.
 257                                         uint32_t lo = (uint32_t)(intptr_t)symbolValue | thumb;
 258                                         *patchSite = (*patchSite & 0xFFF0F000) | ((lo & 0xF000) << 4) | (lo & 0x0FFF);
 259                                 }
 260                                 break;
 261                         case R_ARM_MOVT_ABS:
 262                                 {
 263                                         uint32_t hi = (uint32_t)(intptr_t)(symbolValue) >> 16;
 264                                         *patchSite = (*patchSite & 0xFFF0F000) | ((hi & 0xF000) << 4) | (hi & 0x0FFF);
 265                                 }
 266                                 break;
 267                         default:
 268                                 assert(false && "Unsupported relocation type");
 269                                 return nullptr;
 270                         }
 271                 }
 272                 else
 273                 {
 274                         switch(relocation.getType())
 275                         {
 276                         case R_386_NONE:
 277                                 // No relocation
 278                                 break;
 279                         case R_386_32:
 280                                 *patchSite = (int32_t)((intptr_t)symbolValue + *patchSite);
 281                                 break;
 282                 //      case R_386_PC32:
 283                 //              *patchSite = (int32_t)((intptr_t)symbolValue + *patchSite - (intptr_t)patchSite);
 284                 //              break;
 285                         default:
 286                                 assert(false && "Unsupported relocation type");
 287                                 return nullptr;
 288                         }
 289                 }
 290
 291
 292                 return symbolValue;
 293         }
 294
 295         static void *relocateSymbol(const ElfHeader *elfHeader, const Elf64_Rela &relocation, const SectionHeader &relocationTable)
 296         {
 297                 const SectionHeader *target = elfSection(elfHeader, relocationTable.sh_info);
 298
 299                 intptr_t address = (intptr_t)elfHeader + target->sh_offset;
 300                 int32_t *patchSite = (int*)(address + relocation.r_offset);
 301                 uint32_t index = relocation.getSymbol();
 302                 int table = relocationTable.sh_link;
 303                 void *symbolValue = nullptr;
 304
 305                 if(index != SHN_UNDEF)
 306                 {
 307                         if(table == SHN_UNDEF) return nullptr;
 308                         const SectionHeader *symbolTable = elfSection(elfHeader, table);
 309
 310                         uint32_t symtab_entries = symbolTable->sh_size / symbolTable->sh_entsize;
 311                         if(index >= symtab_entries)
 312                         {
 313                                 assert(index < symtab_entries && "Symbol Index out of range");
 314                                 return nullptr;
 315                         }
 316
 317                         intptr_t symbolAddress = (intptr_t)elfHeader + symbolTable->sh_offset;
 318                         Elf64_Sym &symbol = ((Elf64_Sym*)symbolAddress)[index];
 319                         uint16_t section = symbol.st_shndx;
 320
 321                         if(section != SHN_UNDEF && section < SHN_LORESERVE)
 322                         {
 323                                 const SectionHeader *target = elfSection(elfHeader, symbol.st_shndx);
 324                                 symbolValue = reinterpret_cast<void*>((intptr_t)elfHeader + symbol.st_value + target->sh_offset);
 325                         }
 326                         else
 327                         {
 328                                 return nullptr;
 329                         }
 330                 }
 331
 332                 switch(relocation.getType())
 333                 {
 334                 case R_X86_64_NONE:
 335                         // No relocation
 336                         break;
 337                 case R_X86_64_64:
 338                         *(int64_t*)patchSite = (int64_t)((intptr_t)symbolValue + *(int64_t*)patchSite) + relocation.r_addend;
 339                         break;
 340                 case R_X86_64_PC32:
 341                         *patchSite = (int32_t)((intptr_t)symbolValue + *patchSite - (intptr_t)patchSite) + relocation.r_addend;
 342                         break;
 343                 case R_X86_64_32S:
 344                         *patchSite = (int32_t)((intptr_t)symbolValue + *patchSite) + relocation.r_addend;
 345                         break;
 346                 default:
 347                         assert(false && "Unsupported relocation type");
 348                         return nullptr;
 349                 }
 350
 351                 return symbolValue;
 352         }
 353
 354         void *loadImage(uint8_t *const elfImage, size_t &codeSize)
 355         {
 356                 ElfHeader *elfHeader = (ElfHeader*)elfImage;
 357
 358                 if(!elfHeader->checkMagic())
 359                 {
 360                         return nullptr;
 361                 }
 362
 363                 // Expect ELF bitness to match platform
 364                 assert(sizeof(void*) == 8 ? elfHeader->getFileClass() == ELFCLASS64 : elfHeader->getFileClass() == ELFCLASS32);
 365                 #if defined(__i386__)
 366                         assert(sizeof(void*) == 4 && elfHeader->e_machine == EM_386);
 367                 #elif defined(__x86_64__)
 368                         assert(sizeof(void*) == 8 && elfHeader->e_machine == EM_X86_64);
 369                 #elif defined(__arm__)
 370                         assert(sizeof(void*) == 4 && elfHeader->e_machine == EM_ARM);
 371                 #else
 372                         #error "Unsupported platform"
 373                 #endif
 374
 375                 SectionHeader *sectionHeader = (SectionHeader*)(elfImage + elfHeader->e_shoff);
 376                 void *entry = nullptr;
 377
 378                 for(int i = 0; i < elfHeader->e_shnum; i++)
 379                 {
 380                         if(sectionHeader[i].sh_type == SHT_PROGBITS)
 381                         {
 382                                 if(sectionHeader[i].sh_flags & SHF_EXECINSTR)
 383                                 {
 384                                         entry = elfImage + sectionHeader[i].sh_offset;
 385                                         codeSize = sectionHeader[i].sh_size;
 386                                 }
 387                         }
 388                         else if(sectionHeader[i].sh_type == SHT_REL)
 389                         {
 390                                 assert(sizeof(void*) == 4 && "UNIMPLEMENTED");   // Only expected/implemented for 32-bit code
 391
 392                                 for(Elf32_Word index = 0; index < sectionHeader[i].sh_size / sectionHeader[i].sh_entsize; index++)
 393                                 {
 394                                         const Elf32_Rel &relocation = ((const Elf32_Rel*)(elfImage + sectionHeader[i].sh_offset))[index];
 395                                         relocateSymbol(elfHeader, relocation, sectionHeader[i]);
 396                                 }
 397                         }
 398                         else if(sectionHeader[i].sh_type == SHT_RELA)
 399                         {
 400                                 assert(sizeof(void*) == 8 && "UNIMPLEMENTED");   // Only expected/implemented for 64-bit code
 401
 402                                 for(Elf32_Word index = 0; index < sectionHeader[i].sh_size / sectionHeader[i].sh_entsize; index++)
 403                                 {
 404                                         const Elf64_Rela &relocation = ((const Elf64_Rela*)(elfImage + sectionHeader[i].sh_offset))[index];
 405                                         relocateSymbol(elfHeader, relocation, sectionHeader[i]);
 406                                 }
 407                         }
 408                 }
 409
 410                 return entry;
 411         }
 412
 413         template<typename T>
 414         struct ExecutableAllocator
 415         {
 416                 ExecutableAllocator() {};
 417                 template<class U> ExecutableAllocator(const ExecutableAllocator<U> &other) {};
 418
 419                 using value_type = T;
 420                 using size_type = std::size_t;
 421
 422                 T *allocate(size_type n)
 423                 {
 424                         #if defined(_WIN32)
 425                                 return (T*)VirtualAlloc(NULL, sizeof(T) * n, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
 426                         #else
 427                                 return (T*)mmap(nullptr, sizeof(T) * n, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 428                         #endif
 429                 }
 430
 431                 void deallocate(T *p, size_type n)
 432                 {
 433                         #if defined(_WIN32)
 434                                 VirtualFree(p, 0, MEM_RELEASE);
 435                         #else
 436                                 munmap(p, sizeof(T) * n);
 437                         #endif
 438                 }
 439         };
 440
 441         class ELFMemoryStreamer : public Ice::ELFStreamer, public Routine
 442         {
 443                 ELFMemoryStreamer(const ELFMemoryStreamer &) = delete;
 444                 ELFMemoryStreamer &operator=(const ELFMemoryStreamer &) = delete;
 445
 446         public:
 447                 ELFMemoryStreamer() : Routine(), entry(nullptr)
 448                 {
 449                         position = 0;
 450                         buffer.reserve(0x1000);
 451                 }
 452
 453                 ~ELFMemoryStreamer() override
 454                 {
 455                         #if defined(_WIN32)
 456                                 if(buffer.size() != 0)
 457                                 {
 458                                         DWORD exeProtection;
 459                                         VirtualProtect(&buffer[0], buffer.size(), oldProtection, &exeProtection);
 460                                 }
 461                         #endif
 462                 }
 463
 464                 void write8(uint8_t Value) override
 465                 {
 466                         if(position == (uint64_t)buffer.size())
 467                         {
 468                                 buffer.push_back(Value);
 469                                 position++;
 470                         }
 471                         else if(position < (uint64_t)buffer.size())
 472                         {
 473                                 buffer[position] = Value;
 474                                 position++;
 475                         }
 476                         else assert(false && "UNIMPLEMENTED");
 477                 }
 478
 479                 void writeBytes(llvm::StringRef Bytes) override
 480                 {
 481                         std::size_t oldSize = buffer.size();
 482                         buffer.resize(oldSize + Bytes.size());
 483                         memcpy(&buffer[oldSize], Bytes.begin(), Bytes.size());
 484                         position += Bytes.size();
 485                 }
 486
 487                 uint64_t tell() const override { return position; }
 488
 489                 void seek(uint64_t Off) override { position = Off; }
 490
 491                 const void *getEntry() override
 492                 {
 493                         if(!entry)
 494                         {
 495                                 position = std::numeric_limits<std::size_t>::max();   // Can't stream more data after this
 496
 497                                 size_t codeSize = 0;
 498                                 entry = loadImage(&buffer[0], codeSize);
 499
 500                                 #if defined(_WIN32)
 501                                         VirtualProtect(&buffer[0], buffer.size(), PAGE_EXECUTE_READ, &oldProtection);
 502                                         FlushInstructionCache(GetCurrentProcess(), NULL, 0);
 503                                 #else
 504                                         mprotect(&buffer[0], buffer.size(), PROT_READ | PROT_EXEC);
 505                                         __builtin___clear_cache((char*)entry, (char*)entry + codeSize);
 506                                 #endif
 507                         }
 508
 509                         return entry;
 510                 }
 511
 512         private:
 513                 void *entry;
 514                 std::vector<uint8_t, ExecutableAllocator<uint8_t>> buffer;
 515                 std::size_t position;
 516
 517                 #if defined(_WIN32)
 518                 DWORD oldProtection;
 519                 #endif
 520         };
 521
 522         Nucleus::Nucleus()
 523         {
 524                 ::codegenMutex.lock();   // Reactor is currently not thread safe
 525
 526                 Ice::ClFlags &Flags = Ice::ClFlags::Flags;
 527                 Ice::ClFlags::getParsedClFlags(Flags);
 528
 529                 #if defined(__arm__)
 530                         Flags.setTargetArch(Ice::Target_ARM32);
 531                         Flags.setTargetInstructionSet(Ice::ARM32InstructionSet_HWDivArm);
 532                 #else   // x86
 533                         Flags.setTargetArch(sizeof(void*) == 8 ? Ice::Target_X8664 : Ice::Target_X8632);
 534                         Flags.setTargetInstructionSet(CPUID::SSE4_1 ? Ice::X86InstructionSet_SSE4_1 : Ice::X86InstructionSet_SSE2);
 535                 #endif
 536                 Flags.setOutFileType(Ice::FT_Elf);
 537                 Flags.setOptLevel(Ice::Opt_2);
 538                 Flags.setApplicationBinaryInterface(Ice::ABI_Platform);
 539                 Flags.setVerbose(false ? Ice::IceV_Most : Ice::IceV_None);
 540                 Flags.setDisableHybridAssembly(true);
 541
 542                 static llvm::raw_os_ostream cout(std::cout);
 543                 static llvm::raw_os_ostream cerr(std::cerr);
 544
 545                 if(false)   // Write out to a file
 546                 {
 547                         std::error_code errorCode;
 548                         ::out = new Ice::Fdstream("out.o", errorCode, llvm::sys::fs::F_None);
 549                         ::elfFile = new Ice::ELFFileStreamer(*out);
 550                         ::context = new Ice::GlobalContext(&cout, &cout, &cerr, elfFile);
 551                 }
 552                 else
 553                 {
 554                         ELFMemoryStreamer *elfMemory = new ELFMemoryStreamer();
 555                         ::context = new Ice::GlobalContext(&cout, &cout, &cerr, elfMemory);
 556                         ::routine = elfMemory;
 557                 }
 558         }
 559
 560         Nucleus::~Nucleus()
 561         {
 562                 delete ::routine;
 563
 564                 delete ::allocator;
 565                 delete ::function;
 566                 delete ::context;
 567
 568                 delete ::elfFile;
 569                 delete ::out;
 570
 571                 ::codegenMutex.unlock();
 572         }
 573
 574         Routine *Nucleus::acquireRoutine(const wchar_t *name, bool runOptimizations)
 575         {
 576                 if(basicBlock->getInsts().empty() || basicBlock->getInsts().back().getKind() != Ice::Inst::Ret)
 577                 {
 578                         createRetVoid();
 579                 }
 580
 581                 std::wstring wideName(name);
 582                 std::string asciiName(wideName.begin(), wideName.end());
 583                 ::function->setFunctionName(Ice::GlobalString::createWithString(::context, asciiName));
 584
 585                 optimize();
 586
 587                 ::function->translate();
 588                 assert(!::function->hasError());
 589
 590                 auto globals = ::function->getGlobalInits();
 591
 592                 if(globals && !globals->empty())
 593                 {
 594                         ::context->getGlobals()->merge(globals.get());
 595                 }
 596
 597                 ::context->emitFileHeader();
 598                 ::function->emitIAS();
 599                 auto assembler = ::function->releaseAssembler();
 600                 auto objectWriter = ::context->getObjectWriter();
 601                 assembler->alignFunction();
 602                 objectWriter->writeFunctionCode(::function->getFunctionName(), false, assembler.get());
 603                 ::context->lowerGlobals("last");
 604                 ::context->lowerConstants();
 605                 ::context->lowerJumpTables();
 606                 objectWriter->setUndefinedSyms(::context->getConstantExternSyms());
 607                 objectWriter->writeNonUserSections();
 608
 609                 Routine *handoffRoutine = ::routine;
 610                 ::routine = nullptr;
 611
 612                 return handoffRoutine;
 613         }
 614
 615         void Nucleus::optimize()
 616         {
 617                 sw::optimize(::function);
 618         }
 619
 620         Value *Nucleus::allocateStackVariable(Type *t, int arraySize)
 621         {
 622                 Ice::Type type = T(t);
 623                 int typeSize = Ice::typeWidthInBytes(type);
 624                 int totalSize = typeSize * (arraySize ? arraySize : 1);
 625
 626                 auto bytes = Ice::ConstantInteger32::create(::context, type, totalSize);
 627                 auto address = ::function->makeVariable(T(getPointerType(t)));
 628                 auto alloca = Ice::InstAlloca::create(::function, address, bytes, typeSize);
 629                 ::function->getEntryNode()->getInsts().push_front(alloca);
 630
 631                 return V(address);
 632         }
 633
 634         BasicBlock *Nucleus::createBasicBlock()
 635         {
 636                 return B(::function->makeNode());
 637         }
 638
 639         BasicBlock *Nucleus::getInsertBlock()
 640         {
 641                 return B(::basicBlock);
 642         }
 643
 644         void Nucleus::setInsertBlock(BasicBlock *basicBlock)
 645         {
 646         //      assert(::basicBlock->getInsts().back().getTerminatorEdges().size() >= 0 && "Previous basic block must have a terminator");
 647                 ::basicBlock = basicBlock;
 648         }
 649
 650         void Nucleus::createFunction(Type *ReturnType, std::vector<Type*> &Params)
 651         {
 652                 uint32_t sequenceNumber = 0;
 653                 ::function = Ice::Cfg::create(::context, sequenceNumber).release();
 654                 ::allocator = new Ice::CfgLocalAllocatorScope(::function);
 655
 656                 for(Type *type : Params)
 657                 {
 658                         Ice::Variable *arg = ::function->makeVariable(T(type));
 659                         ::function->addArg(arg);
 660                 }
 661
 662                 Ice::CfgNode *node = ::function->makeNode();
 663                 ::function->setEntryNode(node);
 664                 ::basicBlock = node;
 665         }
 666
 667         Value *Nucleus::getArgument(unsigned int index)
 668         {
 669                 return V(::function->getArgs()[index]);
 670         }
 671
 672         void Nucleus::createRetVoid()
 673         {
 674                 Ice::InstRet *ret = Ice::InstRet::create(::function);
 675                 ::basicBlock->appendInst(ret);
 676         }
 677
 678         void Nucleus::createRet(Value *v)
 679         {
 680                 Ice::InstRet *ret = Ice::InstRet::create(::function, v);
 681                 ::basicBlock->appendInst(ret);
 682         }
 683
 684         void Nucleus::createBr(BasicBlock *dest)
 685         {
 686                 auto br = Ice::InstBr::create(::function, dest);
 687                 ::basicBlock->appendInst(br);
 688         }
 689
 690         void Nucleus::createCondBr(Value *cond, BasicBlock *ifTrue, BasicBlock *ifFalse)
 691         {
 692                 auto br = Ice::InstBr::create(::function, cond, ifTrue, ifFalse);
 693                 ::basicBlock->appendInst(br);
 694         }
 695
 696         static bool isCommutative(Ice::InstArithmetic::OpKind op)
 697         {
 698                 switch(op)
 699                 {
 700                 case Ice::InstArithmetic::Add:
 701                 case Ice::InstArithmetic::Fadd:
 702                 case Ice::InstArithmetic::Mul:
 703                 case Ice::InstArithmetic::Fmul:
 704                 case Ice::InstArithmetic::And:
 705                 case Ice::InstArithmetic::Or:
 706                 case Ice::InstArithmetic::Xor:
 707                         return true;
 708                 default:
 709                         return false;
 710                 }
 711         }
 712
 713         static Value *createArithmetic(Ice::InstArithmetic::OpKind op, Value *lhs, Value *rhs)
 714         {
 715                 assert(lhs->getType() == rhs->getType() || (llvm::isa<Ice::Constant>(rhs) && (op == Ice::InstArithmetic::Shl || Ice::InstArithmetic::Lshr || Ice::InstArithmetic::Ashr)));
 716
 717                 bool swapOperands = llvm::isa<Ice::Constant>(lhs) && isCommutative(op);
 718
 719                 Ice::Variable *result = ::function->makeVariable(lhs->getType());
 720                 Ice::InstArithmetic *arithmetic = Ice::InstArithmetic::create(::function, op, result, swapOperands ? rhs : lhs, swapOperands ? lhs : rhs);
 721                 ::basicBlock->appendInst(arithmetic);
 722
 723                 return V(result);
 724         }
 725
 726         Value *Nucleus::createAdd(Value *lhs, Value *rhs)
 727         {
 728                 return createArithmetic(Ice::InstArithmetic::Add, lhs, rhs);
 729         }
 730
 731         Value *Nucleus::createSub(Value *lhs, Value *rhs)
 732         {
 733                 return createArithmetic(Ice::InstArithmetic::Sub, lhs, rhs);
 734         }
 735
 736         Value *Nucleus::createMul(Value *lhs, Value *rhs)
 737         {
 738                 return createArithmetic(Ice::InstArithmetic::Mul, lhs, rhs);
 739         }
 740
 741         Value *Nucleus::createUDiv(Value *lhs, Value *rhs)
 742         {
 743                 return createArithmetic(Ice::InstArithmetic::Udiv, lhs, rhs);
 744         }
 745
 746         Value *Nucleus::createSDiv(Value *lhs, Value *rhs)
 747         {
 748                 return createArithmetic(Ice::InstArithmetic::Sdiv, lhs, rhs);
 749         }
 750
 751         Value *Nucleus::createFAdd(Value *lhs, Value *rhs)
 752         {
 753                 return createArithmetic(Ice::InstArithmetic::Fadd, lhs, rhs);
 754         }
 755
 756         Value *Nucleus::createFSub(Value *lhs, Value *rhs)
 757         {
 758                 return createArithmetic(Ice::InstArithmetic::Fsub, lhs, rhs);
 759         }
 760
 761         Value *Nucleus::createFMul(Value *lhs, Value *rhs)
 762         {
 763                 return createArithmetic(Ice::InstArithmetic::Fmul, lhs, rhs);
 764         }
 765
 766         Value *Nucleus::createFDiv(Value *lhs, Value *rhs)
 767         {
 768                 return createArithmetic(Ice::InstArithmetic::Fdiv, lhs, rhs);
 769         }
 770
 771         Value *Nucleus::createURem(Value *lhs, Value *rhs)
 772         {
 773                 return createArithmetic(Ice::InstArithmetic::Urem, lhs, rhs);
 774         }
 775
 776         Value *Nucleus::createSRem(Value *lhs, Value *rhs)
 777         {
 778                 return createArithmetic(Ice::InstArithmetic::Srem, lhs, rhs);
 779         }
 780
 781         Value *Nucleus::createFRem(Value *lhs, Value *rhs)
 782         {
 783                 return createArithmetic(Ice::InstArithmetic::Frem, lhs, rhs);
 784         }
 785
 786         Value *Nucleus::createShl(Value *lhs, Value *rhs)
 787         {
 788                 return createArithmetic(Ice::InstArithmetic::Shl, lhs, rhs);
 789         }
 790
 791         Value *Nucleus::createLShr(Value *lhs, Value *rhs)
 792         {
 793                 return createArithmetic(Ice::InstArithmetic::Lshr, lhs, rhs);
 794         }
 795
 796         Value *Nucleus::createAShr(Value *lhs, Value *rhs)
 797         {
 798                 return createArithmetic(Ice::InstArithmetic::Ashr, lhs, rhs);
 799         }
 800
 801         Value *Nucleus::createAnd(Value *lhs, Value *rhs)
 802         {
 803                 return createArithmetic(Ice::InstArithmetic::And, lhs, rhs);
 804         }
 805
 806         Value *Nucleus::createOr(Value *lhs, Value *rhs)
 807         {
 808                 return createArithmetic(Ice::InstArithmetic::Or, lhs, rhs);
 809         }
 810
 811         Value *Nucleus::createXor(Value *lhs, Value *rhs)
 812         {
 813                 return createArithmetic(Ice::InstArithmetic::Xor, lhs, rhs);
 814         }
 815
 816         Value *Nucleus::createNeg(Value *v)
 817         {
 818                 return createSub(createNullValue(T(v->getType())), v);
 819         }
 820
 821         Value *Nucleus::createFNeg(Value *v)
 822         {
 823                 double c[4] = {-0.0, -0.0, -0.0, -0.0};
 824                 Value *negativeZero = Ice::isVectorType(v->getType()) ?
 825                                       createConstantVector(c, T(v->getType())) :
 826                                       V(::context->getConstantFloat(-0.0f));
 827
 828                 return createFSub(negativeZero, v);
 829         }
 830
 831         Value *Nucleus::createNot(Value *v)
 832         {
 833                 if(Ice::isScalarIntegerType(v->getType()))
 834                 {
 835                         return createXor(v, V(::context->getConstantInt(v->getType(), -1)));
 836                 }
 837                 else   // Vector
 838                 {
 839                         int64_t c[16] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1};
 840                         return createXor(v, createConstantVector(c, T(v->getType())));
 841                 }
 842         }
 843
 844         Value *Nucleus::createLoad(Value *ptr, Type *type, bool isVolatile, unsigned int align)
 845         {
 846                 int valueType = (int)reinterpret_cast<intptr_t>(type);
 847                 Ice::Variable *result = ::function->makeVariable(T(type));
 848
 849                 if(valueType & EmulatedBits)
 850                 {
 851                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::LoadSubVector, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
 852                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
 853                         auto load = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
 854                         load->addArg(ptr);
 855                         load->addArg(::context->getConstantInt32(typeSize(type)));
 856                         ::basicBlock->appendInst(load);
 857                 }
 858                 else
 859                 {
 860                         auto load = Ice::InstLoad::create(::function, result, ptr, align);
 861                         ::basicBlock->appendInst(load);
 862                 }
 863
 864                 return V(result);
 865         }
 866
 867         Value *Nucleus::createStore(Value *value, Value *ptr, Type *type, bool isVolatile, unsigned int align)
 868         {
 869                 int valueType = (int)reinterpret_cast<intptr_t>(type);
 870
 871                 if(valueType & EmulatedBits)
 872                 {
 873                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::StoreSubVector, Ice::Intrinsics::SideEffects_T, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_T};
 874                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
 875                         auto store = Ice::InstIntrinsicCall::create(::function, 3, nullptr, target, intrinsic);
 876                         store->addArg(value);
 877                         store->addArg(ptr);
 878                         store->addArg(::context->getConstantInt32(typeSize(type)));
 879                         ::basicBlock->appendInst(store);
 880                 }
 881                 else
 882                 {
 883                         assert(T(value->getType()) == type);
 884
 885                         auto store = Ice::InstStore::create(::function, value, ptr, align);
 886                         ::basicBlock->appendInst(store);
 887                 }
 888
 889                 return value;
 890         }
 891
 892         Value *Nucleus::createGEP(Value *ptr, Type *type, Value *index, bool unsignedIndex)
 893         {
 894                 assert(index->getType() == Ice::IceType_i32);
 895
 896                 if(auto *constant = llvm::dyn_cast<Ice::ConstantInteger32>(index))
 897                 {
 898                         int32_t offset = constant->getValue() * (int)typeSize(type);
 899
 900                         if(offset == 0)
 901                         {
 902                                 return ptr;
 903                         }
 904
 905                         return createAdd(ptr, createConstantInt(offset));
 906                 }
 907
 908                 if(!Ice::isByteSizedType(T(type)))
 909                 {
 910                         index = createMul(index, createConstantInt((int)typeSize(type)));
 911                 }
 912
 913                 if(sizeof(void*) == 8)
 914                 {
 915                         if(unsignedIndex)
 916                         {
 917                                 index = createZExt(index, T(Ice::IceType_i64));
 918                         }
 919                         else
 920                         {
 921                                 index = createSExt(index, T(Ice::IceType_i64));
 922                         }
 923                 }
 924
 925                 return createAdd(ptr, index);
 926         }
 927
 928         Value *Nucleus::createAtomicAdd(Value *ptr, Value *value)
 929         {
 930                 assert(false && "UNIMPLEMENTED"); return nullptr;
 931         }
 932
 933         static Value *createCast(Ice::InstCast::OpKind op, Value *v, Type *destType)
 934         {
 935                 if(v->getType() == T(destType))
 936                 {
 937                         return v;
 938                 }
 939
 940                 Ice::Variable *result = ::function->makeVariable(T(destType));
 941                 Ice::InstCast *cast = Ice::InstCast::create(::function, op, result, v);
 942                 ::basicBlock->appendInst(cast);
 943
 944                 return V(result);
 945         }
 946
 947         Value *Nucleus::createTrunc(Value *v, Type *destType)
 948         {
 949                 return createCast(Ice::InstCast::Trunc, v, destType);
 950         }
 951
 952         Value *Nucleus::createZExt(Value *v, Type *destType)
 953         {
 954                 return createCast(Ice::InstCast::Zext, v, destType);
 955         }
 956
 957         Value *Nucleus::createSExt(Value *v, Type *destType)
 958         {
 959                 return createCast(Ice::InstCast::Sext, v, destType);
 960         }
 961
 962         Value *Nucleus::createFPToSI(Value *v, Type *destType)
 963         {
 964                 return createCast(Ice::InstCast::Fptosi, v, destType);
 965         }
 966
 967         Value *Nucleus::createSIToFP(Value *v, Type *destType)
 968         {
 969                 return createCast(Ice::InstCast::Sitofp, v, destType);
 970         }
 971
 972         Value *Nucleus::createFPTrunc(Value *v, Type *destType)
 973         {
 974                 return createCast(Ice::InstCast::Fptrunc, v, destType);
 975         }
 976
 977         Value *Nucleus::createFPExt(Value *v, Type *destType)
 978         {
 979                 return createCast(Ice::InstCast::Fpext, v, destType);
 980         }
 981
 982         Value *Nucleus::createBitCast(Value *v, Type *destType)
 983         {
 984                 return createCast(Ice::InstCast::Bitcast, v, destType);
 985         }
 986
 987         static Value *createIntCompare(Ice::InstIcmp::ICond condition, Value *lhs, Value *rhs)
 988         {
 989                 assert(lhs->getType() == rhs->getType());
 990
 991                 auto result = ::function->makeVariable(Ice::isScalarIntegerType(lhs->getType()) ? Ice::IceType_i1 : lhs->getType());
 992                 auto cmp = Ice::InstIcmp::create(::function, condition, result, lhs, rhs);
 993                 ::basicBlock->appendInst(cmp);
 994
 995                 return V(result);
 996         }
 997
 998         Value *Nucleus::createICmpEQ(Value *lhs, Value *rhs)
 999         {
1000                 return createIntCompare(Ice::InstIcmp::Eq, lhs, rhs);
1001         }
1002
1003         Value *Nucleus::createICmpNE(Value *lhs, Value *rhs)
1004         {
1005                 return createIntCompare(Ice::InstIcmp::Ne, lhs, rhs);
1006         }
1007
1008         Value *Nucleus::createICmpUGT(Value *lhs, Value *rhs)
1009         {
1010                 return createIntCompare(Ice::InstIcmp::Ugt, lhs, rhs);
1011         }
1012
1013         Value *Nucleus::createICmpUGE(Value *lhs, Value *rhs)
1014         {
1015                 return createIntCompare(Ice::InstIcmp::Uge, lhs, rhs);
1016         }
1017
1018         Value *Nucleus::createICmpULT(Value *lhs, Value *rhs)
1019         {
1020                 return createIntCompare(Ice::InstIcmp::Ult, lhs, rhs);
1021         }
1022
1023         Value *Nucleus::createICmpULE(Value *lhs, Value *rhs)
1024         {
1025                 return createIntCompare(Ice::InstIcmp::Ule, lhs, rhs);
1026         }
1027
1028         Value *Nucleus::createICmpSGT(Value *lhs, Value *rhs)
1029         {
1030                 return createIntCompare(Ice::InstIcmp::Sgt, lhs, rhs);
1031         }
1032
1033         Value *Nucleus::createICmpSGE(Value *lhs, Value *rhs)
1034         {
1035                 return createIntCompare(Ice::InstIcmp::Sge, lhs, rhs);
1036         }
1037
1038         Value *Nucleus::createICmpSLT(Value *lhs, Value *rhs)
1039         {
1040                 return createIntCompare(Ice::InstIcmp::Slt, lhs, rhs);
1041         }
1042
1043         Value *Nucleus::createICmpSLE(Value *lhs, Value *rhs)
1044         {
1045                 return createIntCompare(Ice::InstIcmp::Sle, lhs, rhs);
1046         }
1047
1048         static Value *createFloatCompare(Ice::InstFcmp::FCond condition, Value *lhs, Value *rhs)
1049         {
1050                 assert(lhs->getType() == rhs->getType());
1051                 assert(Ice::isScalarFloatingType(lhs->getType()) || lhs->getType() == Ice::IceType_v4f32);
1052
1053                 auto result = ::function->makeVariable(Ice::isScalarFloatingType(lhs->getType()) ? Ice::IceType_i1 : Ice::IceType_v4i32);
1054                 auto cmp = Ice::InstFcmp::create(::function, condition, result, lhs, rhs);
1055                 ::basicBlock->appendInst(cmp);
1056
1057                 return V(result);
1058         }
1059
1060         Value *Nucleus::createFCmpOEQ(Value *lhs, Value *rhs)
1061         {
1062                 return createFloatCompare(Ice::InstFcmp::Oeq, lhs, rhs);
1063         }
1064
1065         Value *Nucleus::createFCmpOGT(Value *lhs, Value *rhs)
1066         {
1067                 return createFloatCompare(Ice::InstFcmp::Ogt, lhs, rhs);
1068         }
1069
1070         Value *Nucleus::createFCmpOGE(Value *lhs, Value *rhs)
1071         {
1072                 return createFloatCompare(Ice::InstFcmp::Oge, lhs, rhs);
1073         }
1074
1075         Value *Nucleus::createFCmpOLT(Value *lhs, Value *rhs)
1076         {
1077                 return createFloatCompare(Ice::InstFcmp::Olt, lhs, rhs);
1078         }
1079
1080         Value *Nucleus::createFCmpOLE(Value *lhs, Value *rhs)
1081         {
1082                 return createFloatCompare(Ice::InstFcmp::Ole, lhs, rhs);
1083         }
1084
1085         Value *Nucleus::createFCmpONE(Value *lhs, Value *rhs)
1086         {
1087                 return createFloatCompare(Ice::InstFcmp::One, lhs, rhs);
1088         }
1089
1090         Value *Nucleus::createFCmpORD(Value *lhs, Value *rhs)
1091         {
1092                 return createFloatCompare(Ice::InstFcmp::Ord, lhs, rhs);
1093         }
1094
1095         Value *Nucleus::createFCmpUNO(Value *lhs, Value *rhs)
1096         {
1097                 return createFloatCompare(Ice::InstFcmp::Uno, lhs, rhs);
1098         }
1099
1100         Value *Nucleus::createFCmpUEQ(Value *lhs, Value *rhs)
1101         {
1102                 return createFloatCompare(Ice::InstFcmp::Ueq, lhs, rhs);
1103         }
1104
1105         Value *Nucleus::createFCmpUGT(Value *lhs, Value *rhs)
1106         {
1107                 return createFloatCompare(Ice::InstFcmp::Ugt, lhs, rhs);
1108         }
1109
1110         Value *Nucleus::createFCmpUGE(Value *lhs, Value *rhs)
1111         {
1112                 return createFloatCompare(Ice::InstFcmp::Uge, lhs, rhs);
1113         }
1114
1115         Value *Nucleus::createFCmpULT(Value *lhs, Value *rhs)
1116         {
1117                 return createFloatCompare(Ice::InstFcmp::Ult, lhs, rhs);
1118         }
1119
1120         Value *Nucleus::createFCmpULE(Value *lhs, Value *rhs)
1121         {
1122                 return createFloatCompare(Ice::InstFcmp::Ule, lhs, rhs);
1123         }
1124
1125         Value *Nucleus::createFCmpUNE(Value *lhs, Value *rhs)
1126         {
1127                 return createFloatCompare(Ice::InstFcmp::Une, lhs, rhs);
1128         }
1129
1130         Value *Nucleus::createExtractElement(Value *vector, Type *type, int index)
1131         {
1132                 auto result = ::function->makeVariable(T(type));
1133                 auto extract = Ice::InstExtractElement::create(::function, result, vector, ::context->getConstantInt32(index));
1134                 ::basicBlock->appendInst(extract);
1135
1136                 return V(result);
1137         }
1138
1139         Value *Nucleus::createInsertElement(Value *vector, Value *element, int index)
1140         {
1141                 auto result = ::function->makeVariable(vector->getType());
1142                 auto insert = Ice::InstInsertElement::create(::function, result, vector, element, ::context->getConstantInt32(index));
1143                 ::basicBlock->appendInst(insert);
1144
1145                 return V(result);
1146         }
1147
1148         Value *Nucleus::createShuffleVector(Value *V1, Value *V2, const int *select)
1149         {
1150                 assert(V1->getType() == V2->getType());
1151
1152                 int size = Ice::typeNumElements(V1->getType());
1153                 auto result = ::function->makeVariable(V1->getType());
1154                 auto shuffle = Ice::InstShuffleVector::create(::function, result, V1, V2);
1155
1156                 for(int i = 0; i < size; i++)
1157                 {
1158                         shuffle->addIndex(llvm::cast<Ice::ConstantInteger32>(::context->getConstantInt32(select[i])));
1159                 }
1160
1161                 ::basicBlock->appendInst(shuffle);
1162
1163                 return V(result);
1164         }
1165
1166         Value *Nucleus::createSelect(Value *C, Value *ifTrue, Value *ifFalse)
1167         {
1168                 assert(ifTrue->getType() == ifFalse->getType());
1169
1170                 auto result = ::function->makeVariable(ifTrue->getType());
1171                 auto *select = Ice::InstSelect::create(::function, result, C, ifTrue, ifFalse);
1172                 ::basicBlock->appendInst(select);
1173
1174                 return V(result);
1175         }
1176
1177         SwitchCases *Nucleus::createSwitch(Value *control, BasicBlock *defaultBranch, unsigned numCases)
1178         {
1179                 auto switchInst = Ice::InstSwitch::create(::function, numCases, control, defaultBranch);
1180                 ::basicBlock->appendInst(switchInst);
1181
1182                 return reinterpret_cast<SwitchCases*>(switchInst);
1183         }
1184
1185         void Nucleus::addSwitchCase(SwitchCases *switchCases, int label, BasicBlock *branch)
1186         {
1187                 switchCases->addBranch(label, label, branch);
1188         }
1189
1190         void Nucleus::createUnreachable()
1191         {
1192                 Ice::InstUnreachable *unreachable = Ice::InstUnreachable::create(::function);
1193                 ::basicBlock->appendInst(unreachable);
1194         }
1195
1196         static Value *createSwizzle4(Value *val, unsigned char select)
1197         {
1198                 int swizzle[4] =
1199                 {
1200                         (select >> 0) & 0x03,
1201                         (select >> 2) & 0x03,
1202                         (select >> 4) & 0x03,
1203                         (select >> 6) & 0x03,
1204                 };
1205
1206                 return Nucleus::createShuffleVector(val, val, swizzle);
1207         }
1208
1209         static Value *createMask4(Value *lhs, Value *rhs, unsigned char select)
1210         {
1211                 int64_t mask[4] = {0, 0, 0, 0};
1212
1213                 mask[(select >> 0) & 0x03] = -1;
1214                 mask[(select >> 2) & 0x03] = -1;
1215                 mask[(select >> 4) & 0x03] = -1;
1216                 mask[(select >> 6) & 0x03] = -1;
1217
1218                 Value *condition = Nucleus::createConstantVector(mask, T(Ice::IceType_v4i1));
1219                 Value *result = Nucleus::createSelect(condition, rhs, lhs);
1220
1221                 return result;
1222         }
1223
1224         Type *Nucleus::getPointerType(Type *ElementType)
1225         {
1226                 if(sizeof(void*) == 8)
1227                 {
1228                         return T(Ice::IceType_i64);
1229                 }
1230                 else
1231                 {
1232                         return T(Ice::IceType_i32);
1233                 }
1234         }
1235
1236         Value *Nucleus::createNullValue(Type *Ty)
1237         {
1238                 if(Ice::isVectorType(T(Ty)))
1239                 {
1240                         assert(Ice::typeNumElements(T(Ty)) <= 16);
1241                         int64_t c[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
1242                         return createConstantVector(c, Ty);
1243                 }
1244                 else
1245                 {
1246                         return V(::context->getConstantZero(T(Ty)));
1247                 }
1248         }
1249
1250         Value *Nucleus::createConstantLong(int64_t i)
1251         {
1252                 return V(::context->getConstantInt64(i));
1253         }
1254
1255         Value *Nucleus::createConstantInt(int i)
1256         {
1257                 return V(::context->getConstantInt32(i));
1258         }
1259
1260         Value *Nucleus::createConstantInt(unsigned int i)
1261         {
1262                 return V(::context->getConstantInt32(i));
1263         }
1264
1265         Value *Nucleus::createConstantBool(bool b)
1266         {
1267                 return V(::context->getConstantInt1(b));
1268         }
1269
1270         Value *Nucleus::createConstantByte(signed char i)
1271         {
1272                 return V(::context->getConstantInt8(i));
1273         }
1274
1275         Value *Nucleus::createConstantByte(unsigned char i)
1276         {
1277                 return V(::context->getConstantInt8(i));
1278         }
1279
1280         Value *Nucleus::createConstantShort(short i)
1281         {
1282                 return V(::context->getConstantInt16(i));
1283         }
1284
1285         Value *Nucleus::createConstantShort(unsigned short i)
1286         {
1287                 return V(::context->getConstantInt16(i));
1288         }
1289
1290         Value *Nucleus::createConstantFloat(float x)
1291         {
1292                 return V(::context->getConstantFloat(x));
1293         }
1294
1295         Value *Nucleus::createNullPointer(Type *Ty)
1296         {
1297                 return createNullValue(T(sizeof(void*) == 8 ? Ice::IceType_i64 : Ice::IceType_i32));
1298         }
1299
1300         Value *Nucleus::createConstantVector(const int64_t *constants, Type *type)
1301         {
1302                 const int vectorSize = 16;
1303                 assert(Ice::typeWidthInBytes(T(type)) == vectorSize);
1304                 const int alignment = vectorSize;
1305                 auto globalPool = ::function->getGlobalPool();
1306
1307                 const int64_t *i = constants;
1308                 const double *f = reinterpret_cast<const double*>(constants);
1309                 Ice::VariableDeclaration::DataInitializer *dataInitializer = nullptr;
1310
1311                 switch((int)reinterpret_cast<intptr_t>(type))
1312                 {
1313                 case Ice::IceType_v4i32:
1314                 case Ice::IceType_v4i1:
1315                         {
1316                                 const int initializer[4] = {(int)i[0], (int)i[1], (int)i[2], (int)i[3]};
1317                                 static_assert(sizeof(initializer) == vectorSize, "!");
1318                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1319                         }
1320                         break;
1321                 case Ice::IceType_v4f32:
1322                         {
1323                                 const float initializer[4] = {(float)f[0], (float)f[1], (float)f[2], (float)f[3]};
1324                                 static_assert(sizeof(initializer) == vectorSize, "!");
1325                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1326                         }
1327                         break;
1328                 case Ice::IceType_v8i16:
1329                 case Ice::IceType_v8i1:
1330                         {
1331                                 const short initializer[8] = {(short)i[0], (short)i[1], (short)i[2], (short)i[3], (short)i[4], (short)i[5], (short)i[6], (short)i[7]};
1332                                 static_assert(sizeof(initializer) == vectorSize, "!");
1333                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1334                         }
1335                         break;
1336                 case Ice::IceType_v16i8:
1337                 case Ice::IceType_v16i1:
1338                         {
1339                                 const char initializer[16] = {(char)i[0], (char)i[1], (char)i[2], (char)i[3], (char)i[4], (char)i[5], (char)i[6], (char)i[7], (char)i[8], (char)i[9], (char)i[10], (char)i[11], (char)i[12], (char)i[13], (char)i[14], (char)i[15]};
1340                                 static_assert(sizeof(initializer) == vectorSize, "!");
1341                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1342                         }
1343                         break;
1344                 case Type_v2i32:
1345                         {
1346                                 const int initializer[4] = {(int)i[0], (int)i[1], (int)i[0], (int)i[1]};
1347                                 static_assert(sizeof(initializer) == vectorSize, "!");
1348                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1349                         }
1350                         break;
1351                 case Type_v2f32:
1352                         {
1353                                 const float initializer[4] = {(float)f[0], (float)f[1], (float)f[0], (float)f[1]};
1354                                 static_assert(sizeof(initializer) == vectorSize, "!");
1355                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1356                         }
1357                         break;
1358                 case Type_v4i16:
1359                         {
1360                                 const short initializer[8] = {(short)i[0], (short)i[1], (short)i[2], (short)i[3], (short)i[0], (short)i[1], (short)i[2], (short)i[3]};
1361                                 static_assert(sizeof(initializer) == vectorSize, "!");
1362                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1363                         }
1364                         break;
1365                 case Type_v8i8:
1366                         {
1367                                 const char initializer[16] = {(char)i[0], (char)i[1], (char)i[2], (char)i[3], (char)i[4], (char)i[5], (char)i[6], (char)i[7], (char)i[0], (char)i[1], (char)i[2], (char)i[3], (char)i[4], (char)i[5], (char)i[6], (char)i[7]};
1368                                 static_assert(sizeof(initializer) == vectorSize, "!");
1369                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1370                         }
1371                         break;
1372                 case Type_v4i8:
1373                         {
1374                                 const char initializer[16] = {(char)i[0], (char)i[1], (char)i[2], (char)i[3], (char)i[0], (char)i[1], (char)i[2], (char)i[3], (char)i[0], (char)i[1], (char)i[2], (char)i[3], (char)i[0], (char)i[1], (char)i[2], (char)i[3]};
1375                                 static_assert(sizeof(initializer) == vectorSize, "!");
1376                                 dataInitializer = Ice::VariableDeclaration::DataInitializer::create(globalPool, (const char*)initializer, vectorSize);
1377                         }
1378                         break;
1379                 default:
1380                         assert(false && "Unknown constant vector type" && type);
1381                 }
1382
1383                 auto name = Ice::GlobalString::createWithoutString(::context);
1384                 auto *variableDeclaration = Ice::VariableDeclaration::create(globalPool);
1385                 variableDeclaration->setName(name);
1386                 variableDeclaration->setAlignment(alignment);
1387                 variableDeclaration->setIsConstant(true);
1388                 variableDeclaration->addInitializer(dataInitializer);
1389
1390                 ::function->addGlobal(variableDeclaration);
1391
1392                 constexpr int32_t offset = 0;
1393                 Ice::Operand *ptr = ::context->getConstantSym(offset, name);
1394
1395                 Ice::Variable *result = ::function->makeVariable(T(type));
1396                 auto load = Ice::InstLoad::create(::function, result, ptr, alignment);
1397                 ::basicBlock->appendInst(load);
1398
1399                 return V(result);
1400         }
1401
1402         Value *Nucleus::createConstantVector(const double *constants, Type *type)
1403         {
1404                 return createConstantVector((const int64_t*)constants, type);
1405         }
1406
1407         Type *Void::getType()
1408         {
1409                 return T(Ice::IceType_void);
1410         }
1411
1412         Bool::Bool(Argument<Bool> argument)
1413         {
1414                 storeValue(argument.value);
1415         }
1416
1417         Bool::Bool(bool x)
1418         {
1419                 storeValue(Nucleus::createConstantBool(x));
1420         }
1421
1422         Bool::Bool(RValue<Bool> rhs)
1423         {
1424                 storeValue(rhs.value);
1425         }
1426
1427         Bool::Bool(const Bool &rhs)
1428         {
1429                 Value *value = rhs.loadValue();
1430                 storeValue(value);
1431         }
1432
1433         Bool::Bool(const Reference<Bool> &rhs)
1434         {
1435                 Value *value = rhs.loadValue();
1436                 storeValue(value);
1437         }
1438
1439         RValue<Bool> Bool::operator=(RValue<Bool> rhs)
1440         {
1441                 storeValue(rhs.value);
1442
1443                 return rhs;
1444         }
1445
1446         RValue<Bool> Bool::operator=(const Bool &rhs)
1447         {
1448                 Value *value = rhs.loadValue();
1449                 storeValue(value);
1450
1451                 return RValue<Bool>(value);
1452         }
1453
1454         RValue<Bool> Bool::operator=(const Reference<Bool> &rhs)
1455         {
1456                 Value *value = rhs.loadValue();
1457                 storeValue(value);
1458
1459                 return RValue<Bool>(value);
1460         }
1461
1462         RValue<Bool> operator!(RValue<Bool> val)
1463         {
1464                 return RValue<Bool>(Nucleus::createNot(val.value));
1465         }
1466
1467         RValue<Bool> operator&&(RValue<Bool> lhs, RValue<Bool> rhs)
1468         {
1469                 return RValue<Bool>(Nucleus::createAnd(lhs.value, rhs.value));
1470         }
1471
1472         RValue<Bool> operator||(RValue<Bool> lhs, RValue<Bool> rhs)
1473         {
1474                 return RValue<Bool>(Nucleus::createOr(lhs.value, rhs.value));
1475         }
1476
1477         Type *Bool::getType()
1478         {
1479                 return T(Ice::IceType_i1);
1480         }
1481
1482         Byte::Byte(Argument<Byte> argument)
1483         {
1484                 storeValue(argument.value);
1485         }
1486
1487         Byte::Byte(RValue<Int> cast)
1488         {
1489                 Value *integer = Nucleus::createTrunc(cast.value, Byte::getType());
1490
1491                 storeValue(integer);
1492         }
1493
1494         Byte::Byte(RValue<UInt> cast)
1495         {
1496                 Value *integer = Nucleus::createTrunc(cast.value, Byte::getType());
1497
1498                 storeValue(integer);
1499         }
1500
1501         Byte::Byte(RValue<UShort> cast)
1502         {
1503                 Value *integer = Nucleus::createTrunc(cast.value, Byte::getType());
1504
1505                 storeValue(integer);
1506         }
1507
1508         Byte::Byte(int x)
1509         {
1510                 storeValue(Nucleus::createConstantByte((unsigned char)x));
1511         }
1512
1513         Byte::Byte(unsigned char x)
1514         {
1515                 storeValue(Nucleus::createConstantByte(x));
1516         }
1517
1518         Byte::Byte(RValue<Byte> rhs)
1519         {
1520                 storeValue(rhs.value);
1521         }
1522
1523         Byte::Byte(const Byte &rhs)
1524         {
1525                 Value *value = rhs.loadValue();
1526                 storeValue(value);
1527         }
1528
1529         Byte::Byte(const Reference<Byte> &rhs)
1530         {
1531                 Value *value = rhs.loadValue();
1532                 storeValue(value);
1533         }
1534
1535         RValue<Byte> Byte::operator=(RValue<Byte> rhs)
1536         {
1537                 storeValue(rhs.value);
1538
1539                 return rhs;
1540         }
1541
1542         RValue<Byte> Byte::operator=(const Byte &rhs)
1543         {
1544                 Value *value = rhs.loadValue();
1545                 storeValue(value);
1546
1547                 return RValue<Byte>(value);
1548         }
1549
1550         RValue<Byte> Byte::operator=(const Reference<Byte> &rhs)
1551         {
1552                 Value *value = rhs.loadValue();
1553                 storeValue(value);
1554
1555                 return RValue<Byte>(value);
1556         }
1557
1558         RValue<Byte> operator+(RValue<Byte> lhs, RValue<Byte> rhs)
1559         {
1560                 return RValue<Byte>(Nucleus::createAdd(lhs.value, rhs.value));
1561         }
1562
1563         RValue<Byte> operator-(RValue<Byte> lhs, RValue<Byte> rhs)
1564         {
1565                 return RValue<Byte>(Nucleus::createSub(lhs.value, rhs.value));
1566         }
1567
1568         RValue<Byte> operator*(RValue<Byte> lhs, RValue<Byte> rhs)
1569         {
1570                 return RValue<Byte>(Nucleus::createMul(lhs.value, rhs.value));
1571         }
1572
1573         RValue<Byte> operator/(RValue<Byte> lhs, RValue<Byte> rhs)
1574         {
1575                 return RValue<Byte>(Nucleus::createUDiv(lhs.value, rhs.value));
1576         }
1577
1578         RValue<Byte> operator%(RValue<Byte> lhs, RValue<Byte> rhs)
1579         {
1580                 return RValue<Byte>(Nucleus::createURem(lhs.value, rhs.value));
1581         }
1582
1583         RValue<Byte> operator&(RValue<Byte> lhs, RValue<Byte> rhs)
1584         {
1585                 return RValue<Byte>(Nucleus::createAnd(lhs.value, rhs.value));
1586         }
1587
1588         RValue<Byte> operator|(RValue<Byte> lhs, RValue<Byte> rhs)
1589         {
1590                 return RValue<Byte>(Nucleus::createOr(lhs.value, rhs.value));
1591         }
1592
1593         RValue<Byte> operator^(RValue<Byte> lhs, RValue<Byte> rhs)
1594         {
1595                 return RValue<Byte>(Nucleus::createXor(lhs.value, rhs.value));
1596         }
1597
1598         RValue<Byte> operator<<(RValue<Byte> lhs, RValue<Byte> rhs)
1599         {
1600                 return RValue<Byte>(Nucleus::createShl(lhs.value, rhs.value));
1601         }
1602
1603         RValue<Byte> operator>>(RValue<Byte> lhs, RValue<Byte> rhs)
1604         {
1605                 return RValue<Byte>(Nucleus::createLShr(lhs.value, rhs.value));
1606         }
1607
1608         RValue<Byte> operator+=(Byte &lhs, RValue<Byte> rhs)
1609         {
1610                 return lhs = lhs + rhs;
1611         }
1612
1613         RValue<Byte> operator-=(Byte &lhs, RValue<Byte> rhs)
1614         {
1615                 return lhs = lhs - rhs;
1616         }
1617
1618         RValue<Byte> operator*=(Byte &lhs, RValue<Byte> rhs)
1619         {
1620                 return lhs = lhs * rhs;
1621         }
1622
1623         RValue<Byte> operator/=(Byte &lhs, RValue<Byte> rhs)
1624         {
1625                 return lhs = lhs / rhs;
1626         }
1627
1628         RValue<Byte> operator%=(Byte &lhs, RValue<Byte> rhs)
1629         {
1630                 return lhs = lhs % rhs;
1631         }
1632
1633         RValue<Byte> operator&=(Byte &lhs, RValue<Byte> rhs)
1634         {
1635                 return lhs = lhs & rhs;
1636         }
1637
1638         RValue<Byte> operator|=(Byte &lhs, RValue<Byte> rhs)
1639         {
1640                 return lhs = lhs | rhs;
1641         }
1642
1643         RValue<Byte> operator^=(Byte &lhs, RValue<Byte> rhs)
1644         {
1645                 return lhs = lhs ^ rhs;
1646         }
1647
1648         RValue<Byte> operator<<=(Byte &lhs, RValue<Byte> rhs)
1649         {
1650                 return lhs = lhs << rhs;
1651         }
1652
1653         RValue<Byte> operator>>=(Byte &lhs, RValue<Byte> rhs)
1654         {
1655                 return lhs = lhs >> rhs;
1656         }
1657
1658         RValue<Byte> operator+(RValue<Byte> val)
1659         {
1660                 return val;
1661         }
1662
1663         RValue<Byte> operator-(RValue<Byte> val)
1664         {
1665                 return RValue<Byte>(Nucleus::createNeg(val.value));
1666         }
1667
1668         RValue<Byte> operator~(RValue<Byte> val)
1669         {
1670                 return RValue<Byte>(Nucleus::createNot(val.value));
1671         }
1672
1673         RValue<Byte> operator++(Byte &val, int)   // Post-increment
1674         {
1675                 RValue<Byte> res = val;
1676                 val += Byte(1);
1677                 return res;
1678         }
1679
1680         const Byte &operator++(Byte &val)   // Pre-increment
1681         {
1682                 val += Byte(1);
1683                 return val;
1684         }
1685
1686         RValue<Byte> operator--(Byte &val, int)   // Post-decrement
1687         {
1688                 RValue<Byte> res = val;
1689                 val -= Byte(1);
1690                 return res;
1691         }
1692
1693         const Byte &operator--(Byte &val)   // Pre-decrement
1694         {
1695                 val -= Byte(1);
1696                 return val;
1697         }
1698
1699         RValue<Bool> operator<(RValue<Byte> lhs, RValue<Byte> rhs)
1700         {
1701                 return RValue<Bool>(Nucleus::createICmpULT(lhs.value, rhs.value));
1702         }
1703
1704         RValue<Bool> operator<=(RValue<Byte> lhs, RValue<Byte> rhs)
1705         {
1706                 return RValue<Bool>(Nucleus::createICmpULE(lhs.value, rhs.value));
1707         }
1708
1709         RValue<Bool> operator>(RValue<Byte> lhs, RValue<Byte> rhs)
1710         {
1711                 return RValue<Bool>(Nucleus::createICmpUGT(lhs.value, rhs.value));
1712         }
1713
1714         RValue<Bool> operator>=(RValue<Byte> lhs, RValue<Byte> rhs)
1715         {
1716                 return RValue<Bool>(Nucleus::createICmpUGE(lhs.value, rhs.value));
1717         }
1718
1719         RValue<Bool> operator!=(RValue<Byte> lhs, RValue<Byte> rhs)
1720         {
1721                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
1722         }
1723
1724         RValue<Bool> operator==(RValue<Byte> lhs, RValue<Byte> rhs)
1725         {
1726                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
1727         }
1728
1729         Type *Byte::getType()
1730         {
1731                 return T(Ice::IceType_i8);
1732         }
1733
1734         SByte::SByte(Argument<SByte> argument)
1735         {
1736                 storeValue(argument.value);
1737         }
1738
1739         SByte::SByte(RValue<Int> cast)
1740         {
1741                 Value *integer = Nucleus::createTrunc(cast.value, SByte::getType());
1742
1743                 storeValue(integer);
1744         }
1745
1746         SByte::SByte(RValue<Short> cast)
1747         {
1748                 Value *integer = Nucleus::createTrunc(cast.value, SByte::getType());
1749
1750                 storeValue(integer);
1751         }
1752
1753         SByte::SByte(signed char x)
1754         {
1755                 storeValue(Nucleus::createConstantByte(x));
1756         }
1757
1758         SByte::SByte(RValue<SByte> rhs)
1759         {
1760                 storeValue(rhs.value);
1761         }
1762
1763         SByte::SByte(const SByte &rhs)
1764         {
1765                 Value *value = rhs.loadValue();
1766                 storeValue(value);
1767         }
1768
1769         SByte::SByte(const Reference<SByte> &rhs)
1770         {
1771                 Value *value = rhs.loadValue();
1772                 storeValue(value);
1773         }
1774
1775         RValue<SByte> SByte::operator=(RValue<SByte> rhs)
1776         {
1777                 storeValue(rhs.value);
1778
1779                 return rhs;
1780         }
1781
1782         RValue<SByte> SByte::operator=(const SByte &rhs)
1783         {
1784                 Value *value = rhs.loadValue();
1785                 storeValue(value);
1786
1787                 return RValue<SByte>(value);
1788         }
1789
1790         RValue<SByte> SByte::operator=(const Reference<SByte> &rhs)
1791         {
1792                 Value *value = rhs.loadValue();
1793                 storeValue(value);
1794
1795                 return RValue<SByte>(value);
1796         }
1797
1798         RValue<SByte> operator+(RValue<SByte> lhs, RValue<SByte> rhs)
1799         {
1800                 return RValue<SByte>(Nucleus::createAdd(lhs.value, rhs.value));
1801         }
1802
1803         RValue<SByte> operator-(RValue<SByte> lhs, RValue<SByte> rhs)
1804         {
1805                 return RValue<SByte>(Nucleus::createSub(lhs.value, rhs.value));
1806         }
1807
1808         RValue<SByte> operator*(RValue<SByte> lhs, RValue<SByte> rhs)
1809         {
1810                 return RValue<SByte>(Nucleus::createMul(lhs.value, rhs.value));
1811         }
1812
1813         RValue<SByte> operator/(RValue<SByte> lhs, RValue<SByte> rhs)
1814         {
1815                 return RValue<SByte>(Nucleus::createSDiv(lhs.value, rhs.value));
1816         }
1817
1818         RValue<SByte> operator%(RValue<SByte> lhs, RValue<SByte> rhs)
1819         {
1820                 return RValue<SByte>(Nucleus::createSRem(lhs.value, rhs.value));
1821         }
1822
1823         RValue<SByte> operator&(RValue<SByte> lhs, RValue<SByte> rhs)
1824         {
1825                 return RValue<SByte>(Nucleus::createAnd(lhs.value, rhs.value));
1826         }
1827
1828         RValue<SByte> operator|(RValue<SByte> lhs, RValue<SByte> rhs)
1829         {
1830                 return RValue<SByte>(Nucleus::createOr(lhs.value, rhs.value));
1831         }
1832
1833         RValue<SByte> operator^(RValue<SByte> lhs, RValue<SByte> rhs)
1834         {
1835                 return RValue<SByte>(Nucleus::createXor(lhs.value, rhs.value));
1836         }
1837
1838         RValue<SByte> operator<<(RValue<SByte> lhs, RValue<SByte> rhs)
1839         {
1840                 return RValue<SByte>(Nucleus::createShl(lhs.value, rhs.value));
1841         }
1842
1843         RValue<SByte> operator>>(RValue<SByte> lhs, RValue<SByte> rhs)
1844         {
1845                 return RValue<SByte>(Nucleus::createAShr(lhs.value, rhs.value));
1846         }
1847
1848         RValue<SByte> operator+=(SByte &lhs, RValue<SByte> rhs)
1849         {
1850                 return lhs = lhs + rhs;
1851         }
1852
1853         RValue<SByte> operator-=(SByte &lhs, RValue<SByte> rhs)
1854         {
1855                 return lhs = lhs - rhs;
1856         }
1857
1858         RValue<SByte> operator*=(SByte &lhs, RValue<SByte> rhs)
1859         {
1860                 return lhs = lhs * rhs;
1861         }
1862
1863         RValue<SByte> operator/=(SByte &lhs, RValue<SByte> rhs)
1864         {
1865                 return lhs = lhs / rhs;
1866         }
1867
1868         RValue<SByte> operator%=(SByte &lhs, RValue<SByte> rhs)
1869         {
1870                 return lhs = lhs % rhs;
1871         }
1872
1873         RValue<SByte> operator&=(SByte &lhs, RValue<SByte> rhs)
1874         {
1875                 return lhs = lhs & rhs;
1876         }
1877
1878         RValue<SByte> operator|=(SByte &lhs, RValue<SByte> rhs)
1879         {
1880                 return lhs = lhs | rhs;
1881         }
1882
1883         RValue<SByte> operator^=(SByte &lhs, RValue<SByte> rhs)
1884         {
1885                 return lhs = lhs ^ rhs;
1886         }
1887
1888         RValue<SByte> operator<<=(SByte &lhs, RValue<SByte> rhs)
1889         {
1890                 return lhs = lhs << rhs;
1891         }
1892
1893         RValue<SByte> operator>>=(SByte &lhs, RValue<SByte> rhs)
1894         {
1895                 return lhs = lhs >> rhs;
1896         }
1897
1898         RValue<SByte> operator+(RValue<SByte> val)
1899         {
1900                 return val;
1901         }
1902
1903         RValue<SByte> operator-(RValue<SByte> val)
1904         {
1905                 return RValue<SByte>(Nucleus::createNeg(val.value));
1906         }
1907
1908         RValue<SByte> operator~(RValue<SByte> val)
1909         {
1910                 return RValue<SByte>(Nucleus::createNot(val.value));
1911         }
1912
1913         RValue<SByte> operator++(SByte &val, int)   // Post-increment
1914         {
1915                 RValue<SByte> res = val;
1916                 val += SByte(1);
1917                 return res;
1918         }
1919
1920         const SByte &operator++(SByte &val)   // Pre-increment
1921         {
1922                 val += SByte(1);
1923                 return val;
1924         }
1925
1926         RValue<SByte> operator--(SByte &val, int)   // Post-decrement
1927         {
1928                 RValue<SByte> res = val;
1929                 val -= SByte(1);
1930                 return res;
1931         }
1932
1933         const SByte &operator--(SByte &val)   // Pre-decrement
1934         {
1935                 val -= SByte(1);
1936                 return val;
1937         }
1938
1939         RValue<Bool> operator<(RValue<SByte> lhs, RValue<SByte> rhs)
1940         {
1941                 return RValue<Bool>(Nucleus::createICmpSLT(lhs.value, rhs.value));
1942         }
1943
1944         RValue<Bool> operator<=(RValue<SByte> lhs, RValue<SByte> rhs)
1945         {
1946                 return RValue<Bool>(Nucleus::createICmpSLE(lhs.value, rhs.value));
1947         }
1948
1949         RValue<Bool> operator>(RValue<SByte> lhs, RValue<SByte> rhs)
1950         {
1951                 return RValue<Bool>(Nucleus::createICmpSGT(lhs.value, rhs.value));
1952         }
1953
1954         RValue<Bool> operator>=(RValue<SByte> lhs, RValue<SByte> rhs)
1955         {
1956                 return RValue<Bool>(Nucleus::createICmpSGE(lhs.value, rhs.value));
1957         }
1958
1959         RValue<Bool> operator!=(RValue<SByte> lhs, RValue<SByte> rhs)
1960         {
1961                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
1962         }
1963
1964         RValue<Bool> operator==(RValue<SByte> lhs, RValue<SByte> rhs)
1965         {
1966                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
1967         }
1968
1969         Type *SByte::getType()
1970         {
1971                 return T(Ice::IceType_i8);
1972         }
1973
1974         Short::Short(Argument<Short> argument)
1975         {
1976                 storeValue(argument.value);
1977         }
1978
1979         Short::Short(RValue<Int> cast)
1980         {
1981                 Value *integer = Nucleus::createTrunc(cast.value, Short::getType());
1982
1983                 storeValue(integer);
1984         }
1985
1986         Short::Short(short x)
1987         {
1988                 storeValue(Nucleus::createConstantShort(x));
1989         }
1990
1991         Short::Short(RValue<Short> rhs)
1992         {
1993                 storeValue(rhs.value);
1994         }
1995
1996         Short::Short(const Short &rhs)
1997         {
1998                 Value *value = rhs.loadValue();
1999                 storeValue(value);
2000         }
2001
2002         Short::Short(const Reference<Short> &rhs)
2003         {
2004                 Value *value = rhs.loadValue();
2005                 storeValue(value);
2006         }
2007
2008         RValue<Short> Short::operator=(RValue<Short> rhs)
2009         {
2010                 storeValue(rhs.value);
2011
2012                 return rhs;
2013         }
2014
2015         RValue<Short> Short::operator=(const Short &rhs)
2016         {
2017                 Value *value = rhs.loadValue();
2018                 storeValue(value);
2019
2020                 return RValue<Short>(value);
2021         }
2022
2023         RValue<Short> Short::operator=(const Reference<Short> &rhs)
2024         {
2025                 Value *value = rhs.loadValue();
2026                 storeValue(value);
2027
2028                 return RValue<Short>(value);
2029         }
2030
2031         RValue<Short> operator+(RValue<Short> lhs, RValue<Short> rhs)
2032         {
2033                 return RValue<Short>(Nucleus::createAdd(lhs.value, rhs.value));
2034         }
2035
2036         RValue<Short> operator-(RValue<Short> lhs, RValue<Short> rhs)
2037         {
2038                 return RValue<Short>(Nucleus::createSub(lhs.value, rhs.value));
2039         }
2040
2041         RValue<Short> operator*(RValue<Short> lhs, RValue<Short> rhs)
2042         {
2043                 return RValue<Short>(Nucleus::createMul(lhs.value, rhs.value));
2044         }
2045
2046         RValue<Short> operator/(RValue<Short> lhs, RValue<Short> rhs)
2047         {
2048                 return RValue<Short>(Nucleus::createSDiv(lhs.value, rhs.value));
2049         }
2050
2051         RValue<Short> operator%(RValue<Short> lhs, RValue<Short> rhs)
2052         {
2053                 return RValue<Short>(Nucleus::createSRem(lhs.value, rhs.value));
2054         }
2055
2056         RValue<Short> operator&(RValue<Short> lhs, RValue<Short> rhs)
2057         {
2058                 return RValue<Short>(Nucleus::createAnd(lhs.value, rhs.value));
2059         }
2060
2061         RValue<Short> operator|(RValue<Short> lhs, RValue<Short> rhs)
2062         {
2063                 return RValue<Short>(Nucleus::createOr(lhs.value, rhs.value));
2064         }
2065
2066         RValue<Short> operator^(RValue<Short> lhs, RValue<Short> rhs)
2067         {
2068                 return RValue<Short>(Nucleus::createXor(lhs.value, rhs.value));
2069         }
2070
2071         RValue<Short> operator<<(RValue<Short> lhs, RValue<Short> rhs)
2072         {
2073                 return RValue<Short>(Nucleus::createShl(lhs.value, rhs.value));
2074         }
2075
2076         RValue<Short> operator>>(RValue<Short> lhs, RValue<Short> rhs)
2077         {
2078                 return RValue<Short>(Nucleus::createAShr(lhs.value, rhs.value));
2079         }
2080
2081         RValue<Short> operator+=(Short &lhs, RValue<Short> rhs)
2082         {
2083                 return lhs = lhs + rhs;
2084         }
2085
2086         RValue<Short> operator-=(Short &lhs, RValue<Short> rhs)
2087         {
2088                 return lhs = lhs - rhs;
2089         }
2090
2091         RValue<Short> operator*=(Short &lhs, RValue<Short> rhs)
2092         {
2093                 return lhs = lhs * rhs;
2094         }
2095
2096         RValue<Short> operator/=(Short &lhs, RValue<Short> rhs)
2097         {
2098                 return lhs = lhs / rhs;
2099         }
2100
2101         RValue<Short> operator%=(Short &lhs, RValue<Short> rhs)
2102         {
2103                 return lhs = lhs % rhs;
2104         }
2105
2106         RValue<Short> operator&=(Short &lhs, RValue<Short> rhs)
2107         {
2108                 return lhs = lhs & rhs;
2109         }
2110
2111         RValue<Short> operator|=(Short &lhs, RValue<Short> rhs)
2112         {
2113                 return lhs = lhs | rhs;
2114         }
2115
2116         RValue<Short> operator^=(Short &lhs, RValue<Short> rhs)
2117         {
2118                 return lhs = lhs ^ rhs;
2119         }
2120
2121         RValue<Short> operator<<=(Short &lhs, RValue<Short> rhs)
2122         {
2123                 return lhs = lhs << rhs;
2124         }
2125
2126         RValue<Short> operator>>=(Short &lhs, RValue<Short> rhs)
2127         {
2128                 return lhs = lhs >> rhs;
2129         }
2130
2131         RValue<Short> operator+(RValue<Short> val)
2132         {
2133                 return val;
2134         }
2135
2136         RValue<Short> operator-(RValue<Short> val)
2137         {
2138                 return RValue<Short>(Nucleus::createNeg(val.value));
2139         }
2140
2141         RValue<Short> operator~(RValue<Short> val)
2142         {
2143                 return RValue<Short>(Nucleus::createNot(val.value));
2144         }
2145
2146         RValue<Short> operator++(Short &val, int)   // Post-increment
2147         {
2148                 RValue<Short> res = val;
2149                 val += Short(1);
2150                 return res;
2151         }
2152
2153         const Short &operator++(Short &val)   // Pre-increment
2154         {
2155                 val += Short(1);
2156                 return val;
2157         }
2158
2159         RValue<Short> operator--(Short &val, int)   // Post-decrement
2160         {
2161                 RValue<Short> res = val;
2162                 val -= Short(1);
2163                 return res;
2164         }
2165
2166         const Short &operator--(Short &val)   // Pre-decrement
2167         {
2168                 val -= Short(1);
2169                 return val;
2170         }
2171
2172         RValue<Bool> operator<(RValue<Short> lhs, RValue<Short> rhs)
2173         {
2174                 return RValue<Bool>(Nucleus::createICmpSLT(lhs.value, rhs.value));
2175         }
2176
2177         RValue<Bool> operator<=(RValue<Short> lhs, RValue<Short> rhs)
2178         {
2179                 return RValue<Bool>(Nucleus::createICmpSLE(lhs.value, rhs.value));
2180         }
2181
2182         RValue<Bool> operator>(RValue<Short> lhs, RValue<Short> rhs)
2183         {
2184                 return RValue<Bool>(Nucleus::createICmpSGT(lhs.value, rhs.value));
2185         }
2186
2187         RValue<Bool> operator>=(RValue<Short> lhs, RValue<Short> rhs)
2188         {
2189                 return RValue<Bool>(Nucleus::createICmpSGE(lhs.value, rhs.value));
2190         }
2191
2192         RValue<Bool> operator!=(RValue<Short> lhs, RValue<Short> rhs)
2193         {
2194                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
2195         }
2196
2197         RValue<Bool> operator==(RValue<Short> lhs, RValue<Short> rhs)
2198         {
2199                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
2200         }
2201
2202         Type *Short::getType()
2203         {
2204                 return T(Ice::IceType_i16);
2205         }
2206
2207         UShort::UShort(Argument<UShort> argument)
2208         {
2209                 storeValue(argument.value);
2210         }
2211
2212         UShort::UShort(RValue<UInt> cast)
2213         {
2214                 Value *integer = Nucleus::createTrunc(cast.value, UShort::getType());
2215
2216                 storeValue(integer);
2217         }
2218
2219         UShort::UShort(RValue<Int> cast)
2220         {
2221                 Value *integer = Nucleus::createTrunc(cast.value, UShort::getType());
2222
2223                 storeValue(integer);
2224         }
2225
2226         UShort::UShort(unsigned short x)
2227         {
2228                 storeValue(Nucleus::createConstantShort(x));
2229         }
2230
2231         UShort::UShort(RValue<UShort> rhs)
2232         {
2233                 storeValue(rhs.value);
2234         }
2235
2236         UShort::UShort(const UShort &rhs)
2237         {
2238                 Value *value = rhs.loadValue();
2239                 storeValue(value);
2240         }
2241
2242         UShort::UShort(const Reference<UShort> &rhs)
2243         {
2244                 Value *value = rhs.loadValue();
2245                 storeValue(value);
2246         }
2247
2248         RValue<UShort> UShort::operator=(RValue<UShort> rhs)
2249         {
2250                 storeValue(rhs.value);
2251
2252                 return rhs;
2253         }
2254
2255         RValue<UShort> UShort::operator=(const UShort &rhs)
2256         {
2257                 Value *value = rhs.loadValue();
2258                 storeValue(value);
2259
2260                 return RValue<UShort>(value);
2261         }
2262
2263         RValue<UShort> UShort::operator=(const Reference<UShort> &rhs)
2264         {
2265                 Value *value = rhs.loadValue();
2266                 storeValue(value);
2267
2268                 return RValue<UShort>(value);
2269         }
2270
2271         RValue<UShort> operator+(RValue<UShort> lhs, RValue<UShort> rhs)
2272         {
2273                 return RValue<UShort>(Nucleus::createAdd(lhs.value, rhs.value));
2274         }
2275
2276         RValue<UShort> operator-(RValue<UShort> lhs, RValue<UShort> rhs)
2277         {
2278                 return RValue<UShort>(Nucleus::createSub(lhs.value, rhs.value));
2279         }
2280
2281         RValue<UShort> operator*(RValue<UShort> lhs, RValue<UShort> rhs)
2282         {
2283                 return RValue<UShort>(Nucleus::createMul(lhs.value, rhs.value));
2284         }
2285
2286         RValue<UShort> operator/(RValue<UShort> lhs, RValue<UShort> rhs)
2287         {
2288                 return RValue<UShort>(Nucleus::createUDiv(lhs.value, rhs.value));
2289         }
2290
2291         RValue<UShort> operator%(RValue<UShort> lhs, RValue<UShort> rhs)
2292         {
2293                 return RValue<UShort>(Nucleus::createURem(lhs.value, rhs.value));
2294         }
2295
2296         RValue<UShort> operator&(RValue<UShort> lhs, RValue<UShort> rhs)
2297         {
2298                 return RValue<UShort>(Nucleus::createAnd(lhs.value, rhs.value));
2299         }
2300
2301         RValue<UShort> operator|(RValue<UShort> lhs, RValue<UShort> rhs)
2302         {
2303                 return RValue<UShort>(Nucleus::createOr(lhs.value, rhs.value));
2304         }
2305
2306         RValue<UShort> operator^(RValue<UShort> lhs, RValue<UShort> rhs)
2307         {
2308                 return RValue<UShort>(Nucleus::createXor(lhs.value, rhs.value));
2309         }
2310
2311         RValue<UShort> operator<<(RValue<UShort> lhs, RValue<UShort> rhs)
2312         {
2313                 return RValue<UShort>(Nucleus::createShl(lhs.value, rhs.value));
2314         }
2315
2316         RValue<UShort> operator>>(RValue<UShort> lhs, RValue<UShort> rhs)
2317         {
2318                 return RValue<UShort>(Nucleus::createLShr(lhs.value, rhs.value));
2319         }
2320
2321         RValue<UShort> operator+=(UShort &lhs, RValue<UShort> rhs)
2322         {
2323                 return lhs = lhs + rhs;
2324         }
2325
2326         RValue<UShort> operator-=(UShort &lhs, RValue<UShort> rhs)
2327         {
2328                 return lhs = lhs - rhs;
2329         }
2330
2331         RValue<UShort> operator*=(UShort &lhs, RValue<UShort> rhs)
2332         {
2333                 return lhs = lhs * rhs;
2334         }
2335
2336         RValue<UShort> operator/=(UShort &lhs, RValue<UShort> rhs)
2337         {
2338                 return lhs = lhs / rhs;
2339         }
2340
2341         RValue<UShort> operator%=(UShort &lhs, RValue<UShort> rhs)
2342         {
2343                 return lhs = lhs % rhs;
2344         }
2345
2346         RValue<UShort> operator&=(UShort &lhs, RValue<UShort> rhs)
2347         {
2348                 return lhs = lhs & rhs;
2349         }
2350
2351         RValue<UShort> operator|=(UShort &lhs, RValue<UShort> rhs)
2352         {
2353                 return lhs = lhs | rhs;
2354         }
2355
2356         RValue<UShort> operator^=(UShort &lhs, RValue<UShort> rhs)
2357         {
2358                 return lhs = lhs ^ rhs;
2359         }
2360
2361         RValue<UShort> operator<<=(UShort &lhs, RValue<UShort> rhs)
2362         {
2363                 return lhs = lhs << rhs;
2364         }
2365
2366         RValue<UShort> operator>>=(UShort &lhs, RValue<UShort> rhs)
2367         {
2368                 return lhs = lhs >> rhs;
2369         }
2370
2371         RValue<UShort> operator+(RValue<UShort> val)
2372         {
2373                 return val;
2374         }
2375
2376         RValue<UShort> operator-(RValue<UShort> val)
2377         {
2378                 return RValue<UShort>(Nucleus::createNeg(val.value));
2379         }
2380
2381         RValue<UShort> operator~(RValue<UShort> val)
2382         {
2383                 return RValue<UShort>(Nucleus::createNot(val.value));
2384         }
2385
2386         RValue<UShort> operator++(UShort &val, int)   // Post-increment
2387         {
2388                 RValue<UShort> res = val;
2389                 val += UShort(1);
2390                 return res;
2391         }
2392
2393         const UShort &operator++(UShort &val)   // Pre-increment
2394         {
2395                 val += UShort(1);
2396                 return val;
2397         }
2398
2399         RValue<UShort> operator--(UShort &val, int)   // Post-decrement
2400         {
2401                 RValue<UShort> res = val;
2402                 val -= UShort(1);
2403                 return res;
2404         }
2405
2406         const UShort &operator--(UShort &val)   // Pre-decrement
2407         {
2408                 val -= UShort(1);
2409                 return val;
2410         }
2411
2412         RValue<Bool> operator<(RValue<UShort> lhs, RValue<UShort> rhs)
2413         {
2414                 return RValue<Bool>(Nucleus::createICmpULT(lhs.value, rhs.value));
2415         }
2416
2417         RValue<Bool> operator<=(RValue<UShort> lhs, RValue<UShort> rhs)
2418         {
2419                 return RValue<Bool>(Nucleus::createICmpULE(lhs.value, rhs.value));
2420         }
2421
2422         RValue<Bool> operator>(RValue<UShort> lhs, RValue<UShort> rhs)
2423         {
2424                 return RValue<Bool>(Nucleus::createICmpUGT(lhs.value, rhs.value));
2425         }
2426
2427         RValue<Bool> operator>=(RValue<UShort> lhs, RValue<UShort> rhs)
2428         {
2429                 return RValue<Bool>(Nucleus::createICmpUGE(lhs.value, rhs.value));
2430         }
2431
2432         RValue<Bool> operator!=(RValue<UShort> lhs, RValue<UShort> rhs)
2433         {
2434                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
2435         }
2436
2437         RValue<Bool> operator==(RValue<UShort> lhs, RValue<UShort> rhs)
2438         {
2439                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
2440         }
2441
2442         Type *UShort::getType()
2443         {
2444                 return T(Ice::IceType_i16);
2445         }
2446
2447         Byte4::Byte4(RValue<Byte8> cast)
2448         {
2449                 storeValue(Nucleus::createBitCast(cast.value, getType()));
2450         }
2451
2452         Byte4::Byte4(const Reference<Byte4> &rhs)
2453         {
2454                 Value *value = rhs.loadValue();
2455                 storeValue(value);
2456         }
2457
2458         Type *Byte4::getType()
2459         {
2460                 return T(Type_v4i8);
2461         }
2462
2463         Type *SByte4::getType()
2464         {
2465                 return T(Type_v4i8);
2466         }
2467
2468         Byte8::Byte8(uint8_t x0, uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, uint8_t x5, uint8_t x6, uint8_t x7)
2469         {
2470                 int64_t constantVector[8] = {x0, x1, x2, x3, x4, x5, x6, x7};
2471                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
2472         }
2473
2474         Byte8::Byte8(RValue<Byte8> rhs)
2475         {
2476                 storeValue(rhs.value);
2477         }
2478
2479         Byte8::Byte8(const Byte8 &rhs)
2480         {
2481                 Value *value = rhs.loadValue();
2482                 storeValue(value);
2483         }
2484
2485         Byte8::Byte8(const Reference<Byte8> &rhs)
2486         {
2487                 Value *value = rhs.loadValue();
2488                 storeValue(value);
2489         }
2490
2491         RValue<Byte8> Byte8::operator=(RValue<Byte8> rhs)
2492         {
2493                 storeValue(rhs.value);
2494
2495                 return rhs;
2496         }
2497
2498         RValue<Byte8> Byte8::operator=(const Byte8 &rhs)
2499         {
2500                 Value *value = rhs.loadValue();
2501                 storeValue(value);
2502
2503                 return RValue<Byte8>(value);
2504         }
2505
2506         RValue<Byte8> Byte8::operator=(const Reference<Byte8> &rhs)
2507         {
2508                 Value *value = rhs.loadValue();
2509                 storeValue(value);
2510
2511                 return RValue<Byte8>(value);
2512         }
2513
2514         RValue<Byte8> operator+(RValue<Byte8> lhs, RValue<Byte8> rhs)
2515         {
2516                 return RValue<Byte8>(Nucleus::createAdd(lhs.value, rhs.value));
2517         }
2518
2519         RValue<Byte8> operator-(RValue<Byte8> lhs, RValue<Byte8> rhs)
2520         {
2521                 return RValue<Byte8>(Nucleus::createSub(lhs.value, rhs.value));
2522         }
2523
2524 //      RValue<Byte8> operator*(RValue<Byte8> lhs, RValue<Byte8> rhs)
2525 //      {
2526 //              return RValue<Byte8>(Nucleus::createMul(lhs.value, rhs.value));
2527 //      }
2528
2529 //      RValue<Byte8> operator/(RValue<Byte8> lhs, RValue<Byte8> rhs)
2530 //      {
2531 //              return RValue<Byte8>(Nucleus::createUDiv(lhs.value, rhs.value));
2532 //      }
2533
2534 //      RValue<Byte8> operator%(RValue<Byte8> lhs, RValue<Byte8> rhs)
2535 //      {
2536 //              return RValue<Byte8>(Nucleus::createURem(lhs.value, rhs.value));
2537 //      }
2538
2539         RValue<Byte8> operator&(RValue<Byte8> lhs, RValue<Byte8> rhs)
2540         {
2541                 return RValue<Byte8>(Nucleus::createAnd(lhs.value, rhs.value));
2542         }
2543
2544         RValue<Byte8> operator|(RValue<Byte8> lhs, RValue<Byte8> rhs)
2545         {
2546                 return RValue<Byte8>(Nucleus::createOr(lhs.value, rhs.value));
2547         }
2548
2549         RValue<Byte8> operator^(RValue<Byte8> lhs, RValue<Byte8> rhs)
2550         {
2551                 return RValue<Byte8>(Nucleus::createXor(lhs.value, rhs.value));
2552         }
2553
2554 //      RValue<Byte8> operator<<(RValue<Byte8> lhs, unsigned char rhs)
2555 //      {
2556 //              return RValue<Byte8>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
2557 //      }
2558
2559 //      RValue<Byte8> operator>>(RValue<Byte8> lhs, unsigned char rhs)
2560 //      {
2561 //              return RValue<Byte8>(Nucleus::createLShr(lhs.value, V(::context->getConstantInt32(rhs))));
2562 //      }
2563
2564         RValue<Byte8> operator+=(Byte8 &lhs, RValue<Byte8> rhs)
2565         {
2566                 return lhs = lhs + rhs;
2567         }
2568
2569         RValue<Byte8> operator-=(Byte8 &lhs, RValue<Byte8> rhs)
2570         {
2571                 return lhs = lhs - rhs;
2572         }
2573
2574 //      RValue<Byte8> operator*=(Byte8 &lhs, RValue<Byte8> rhs)
2575 //      {
2576 //              return lhs = lhs * rhs;
2577 //      }
2578
2579 //      RValue<Byte8> operator/=(Byte8 &lhs, RValue<Byte8> rhs)
2580 //      {
2581 //              return lhs = lhs / rhs;
2582 //      }
2583
2584 //      RValue<Byte8> operator%=(Byte8 &lhs, RValue<Byte8> rhs)
2585 //      {
2586 //              return lhs = lhs % rhs;
2587 //      }
2588
2589         RValue<Byte8> operator&=(Byte8 &lhs, RValue<Byte8> rhs)
2590         {
2591                 return lhs = lhs & rhs;
2592         }
2593
2594         RValue<Byte8> operator|=(Byte8 &lhs, RValue<Byte8> rhs)
2595         {
2596                 return lhs = lhs | rhs;
2597         }
2598
2599         RValue<Byte8> operator^=(Byte8 &lhs, RValue<Byte8> rhs)
2600         {
2601                 return lhs = lhs ^ rhs;
2602         }
2603
2604 //      RValue<Byte8> operator<<=(Byte8 &lhs, RValue<Byte8> rhs)
2605 //      {
2606 //              return lhs = lhs << rhs;
2607 //      }
2608
2609 //      RValue<Byte8> operator>>=(Byte8 &lhs, RValue<Byte8> rhs)
2610 //      {
2611 //              return lhs = lhs >> rhs;
2612 //      }
2613
2614 //      RValue<Byte8> operator+(RValue<Byte8> val)
2615 //      {
2616 //              return val;
2617 //      }
2618
2619 //      RValue<Byte8> operator-(RValue<Byte8> val)
2620 //      {
2621 //              return RValue<Byte8>(Nucleus::createNeg(val.value));
2622 //      }
2623
2624         RValue<Byte8> operator~(RValue<Byte8> val)
2625         {
2626                 return RValue<Byte8>(Nucleus::createNot(val.value));
2627         }
2628
2629         RValue<Byte8> AddSat(RValue<Byte8> x, RValue<Byte8> y)
2630         {
2631                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
2632                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::AddSaturateUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
2633                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
2634                 auto paddusb = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
2635                 paddusb->addArg(x.value);
2636                 paddusb->addArg(y.value);
2637                 ::basicBlock->appendInst(paddusb);
2638
2639                 return RValue<Byte8>(V(result));
2640         }
2641
2642         RValue<Byte8> SubSat(RValue<Byte8> x, RValue<Byte8> y)
2643         {
2644                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
2645                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SubtractSaturateUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
2646                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
2647                 auto psubusw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
2648                 psubusw->addArg(x.value);
2649                 psubusw->addArg(y.value);
2650                 ::basicBlock->appendInst(psubusw);
2651
2652                 return RValue<Byte8>(V(result));
2653         }
2654
2655         RValue<Short4> Unpack(RValue<Byte4> x)
2656         {
2657                 int shuffle[16] = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7};   // Real type is v16i8
2658                 return As<Short4>(Nucleus::createShuffleVector(x.value, x.value, shuffle));
2659         }
2660
2661         RValue<Short4> Unpack(RValue<Byte4> x, RValue<Byte4> y)
2662         {
2663                 return UnpackLow(As<Byte8>(x), As<Byte8>(y));
2664         }
2665
2666         RValue<Short4> UnpackLow(RValue<Byte8> x, RValue<Byte8> y)
2667         {
2668                 int shuffle[16] = {0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23};   // Real type is v16i8
2669                 return As<Short4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
2670         }
2671
2672         RValue<Short4> UnpackHigh(RValue<Byte8> x, RValue<Byte8> y)
2673         {
2674                 int shuffle[16] = {0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23};   // Real type is v16i8
2675                 auto lowHigh = RValue<Byte16>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
2676                 return As<Short4>(Swizzle(As<Int4>(lowHigh), 0xEE));
2677         }
2678
2679         RValue<Int> SignMask(RValue<Byte8> x)
2680         {
2681                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_i32);
2682                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SignMask, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
2683                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
2684                 auto movmsk = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
2685                 movmsk->addArg(x.value);
2686                 ::basicBlock->appendInst(movmsk);
2687
2688                 return RValue<Int>(V(result));
2689         }
2690
2691 //      RValue<Byte8> CmpGT(RValue<Byte8> x, RValue<Byte8> y)
2692 //      {
2693 //              return RValue<Byte8>(createIntCompare(Ice::InstIcmp::Ugt, x.value, y.value));
2694 //      }
2695
2696         RValue<Byte8> CmpEQ(RValue<Byte8> x, RValue<Byte8> y)
2697         {
2698                 return RValue<Byte8>(Nucleus::createICmpEQ(x.value, y.value));
2699         }
2700
2701         Type *Byte8::getType()
2702         {
2703                 return T(Type_v8i8);
2704         }
2705
2706         SByte8::SByte8(uint8_t x0, uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, uint8_t x5, uint8_t x6, uint8_t x7)
2707         {
2708                 int64_t constantVector[8] = { x0, x1, x2, x3, x4, x5, x6, x7 };
2709                 Value *vector = V(Nucleus::createConstantVector(constantVector, getType()));
2710
2711                 storeValue(Nucleus::createBitCast(vector, getType()));
2712         }
2713
2714         SByte8::SByte8(RValue<SByte8> rhs)
2715         {
2716                 storeValue(rhs.value);
2717         }
2718
2719         SByte8::SByte8(const SByte8 &rhs)
2720         {
2721                 Value *value = rhs.loadValue();
2722                 storeValue(value);
2723         }
2724
2725         SByte8::SByte8(const Reference<SByte8> &rhs)
2726         {
2727                 Value *value = rhs.loadValue();
2728                 storeValue(value);
2729         }
2730
2731         RValue<SByte8> SByte8::operator=(RValue<SByte8> rhs)
2732         {
2733                 storeValue(rhs.value);
2734
2735                 return rhs;
2736         }
2737
2738         RValue<SByte8> SByte8::operator=(const SByte8 &rhs)
2739         {
2740                 Value *value = rhs.loadValue();
2741                 storeValue(value);
2742
2743                 return RValue<SByte8>(value);
2744         }
2745
2746         RValue<SByte8> SByte8::operator=(const Reference<SByte8> &rhs)
2747         {
2748                 Value *value = rhs.loadValue();
2749                 storeValue(value);
2750
2751                 return RValue<SByte8>(value);
2752         }
2753
2754         RValue<SByte8> operator+(RValue<SByte8> lhs, RValue<SByte8> rhs)
2755         {
2756                 return RValue<SByte8>(Nucleus::createAdd(lhs.value, rhs.value));
2757         }
2758
2759         RValue<SByte8> operator-(RValue<SByte8> lhs, RValue<SByte8> rhs)
2760         {
2761                 return RValue<SByte8>(Nucleus::createSub(lhs.value, rhs.value));
2762         }
2763
2764 //      RValue<SByte8> operator*(RValue<SByte8> lhs, RValue<SByte8> rhs)
2765 //      {
2766 //              return RValue<SByte8>(Nucleus::createMul(lhs.value, rhs.value));
2767 //      }
2768
2769 //      RValue<SByte8> operator/(RValue<SByte8> lhs, RValue<SByte8> rhs)
2770 //      {
2771 //              return RValue<SByte8>(Nucleus::createSDiv(lhs.value, rhs.value));
2772 //      }
2773
2774 //      RValue<SByte8> operator%(RValue<SByte8> lhs, RValue<SByte8> rhs)
2775 //      {
2776 //              return RValue<SByte8>(Nucleus::createSRem(lhs.value, rhs.value));
2777 //      }
2778
2779         RValue<SByte8> operator&(RValue<SByte8> lhs, RValue<SByte8> rhs)
2780         {
2781                 return RValue<SByte8>(Nucleus::createAnd(lhs.value, rhs.value));
2782         }
2783
2784         RValue<SByte8> operator|(RValue<SByte8> lhs, RValue<SByte8> rhs)
2785         {
2786                 return RValue<SByte8>(Nucleus::createOr(lhs.value, rhs.value));
2787         }
2788
2789         RValue<SByte8> operator^(RValue<SByte8> lhs, RValue<SByte8> rhs)
2790         {
2791                 return RValue<SByte8>(Nucleus::createXor(lhs.value, rhs.value));
2792         }
2793
2794 //      RValue<SByte8> operator<<(RValue<SByte8> lhs, unsigned char rhs)
2795 //      {
2796 //              return RValue<SByte8>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
2797 //      }
2798
2799 //      RValue<SByte8> operator>>(RValue<SByte8> lhs, unsigned char rhs)
2800 //      {
2801 //              return RValue<SByte8>(Nucleus::createAShr(lhs.value, V(::context->getConstantInt32(rhs))));
2802 //      }
2803
2804         RValue<SByte8> operator+=(SByte8 &lhs, RValue<SByte8> rhs)
2805         {
2806                 return lhs = lhs + rhs;
2807         }
2808
2809         RValue<SByte8> operator-=(SByte8 &lhs, RValue<SByte8> rhs)
2810         {
2811                 return lhs = lhs - rhs;
2812         }
2813
2814 //      RValue<SByte8> operator*=(SByte8 &lhs, RValue<SByte8> rhs)
2815 //      {
2816 //              return lhs = lhs * rhs;
2817 //      }
2818
2819 //      RValue<SByte8> operator/=(SByte8 &lhs, RValue<SByte8> rhs)
2820 //      {
2821 //              return lhs = lhs / rhs;
2822 //      }
2823
2824 //      RValue<SByte8> operator%=(SByte8 &lhs, RValue<SByte8> rhs)
2825 //      {
2826 //              return lhs = lhs % rhs;
2827 //      }
2828
2829         RValue<SByte8> operator&=(SByte8 &lhs, RValue<SByte8> rhs)
2830         {
2831                 return lhs = lhs & rhs;
2832         }
2833
2834         RValue<SByte8> operator|=(SByte8 &lhs, RValue<SByte8> rhs)
2835         {
2836                 return lhs = lhs | rhs;
2837         }
2838
2839         RValue<SByte8> operator^=(SByte8 &lhs, RValue<SByte8> rhs)
2840         {
2841                 return lhs = lhs ^ rhs;
2842         }
2843
2844 //      RValue<SByte8> operator<<=(SByte8 &lhs, RValue<SByte8> rhs)
2845 //      {
2846 //              return lhs = lhs << rhs;
2847 //      }
2848
2849 //      RValue<SByte8> operator>>=(SByte8 &lhs, RValue<SByte8> rhs)
2850 //      {
2851 //              return lhs = lhs >> rhs;
2852 //      }
2853
2854 //      RValue<SByte8> operator+(RValue<SByte8> val)
2855 //      {
2856 //              return val;
2857 //      }
2858
2859 //      RValue<SByte8> operator-(RValue<SByte8> val)
2860 //      {
2861 //              return RValue<SByte8>(Nucleus::createNeg(val.value));
2862 //      }
2863
2864         RValue<SByte8> operator~(RValue<SByte8> val)
2865         {
2866                 return RValue<SByte8>(Nucleus::createNot(val.value));
2867         }
2868
2869         RValue<SByte8> AddSat(RValue<SByte8> x, RValue<SByte8> y)
2870         {
2871                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
2872                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::AddSaturateSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
2873                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
2874                 auto paddsb = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
2875                 paddsb->addArg(x.value);
2876                 paddsb->addArg(y.value);
2877                 ::basicBlock->appendInst(paddsb);
2878
2879                 return RValue<SByte8>(V(result));
2880         }
2881
2882         RValue<SByte8> SubSat(RValue<SByte8> x, RValue<SByte8> y)
2883         {
2884                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
2885                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SubtractSaturateSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
2886                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
2887                 auto psubsb = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
2888                 psubsb->addArg(x.value);
2889                 psubsb->addArg(y.value);
2890                 ::basicBlock->appendInst(psubsb);
2891
2892                 return RValue<SByte8>(V(result));
2893         }
2894
2895         RValue<Short4> UnpackLow(RValue<SByte8> x, RValue<SByte8> y)
2896         {
2897                 int shuffle[16] = {0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23};   // Real type is v16i8
2898                 return As<Short4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
2899         }
2900
2901         RValue<Short4> UnpackHigh(RValue<SByte8> x, RValue<SByte8> y)
2902         {
2903                 int shuffle[16] = {0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23};   // Real type is v16i8
2904                 auto lowHigh = RValue<Byte16>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
2905                 return As<Short4>(Swizzle(As<Int4>(lowHigh), 0xEE));
2906         }
2907
2908         RValue<Int> SignMask(RValue<SByte8> x)
2909         {
2910                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_i32);
2911                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SignMask, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
2912                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
2913                 auto movmsk = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
2914                 movmsk->addArg(x.value);
2915                 ::basicBlock->appendInst(movmsk);
2916
2917                 return RValue<Int>(V(result));
2918         }
2919
2920         RValue<Byte8> CmpGT(RValue<SByte8> x, RValue<SByte8> y)
2921         {
2922                 return RValue<Byte8>(createIntCompare(Ice::InstIcmp::Sgt, x.value, y.value));
2923         }
2924
2925         RValue<Byte8> CmpEQ(RValue<SByte8> x, RValue<SByte8> y)
2926         {
2927                 return RValue<Byte8>(Nucleus::createICmpEQ(x.value, y.value));
2928         }
2929
2930         Type *SByte8::getType()
2931         {
2932                 return T(Type_v8i8);
2933         }
2934
2935         Byte16::Byte16(RValue<Byte16> rhs)
2936         {
2937                 storeValue(rhs.value);
2938         }
2939
2940         Byte16::Byte16(const Byte16 &rhs)
2941         {
2942                 Value *value = rhs.loadValue();
2943                 storeValue(value);
2944         }
2945
2946         Byte16::Byte16(const Reference<Byte16> &rhs)
2947         {
2948                 Value *value = rhs.loadValue();
2949                 storeValue(value);
2950         }
2951
2952         RValue<Byte16> Byte16::operator=(RValue<Byte16> rhs)
2953         {
2954                 storeValue(rhs.value);
2955
2956                 return rhs;
2957         }
2958
2959         RValue<Byte16> Byte16::operator=(const Byte16 &rhs)
2960         {
2961                 Value *value = rhs.loadValue();
2962                 storeValue(value);
2963
2964                 return RValue<Byte16>(value);
2965         }
2966
2967         RValue<Byte16> Byte16::operator=(const Reference<Byte16> &rhs)
2968         {
2969                 Value *value = rhs.loadValue();
2970                 storeValue(value);
2971
2972                 return RValue<Byte16>(value);
2973         }
2974
2975         Type *Byte16::getType()
2976         {
2977                 return T(Ice::IceType_v16i8);
2978         }
2979
2980         Type *SByte16::getType()
2981         {
2982                 return T(Ice::IceType_v16i8);
2983         }
2984
2985         Short2::Short2(RValue<Short4> cast)
2986         {
2987                 storeValue(Nucleus::createBitCast(cast.value, getType()));
2988         }
2989
2990         Type *Short2::getType()
2991         {
2992                 return T(Type_v2i16);
2993         }
2994
2995         UShort2::UShort2(RValue<UShort4> cast)
2996         {
2997                 storeValue(Nucleus::createBitCast(cast.value, getType()));
2998         }
2999
3000         Type *UShort2::getType()
3001         {
3002                 return T(Type_v2i16);
3003         }
3004
3005         Short4::Short4(RValue<Int> cast)
3006         {
3007                 Value *vector = loadValue();
3008                 Value *element = Nucleus::createTrunc(cast.value, Short::getType());
3009                 Value *insert = Nucleus::createInsertElement(vector, element, 0);
3010                 Value *swizzle = Swizzle(RValue<Short4>(insert), 0x00).value;
3011
3012                 storeValue(swizzle);
3013         }
3014
3015         Short4::Short4(RValue<Int4> cast)
3016         {
3017                 int select[8] = {0, 2, 4, 6, 0, 2, 4, 6};
3018                 Value *short8 = Nucleus::createBitCast(cast.value, Short8::getType());
3019                 Value *packed = Nucleus::createShuffleVector(short8, short8, select);
3020
3021                 Value *int2 = RValue<Int2>(Int2(As<Int4>(packed))).value;
3022                 Value *short4 = Nucleus::createBitCast(int2, Short4::getType());
3023
3024                 storeValue(short4);
3025         }
3026
3027 //      Short4::Short4(RValue<Float> cast)
3028 //      {
3029 //      }
3030
3031         Short4::Short4(RValue<Float4> cast)
3032         {
3033                 assert(false && "UNIMPLEMENTED");
3034         }
3035
3036         Short4::Short4(short xyzw)
3037         {
3038                 int64_t constantVector[4] = {xyzw, xyzw, xyzw, xyzw};
3039                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
3040         }
3041
3042         Short4::Short4(short x, short y, short z, short w)
3043         {
3044                 int64_t constantVector[4] = {x, y, z, w};
3045                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
3046         }
3047
3048         Short4::Short4(RValue<Short4> rhs)
3049         {
3050                 storeValue(rhs.value);
3051         }
3052
3053         Short4::Short4(const Short4 &rhs)
3054         {
3055                 Value *value = rhs.loadValue();
3056                 storeValue(value);
3057         }
3058
3059         Short4::Short4(const Reference<Short4> &rhs)
3060         {
3061                 Value *value = rhs.loadValue();
3062                 storeValue(value);
3063         }
3064
3065         Short4::Short4(RValue<UShort4> rhs)
3066         {
3067                 storeValue(rhs.value);
3068         }
3069
3070         Short4::Short4(const UShort4 &rhs)
3071         {
3072                 storeValue(rhs.loadValue());
3073         }
3074
3075         Short4::Short4(const Reference<UShort4> &rhs)
3076         {
3077                 storeValue(rhs.loadValue());
3078         }
3079
3080         RValue<Short4> Short4::operator=(RValue<Short4> rhs)
3081         {
3082                 storeValue(rhs.value);
3083
3084                 return rhs;
3085         }
3086
3087         RValue<Short4> Short4::operator=(const Short4 &rhs)
3088         {
3089                 Value *value = rhs.loadValue();
3090                 storeValue(value);
3091
3092                 return RValue<Short4>(value);
3093         }
3094
3095         RValue<Short4> Short4::operator=(const Reference<Short4> &rhs)
3096         {
3097                 Value *value = rhs.loadValue();
3098                 storeValue(value);
3099
3100                 return RValue<Short4>(value);
3101         }
3102
3103         RValue<Short4> Short4::operator=(RValue<UShort4> rhs)
3104         {
3105                 storeValue(rhs.value);
3106
3107                 return RValue<Short4>(rhs);
3108         }
3109
3110         RValue<Short4> Short4::operator=(const UShort4 &rhs)
3111         {
3112                 Value *value = rhs.loadValue();
3113                 storeValue(value);
3114
3115                 return RValue<Short4>(value);
3116         }
3117
3118         RValue<Short4> Short4::operator=(const Reference<UShort4> &rhs)
3119         {
3120                 Value *value = rhs.loadValue();
3121                 storeValue(value);
3122
3123                 return RValue<Short4>(value);
3124         }
3125
3126         RValue<Short4> operator+(RValue<Short4> lhs, RValue<Short4> rhs)
3127         {
3128                 return RValue<Short4>(Nucleus::createAdd(lhs.value, rhs.value));
3129         }
3130
3131         RValue<Short4> operator-(RValue<Short4> lhs, RValue<Short4> rhs)
3132         {
3133                 return RValue<Short4>(Nucleus::createSub(lhs.value, rhs.value));
3134         }
3135
3136         RValue<Short4> operator*(RValue<Short4> lhs, RValue<Short4> rhs)
3137         {
3138                 return RValue<Short4>(Nucleus::createMul(lhs.value, rhs.value));
3139         }
3140
3141 //      RValue<Short4> operator/(RValue<Short4> lhs, RValue<Short4> rhs)
3142 //      {
3143 //              return RValue<Short4>(Nucleus::createSDiv(lhs.value, rhs.value));
3144 //      }
3145
3146 //      RValue<Short4> operator%(RValue<Short4> lhs, RValue<Short4> rhs)
3147 //      {
3148 //              return RValue<Short4>(Nucleus::createSRem(lhs.value, rhs.value));
3149 //      }
3150
3151         RValue<Short4> operator&(RValue<Short4> lhs, RValue<Short4> rhs)
3152         {
3153                 return RValue<Short4>(Nucleus::createAnd(lhs.value, rhs.value));
3154         }
3155
3156         RValue<Short4> operator|(RValue<Short4> lhs, RValue<Short4> rhs)
3157         {
3158                 return RValue<Short4>(Nucleus::createOr(lhs.value, rhs.value));
3159         }
3160
3161         RValue<Short4> operator^(RValue<Short4> lhs, RValue<Short4> rhs)
3162         {
3163                 return RValue<Short4>(Nucleus::createXor(lhs.value, rhs.value));
3164         }
3165
3166         RValue<Short4> operator<<(RValue<Short4> lhs, unsigned char rhs)
3167         {
3168                 return RValue<Short4>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
3169         }
3170
3171         RValue<Short4> operator>>(RValue<Short4> lhs, unsigned char rhs)
3172         {
3173                 return RValue<Short4>(Nucleus::createAShr(lhs.value, V(::context->getConstantInt32(rhs))));
3174         }
3175
3176         RValue<Short4> operator+=(Short4 &lhs, RValue<Short4> rhs)
3177         {
3178                 return lhs = lhs + rhs;
3179         }
3180
3181         RValue<Short4> operator-=(Short4 &lhs, RValue<Short4> rhs)
3182         {
3183                 return lhs = lhs - rhs;
3184         }
3185
3186         RValue<Short4> operator*=(Short4 &lhs, RValue<Short4> rhs)
3187         {
3188                 return lhs = lhs * rhs;
3189         }
3190
3191 //      RValue<Short4> operator/=(Short4 &lhs, RValue<Short4> rhs)
3192 //      {
3193 //              return lhs = lhs / rhs;
3194 //      }
3195
3196 //      RValue<Short4> operator%=(Short4 &lhs, RValue<Short4> rhs)
3197 //      {
3198 //              return lhs = lhs % rhs;
3199 //      }
3200
3201         RValue<Short4> operator&=(Short4 &lhs, RValue<Short4> rhs)
3202         {
3203                 return lhs = lhs & rhs;
3204         }
3205
3206         RValue<Short4> operator|=(Short4 &lhs, RValue<Short4> rhs)
3207         {
3208                 return lhs = lhs | rhs;
3209         }
3210
3211         RValue<Short4> operator^=(Short4 &lhs, RValue<Short4> rhs)
3212         {
3213                 return lhs = lhs ^ rhs;
3214         }
3215
3216         RValue<Short4> operator<<=(Short4 &lhs, unsigned char rhs)
3217         {
3218                 return lhs = lhs << rhs;
3219         }
3220
3221         RValue<Short4> operator>>=(Short4 &lhs, unsigned char rhs)
3222         {
3223                 return lhs = lhs >> rhs;
3224         }
3225
3226 //      RValue<Short4> operator+(RValue<Short4> val)
3227 //      {
3228 //              return val;
3229 //      }
3230
3231         RValue<Short4> operator-(RValue<Short4> val)
3232         {
3233                 return RValue<Short4>(Nucleus::createNeg(val.value));
3234         }
3235
3236         RValue<Short4> operator~(RValue<Short4> val)
3237         {
3238                 return RValue<Short4>(Nucleus::createNot(val.value));
3239         }
3240
3241         RValue<Short4> RoundShort4(RValue<Float4> cast)
3242         {
3243                 RValue<Int4> int4 = RoundInt(cast);
3244                 return As<Short4>(Pack(int4, int4));
3245         }
3246
3247         RValue<Short4> Max(RValue<Short4> x, RValue<Short4> y)
3248         {
3249                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v8i1);
3250                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Sle, condition, x.value, y.value);
3251                 ::basicBlock->appendInst(cmp);
3252
3253                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3254                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
3255                 ::basicBlock->appendInst(select);
3256
3257                 return RValue<Short4>(V(result));
3258         }
3259
3260         RValue<Short4> Min(RValue<Short4> x, RValue<Short4> y)
3261         {
3262                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v8i1);
3263                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Sgt, condition, x.value, y.value);
3264                 ::basicBlock->appendInst(cmp);
3265
3266                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3267                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
3268                 ::basicBlock->appendInst(select);
3269
3270                 return RValue<Short4>(V(result));
3271         }
3272
3273         RValue<Short4> AddSat(RValue<Short4> x, RValue<Short4> y)
3274         {
3275                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3276                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::AddSaturateSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3277                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
3278                 auto paddsw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3279                 paddsw->addArg(x.value);
3280                 paddsw->addArg(y.value);
3281                 ::basicBlock->appendInst(paddsw);
3282
3283                 return RValue<Short4>(V(result));
3284         }
3285
3286         RValue<Short4> SubSat(RValue<Short4> x, RValue<Short4> y)
3287         {
3288                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3289                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SubtractSaturateSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3290                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
3291                 auto psubsw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3292                 psubsw->addArg(x.value);
3293                 psubsw->addArg(y.value);
3294                 ::basicBlock->appendInst(psubsw);
3295
3296                 return RValue<Short4>(V(result));
3297         }
3298
3299         RValue<Short4> MulHigh(RValue<Short4> x, RValue<Short4> y)
3300         {
3301                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3302                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::MultiplyHighSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3303                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
3304                 auto pmulhw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3305                 pmulhw->addArg(x.value);
3306                 pmulhw->addArg(y.value);
3307                 ::basicBlock->appendInst(pmulhw);
3308
3309                 return RValue<Short4>(V(result));
3310         }
3311
3312         RValue<Int2> MulAdd(RValue<Short4> x, RValue<Short4> y)
3313         {
3314                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3315                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::MultiplyAddPairs, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3316                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
3317                 auto pmaddwd = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3318                 pmaddwd->addArg(x.value);
3319                 pmaddwd->addArg(y.value);
3320                 ::basicBlock->appendInst(pmaddwd);
3321
3322                 return As<Int2>(V(result));
3323         }
3324
3325         RValue<SByte8> Pack(RValue<Short4> x, RValue<Short4> y)
3326         {
3327                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
3328                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::VectorPackSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3329                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
3330                 auto pack = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3331                 pack->addArg(x.value);
3332                 pack->addArg(y.value);
3333                 ::basicBlock->appendInst(pack);
3334
3335                 return As<SByte8>(Swizzle(As<Int4>(V(result)), 0x88));
3336         }
3337
3338         RValue<Int2> UnpackLow(RValue<Short4> x, RValue<Short4> y)
3339         {
3340                 int shuffle[8] = {0, 8, 1, 9, 2, 10, 3, 11};   // Real type is v8i16
3341                 return As<Int2>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
3342         }
3343
3344         RValue<Int2> UnpackHigh(RValue<Short4> x, RValue<Short4> y)
3345         {
3346                 int shuffle[8] = {0, 8, 1, 9, 2, 10, 3, 11};   // Real type is v8i16
3347                 auto lowHigh = RValue<Short8>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
3348                 return As<Int2>(Swizzle(As<Int4>(lowHigh), 0xEE));
3349         }
3350
3351         RValue<Short4> Swizzle(RValue<Short4> x, unsigned char select)
3352         {
3353                 // Real type is v8i16
3354                 int shuffle[8] =
3355                 {
3356                         (select >> 0) & 0x03,
3357                         (select >> 2) & 0x03,
3358                         (select >> 4) & 0x03,
3359                         (select >> 6) & 0x03,
3360                         (select >> 0) & 0x03,
3361                         (select >> 2) & 0x03,
3362                         (select >> 4) & 0x03,
3363                         (select >> 6) & 0x03,
3364                 };
3365
3366                 return RValue<Short4>(Nucleus::createShuffleVector(x.value, x.value, shuffle));
3367         }
3368
3369         RValue<Short4> Insert(RValue<Short4> val, RValue<Short> element, int i)
3370         {
3371                 return RValue<Short4>(Nucleus::createInsertElement(val.value, element.value, i));
3372         }
3373
3374         RValue<Short> Extract(RValue<Short4> val, int i)
3375         {
3376                 return RValue<Short>(Nucleus::createExtractElement(val.value, Short::getType(), i));
3377         }
3378
3379         RValue<Short4> CmpGT(RValue<Short4> x, RValue<Short4> y)
3380         {
3381                 return RValue<Short4>(createIntCompare(Ice::InstIcmp::Sgt, x.value, y.value));
3382         }
3383
3384         RValue<Short4> CmpEQ(RValue<Short4> x, RValue<Short4> y)
3385         {
3386                 return RValue<Short4>(Nucleus::createICmpEQ(x.value, y.value));
3387         }
3388
3389         Type *Short4::getType()
3390         {
3391                 return T(Type_v4i16);
3392         }
3393
3394         UShort4::UShort4(RValue<Int4> cast)
3395         {
3396                 *this = Short4(cast);
3397         }
3398
3399         UShort4::UShort4(RValue<Float4> cast, bool saturate)
3400         {
3401                 if(saturate)
3402                 {
3403                         if(CPUID::SSE4_1)
3404                         {
3405                                 Int4 int4(Min(cast, Float4(0xFFFF)));   // packusdw takes care of 0x0000 saturation
3406                                 *this = As<Short4>(Pack(As<UInt4>(int4), As<UInt4>(int4)));
3407                         }
3408                         else
3409                         {
3410                                 *this = Short4(Int4(Max(Min(cast, Float4(0xFFFF)), Float4(0x0000))));
3411                         }
3412                 }
3413                 else
3414                 {
3415                         *this = Short4(Int4(cast));
3416                 }
3417         }
3418
3419         UShort4::UShort4(unsigned short xyzw)
3420         {
3421                 int64_t constantVector[4] = {xyzw, xyzw, xyzw, xyzw};
3422                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
3423         }
3424
3425         UShort4::UShort4(unsigned short x, unsigned short y, unsigned short z, unsigned short w)
3426         {
3427                 int64_t constantVector[4] = {x, y, z, w};
3428                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
3429         }
3430
3431         UShort4::UShort4(RValue<UShort4> rhs)
3432         {
3433                 storeValue(rhs.value);
3434         }
3435
3436         UShort4::UShort4(const UShort4 &rhs)
3437         {
3438                 Value *value = rhs.loadValue();
3439                 storeValue(value);
3440         }
3441
3442         UShort4::UShort4(const Reference<UShort4> &rhs)
3443         {
3444                 Value *value = rhs.loadValue();
3445                 storeValue(value);
3446         }
3447
3448         UShort4::UShort4(RValue<Short4> rhs)
3449         {
3450                 storeValue(rhs.value);
3451         }
3452
3453         UShort4::UShort4(const Short4 &rhs)
3454         {
3455                 Value *value = rhs.loadValue();
3456                 storeValue(value);
3457         }
3458
3459         UShort4::UShort4(const Reference<Short4> &rhs)
3460         {
3461                 Value *value = rhs.loadValue();
3462                 storeValue(value);
3463         }
3464
3465         RValue<UShort4> UShort4::operator=(RValue<UShort4> rhs)
3466         {
3467                 storeValue(rhs.value);
3468
3469                 return rhs;
3470         }
3471
3472         RValue<UShort4> UShort4::operator=(const UShort4 &rhs)
3473         {
3474                 Value *value = rhs.loadValue();
3475                 storeValue(value);
3476
3477                 return RValue<UShort4>(value);
3478         }
3479
3480         RValue<UShort4> UShort4::operator=(const Reference<UShort4> &rhs)
3481         {
3482                 Value *value = rhs.loadValue();
3483                 storeValue(value);
3484
3485                 return RValue<UShort4>(value);
3486         }
3487
3488         RValue<UShort4> UShort4::operator=(RValue<Short4> rhs)
3489         {
3490                 storeValue(rhs.value);
3491
3492                 return RValue<UShort4>(rhs);
3493         }
3494
3495         RValue<UShort4> UShort4::operator=(const Short4 &rhs)
3496         {
3497                 Value *value = rhs.loadValue();
3498                 storeValue(value);
3499
3500                 return RValue<UShort4>(value);
3501         }
3502
3503         RValue<UShort4> UShort4::operator=(const Reference<Short4> &rhs)
3504         {
3505                 Value *value = rhs.loadValue();
3506                 storeValue(value);
3507
3508                 return RValue<UShort4>(value);
3509         }
3510
3511         RValue<UShort4> operator+(RValue<UShort4> lhs, RValue<UShort4> rhs)
3512         {
3513                 return RValue<UShort4>(Nucleus::createAdd(lhs.value, rhs.value));
3514         }
3515
3516         RValue<UShort4> operator-(RValue<UShort4> lhs, RValue<UShort4> rhs)
3517         {
3518                 return RValue<UShort4>(Nucleus::createSub(lhs.value, rhs.value));
3519         }
3520
3521         RValue<UShort4> operator*(RValue<UShort4> lhs, RValue<UShort4> rhs)
3522         {
3523                 return RValue<UShort4>(Nucleus::createMul(lhs.value, rhs.value));
3524         }
3525
3526         RValue<UShort4> operator&(RValue<UShort4> lhs, RValue<UShort4> rhs)
3527         {
3528                 return RValue<UShort4>(Nucleus::createAnd(lhs.value, rhs.value));
3529         }
3530
3531         RValue<UShort4> operator|(RValue<UShort4> lhs, RValue<UShort4> rhs)
3532         {
3533                 return RValue<UShort4>(Nucleus::createOr(lhs.value, rhs.value));
3534         }
3535
3536         RValue<UShort4> operator^(RValue<UShort4> lhs, RValue<UShort4> rhs)
3537         {
3538                 return RValue<UShort4>(Nucleus::createXor(lhs.value, rhs.value));
3539         }
3540
3541         RValue<UShort4> operator<<(RValue<UShort4> lhs, unsigned char rhs)
3542         {
3543                 return RValue<UShort4>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
3544         }
3545
3546         RValue<UShort4> operator>>(RValue<UShort4> lhs, unsigned char rhs)
3547         {
3548                 return RValue<UShort4>(Nucleus::createLShr(lhs.value, V(::context->getConstantInt32(rhs))));
3549         }
3550
3551         RValue<UShort4> operator<<=(UShort4 &lhs, unsigned char rhs)
3552         {
3553                 return lhs = lhs << rhs;
3554         }
3555
3556         RValue<UShort4> operator>>=(UShort4 &lhs, unsigned char rhs)
3557         {
3558                 return lhs = lhs >> rhs;
3559         }
3560
3561         RValue<UShort4> operator~(RValue<UShort4> val)
3562         {
3563                 return RValue<UShort4>(Nucleus::createNot(val.value));
3564         }
3565
3566         RValue<UShort4> Max(RValue<UShort4> x, RValue<UShort4> y)
3567         {
3568                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v8i1);
3569                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Ule, condition, x.value, y.value);
3570                 ::basicBlock->appendInst(cmp);
3571
3572                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3573                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
3574                 ::basicBlock->appendInst(select);
3575
3576                 return RValue<UShort4>(V(result));
3577         }
3578
3579         RValue<UShort4> Min(RValue<UShort4> x, RValue<UShort4> y)
3580         {
3581                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v8i1);
3582                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Ugt, condition, x.value, y.value);
3583                 ::basicBlock->appendInst(cmp);
3584
3585                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3586                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
3587                 ::basicBlock->appendInst(select);
3588
3589                 return RValue<UShort4>(V(result));
3590         }
3591
3592         RValue<UShort4> AddSat(RValue<UShort4> x, RValue<UShort4> y)
3593         {
3594                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3595                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::AddSaturateUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3596                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
3597                 auto paddusw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3598                 paddusw->addArg(x.value);
3599                 paddusw->addArg(y.value);
3600                 ::basicBlock->appendInst(paddusw);
3601
3602                 return RValue<UShort4>(V(result));
3603         }
3604
3605         RValue<UShort4> SubSat(RValue<UShort4> x, RValue<UShort4> y)
3606         {
3607                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3608                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SubtractSaturateUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3609                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
3610                 auto psubusw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3611                 psubusw->addArg(x.value);
3612                 psubusw->addArg(y.value);
3613                 ::basicBlock->appendInst(psubusw);
3614
3615                 return RValue<UShort4>(V(result));
3616         }
3617
3618         RValue<UShort4> MulHigh(RValue<UShort4> x, RValue<UShort4> y)
3619         {
3620                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
3621                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::MultiplyHighUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3622                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
3623                 auto pmulhuw = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3624                 pmulhuw->addArg(x.value);
3625                 pmulhuw->addArg(y.value);
3626                 ::basicBlock->appendInst(pmulhuw);
3627
3628                 return RValue<UShort4>(V(result));
3629         }
3630
3631         RValue<UShort4> Average(RValue<UShort4> x, RValue<UShort4> y)
3632         {
3633                 assert(false && "UNIMPLEMENTED"); return RValue<UShort4>(V(nullptr));
3634         }
3635
3636         RValue<Byte8> Pack(RValue<UShort4> x, RValue<UShort4> y)
3637         {
3638                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v16i8);
3639                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::VectorPackUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
3640                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
3641                 auto pack = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
3642                 pack->addArg(x.value);
3643                 pack->addArg(y.value);
3644                 ::basicBlock->appendInst(pack);
3645
3646                 return As<Byte8>(Swizzle(As<Int4>(V(result)), 0x88));
3647         }
3648
3649         Type *UShort4::getType()
3650         {
3651                 return T(Type_v4i16);
3652         }
3653
3654         Short8::Short8(short c)
3655         {
3656                 int64_t constantVector[8] = {c, c, c, c, c, c, c, c};
3657                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
3658         }
3659
3660         Short8::Short8(short c0, short c1, short c2, short c3, short c4, short c5, short c6, short c7)
3661         {
3662                 int64_t constantVector[8] = {c0, c1, c2, c3, c4, c5, c6, c7};
3663                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
3664         }
3665
3666         Short8::Short8(RValue<Short8> rhs)
3667         {
3668                 storeValue(rhs.value);
3669         }
3670
3671         Short8::Short8(const Reference<Short8> &rhs)
3672         {
3673                 Value *value = rhs.loadValue();
3674                 storeValue(value);
3675         }
3676
3677         Short8::Short8(RValue<Short4> lo, RValue<Short4> hi)
3678         {
3679                 int shuffle[8] = {0, 1, 2, 3, 8, 9, 10, 11};   // Real type is v8i16
3680                 Value *packed = Nucleus::createShuffleVector(lo.value, hi.value, shuffle);
3681
3682                 storeValue(packed);
3683         }
3684
3685         RValue<Short8> operator+(RValue<Short8> lhs, RValue<Short8> rhs)
3686         {
3687                 return RValue<Short8>(Nucleus::createAdd(lhs.value, rhs.value));
3688         }
3689
3690         RValue<Short8> operator&(RValue<Short8> lhs, RValue<Short8> rhs)
3691         {
3692                 return RValue<Short8>(Nucleus::createAnd(lhs.value, rhs.value));
3693         }
3694
3695         RValue<Short8> operator<<(RValue<Short8> lhs, unsigned char rhs)
3696         {
3697                 return RValue<Short8>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
3698         }
3699
3700         RValue<Short8> operator>>(RValue<Short8> lhs, unsigned char rhs)
3701         {
3702                 return RValue<Short8>(Nucleus::createAShr(lhs.value, V(::context->getConstantInt32(rhs))));
3703         }
3704
3705         RValue<Int4> MulAdd(RValue<Short8> x, RValue<Short8> y)
3706         {
3707                 assert(false && "UNIMPLEMENTED"); return RValue<Int4>(V(nullptr));
3708         }
3709
3710         RValue<Int4> Abs(RValue<Int4> x)
3711         {
3712                 auto negative = x >> 31;
3713                 return (x ^ negative) - negative;
3714         }
3715
3716         RValue<Short8> MulHigh(RValue<Short8> x, RValue<Short8> y)
3717         {
3718                 assert(false && "UNIMPLEMENTED"); return RValue<Short8>(V(nullptr));
3719         }
3720
3721         Type *Short8::getType()
3722         {
3723                 return T(Ice::IceType_v8i16);
3724         }
3725
3726         UShort8::UShort8(unsigned short c)
3727         {
3728                 int64_t constantVector[8] = {c, c, c, c, c, c, c, c};
3729                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
3730         }
3731
3732         UShort8::UShort8(unsigned short c0, unsigned short c1, unsigned short c2, unsigned short c3, unsigned short c4, unsigned short c5, unsigned short c6, unsigned short c7)
3733         {
3734                 int64_t constantVector[8] = {c0, c1, c2, c3, c4, c5, c6, c7};
3735                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
3736         }
3737
3738         UShort8::UShort8(RValue<UShort8> rhs)
3739         {
3740                 storeValue(rhs.value);
3741         }
3742
3743         UShort8::UShort8(const Reference<UShort8> &rhs)
3744         {
3745                 Value *value = rhs.loadValue();
3746                 storeValue(value);
3747         }
3748
3749         UShort8::UShort8(RValue<UShort4> lo, RValue<UShort4> hi)
3750         {
3751                 int shuffle[8] = {0, 1, 2, 3, 8, 9, 10, 11};   // Real type is v8i16
3752                 Value *packed = Nucleus::createShuffleVector(lo.value, hi.value, shuffle);
3753
3754                 storeValue(packed);
3755         }
3756
3757         RValue<UShort8> UShort8::operator=(RValue<UShort8> rhs)
3758         {
3759                 storeValue(rhs.value);
3760
3761                 return rhs;
3762         }
3763
3764         RValue<UShort8> UShort8::operator=(const UShort8 &rhs)
3765         {
3766                 Value *value = rhs.loadValue();
3767                 storeValue(value);
3768
3769                 return RValue<UShort8>(value);
3770         }
3771
3772         RValue<UShort8> UShort8::operator=(const Reference<UShort8> &rhs)
3773         {
3774                 Value *value = rhs.loadValue();
3775                 storeValue(value);
3776
3777                 return RValue<UShort8>(value);
3778         }
3779
3780         RValue<UShort8> operator&(RValue<UShort8> lhs, RValue<UShort8> rhs)
3781         {
3782                 return RValue<UShort8>(Nucleus::createAnd(lhs.value, rhs.value));
3783         }
3784
3785         RValue<UShort8> operator<<(RValue<UShort8> lhs, unsigned char rhs)
3786         {
3787                 return RValue<UShort8>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
3788         }
3789
3790         RValue<UShort8> operator>>(RValue<UShort8> lhs, unsigned char rhs)
3791         {
3792                 return RValue<UShort8>(Nucleus::createLShr(lhs.value, V(::context->getConstantInt32(rhs))));
3793         }
3794
3795         RValue<UShort8> operator+(RValue<UShort8> lhs, RValue<UShort8> rhs)
3796         {
3797                 return RValue<UShort8>(Nucleus::createAdd(lhs.value, rhs.value));
3798         }
3799
3800         RValue<UShort8> operator*(RValue<UShort8> lhs, RValue<UShort8> rhs)
3801         {
3802                 return RValue<UShort8>(Nucleus::createMul(lhs.value, rhs.value));
3803         }
3804
3805         RValue<UShort8> operator+=(UShort8 &lhs, RValue<UShort8> rhs)
3806         {
3807                 return lhs = lhs + rhs;
3808         }
3809
3810         RValue<UShort8> operator~(RValue<UShort8> val)
3811         {
3812                 return RValue<UShort8>(Nucleus::createNot(val.value));
3813         }
3814
3815         RValue<UShort8> Swizzle(RValue<UShort8> x, char select0, char select1, char select2, char select3, char select4, char select5, char select6, char select7)
3816         {
3817                 assert(false && "UNIMPLEMENTED"); return RValue<UShort8>(V(nullptr));
3818         }
3819
3820         RValue<UShort8> MulHigh(RValue<UShort8> x, RValue<UShort8> y)
3821         {
3822                 assert(false && "UNIMPLEMENTED"); return RValue<UShort8>(V(nullptr));
3823         }
3824
3825         // FIXME: Implement as Shuffle(x, y, Select(i0, ..., i16)) and Shuffle(x, y, SELECT_PACK_REPEAT(element))
3826 //      RValue<UShort8> PackRepeat(RValue<Byte16> x, RValue<Byte16> y, int element)
3827 //      {
3828 //              assert(false && "UNIMPLEMENTED"); return RValue<UShort8>(V(nullptr));
3829 //      }
3830
3831         Type *UShort8::getType()
3832         {
3833                 return T(Ice::IceType_v8i16);
3834         }
3835
3836         Int::Int(Argument<Int> argument)
3837         {
3838                 storeValue(argument.value);
3839         }
3840
3841         Int::Int(RValue<Byte> cast)
3842         {
3843                 Value *integer = Nucleus::createZExt(cast.value, Int::getType());
3844
3845                 storeValue(integer);
3846         }
3847
3848         Int::Int(RValue<SByte> cast)
3849         {
3850                 Value *integer = Nucleus::createSExt(cast.value, Int::getType());
3851
3852                 storeValue(integer);
3853         }
3854
3855         Int::Int(RValue<Short> cast)
3856         {
3857                 Value *integer = Nucleus::createSExt(cast.value, Int::getType());
3858
3859                 storeValue(integer);
3860         }
3861
3862         Int::Int(RValue<UShort> cast)
3863         {
3864                 Value *integer = Nucleus::createZExt(cast.value, Int::getType());
3865
3866                 storeValue(integer);
3867         }
3868
3869         Int::Int(RValue<Int2> cast)
3870         {
3871                 *this = Extract(cast, 0);
3872         }
3873
3874         Int::Int(RValue<Long> cast)
3875         {
3876                 Value *integer = Nucleus::createTrunc(cast.value, Int::getType());
3877
3878                 storeValue(integer);
3879         }
3880
3881         Int::Int(RValue<Float> cast)
3882         {
3883                 Value *integer = Nucleus::createFPToSI(cast.value, Int::getType());
3884
3885                 storeValue(integer);
3886         }
3887
3888         Int::Int(int x)
3889         {
3890                 storeValue(Nucleus::createConstantInt(x));
3891         }
3892
3893         Int::Int(RValue<Int> rhs)
3894         {
3895                 storeValue(rhs.value);
3896         }
3897
3898         Int::Int(RValue<UInt> rhs)
3899         {
3900                 storeValue(rhs.value);
3901         }
3902
3903         Int::Int(const Int &rhs)
3904         {
3905                 Value *value = rhs.loadValue();
3906                 storeValue(value);
3907         }
3908
3909         Int::Int(const Reference<Int> &rhs)
3910         {
3911                 Value *value = rhs.loadValue();
3912                 storeValue(value);
3913         }
3914
3915         Int::Int(const UInt &rhs)
3916         {
3917                 Value *value = rhs.loadValue();
3918                 storeValue(value);
3919         }
3920
3921         Int::Int(const Reference<UInt> &rhs)
3922         {
3923                 Value *value = rhs.loadValue();
3924                 storeValue(value);
3925         }
3926
3927         RValue<Int> Int::operator=(int rhs)
3928         {
3929                 return RValue<Int>(storeValue(Nucleus::createConstantInt(rhs)));
3930         }
3931
3932         RValue<Int> Int::operator=(RValue<Int> rhs)
3933         {
3934                 storeValue(rhs.value);
3935
3936                 return rhs;
3937         }
3938
3939         RValue<Int> Int::operator=(RValue<UInt> rhs)
3940         {
3941                 storeValue(rhs.value);
3942
3943                 return RValue<Int>(rhs);
3944         }
3945
3946         RValue<Int> Int::operator=(const Int &rhs)
3947         {
3948                 Value *value = rhs.loadValue();
3949                 storeValue(value);
3950
3951                 return RValue<Int>(value);
3952         }
3953
3954         RValue<Int> Int::operator=(const Reference<Int> &rhs)
3955         {
3956                 Value *value = rhs.loadValue();
3957                 storeValue(value);
3958
3959                 return RValue<Int>(value);
3960         }
3961
3962         RValue<Int> Int::operator=(const UInt &rhs)
3963         {
3964                 Value *value = rhs.loadValue();
3965                 storeValue(value);
3966
3967                 return RValue<Int>(value);
3968         }
3969
3970         RValue<Int> Int::operator=(const Reference<UInt> &rhs)
3971         {
3972                 Value *value = rhs.loadValue();
3973                 storeValue(value);
3974
3975                 return RValue<Int>(value);
3976         }
3977
3978         RValue<Int> operator+(RValue<Int> lhs, RValue<Int> rhs)
3979         {
3980                 return RValue<Int>(Nucleus::createAdd(lhs.value, rhs.value));
3981         }
3982
3983         RValue<Int> operator-(RValue<Int> lhs, RValue<Int> rhs)
3984         {
3985                 return RValue<Int>(Nucleus::createSub(lhs.value, rhs.value));
3986         }
3987
3988         RValue<Int> operator*(RValue<Int> lhs, RValue<Int> rhs)
3989         {
3990                 return RValue<Int>(Nucleus::createMul(lhs.value, rhs.value));
3991         }
3992
3993         RValue<Int> operator/(RValue<Int> lhs, RValue<Int> rhs)
3994         {
3995                 return RValue<Int>(Nucleus::createSDiv(lhs.value, rhs.value));
3996         }
3997
3998         RValue<Int> operator%(RValue<Int> lhs, RValue<Int> rhs)
3999         {
4000                 return RValue<Int>(Nucleus::createSRem(lhs.value, rhs.value));
4001         }
4002
4003         RValue<Int> operator&(RValue<Int> lhs, RValue<Int> rhs)
4004         {
4005                 return RValue<Int>(Nucleus::createAnd(lhs.value, rhs.value));
4006         }
4007
4008         RValue<Int> operator|(RValue<Int> lhs, RValue<Int> rhs)
4009         {
4010                 return RValue<Int>(Nucleus::createOr(lhs.value, rhs.value));
4011         }
4012
4013         RValue<Int> operator^(RValue<Int> lhs, RValue<Int> rhs)
4014         {
4015                 return RValue<Int>(Nucleus::createXor(lhs.value, rhs.value));
4016         }
4017
4018         RValue<Int> operator<<(RValue<Int> lhs, RValue<Int> rhs)
4019         {
4020                 return RValue<Int>(Nucleus::createShl(lhs.value, rhs.value));
4021         }
4022
4023         RValue<Int> operator>>(RValue<Int> lhs, RValue<Int> rhs)
4024         {
4025                 return RValue<Int>(Nucleus::createAShr(lhs.value, rhs.value));
4026         }
4027
4028         RValue<Int> operator+=(Int &lhs, RValue<Int> rhs)
4029         {
4030                 return lhs = lhs + rhs;
4031         }
4032
4033         RValue<Int> operator-=(Int &lhs, RValue<Int> rhs)
4034         {
4035                 return lhs = lhs - rhs;
4036         }
4037
4038         RValue<Int> operator*=(Int &lhs, RValue<Int> rhs)
4039         {
4040                 return lhs = lhs * rhs;
4041         }
4042
4043         RValue<Int> operator/=(Int &lhs, RValue<Int> rhs)
4044         {
4045                 return lhs = lhs / rhs;
4046         }
4047
4048         RValue<Int> operator%=(Int &lhs, RValue<Int> rhs)
4049         {
4050                 return lhs = lhs % rhs;
4051         }
4052
4053         RValue<Int> operator&=(Int &lhs, RValue<Int> rhs)
4054         {
4055                 return lhs = lhs & rhs;
4056         }
4057
4058         RValue<Int> operator|=(Int &lhs, RValue<Int> rhs)
4059         {
4060                 return lhs = lhs | rhs;
4061         }
4062
4063         RValue<Int> operator^=(Int &lhs, RValue<Int> rhs)
4064         {
4065                 return lhs = lhs ^ rhs;
4066         }
4067
4068         RValue<Int> operator<<=(Int &lhs, RValue<Int> rhs)
4069         {
4070                 return lhs = lhs << rhs;
4071         }
4072
4073         RValue<Int> operator>>=(Int &lhs, RValue<Int> rhs)
4074         {
4075                 return lhs = lhs >> rhs;
4076         }
4077
4078         RValue<Int> operator+(RValue<Int> val)
4079         {
4080                 return val;
4081         }
4082
4083         RValue<Int> operator-(RValue<Int> val)
4084         {
4085                 return RValue<Int>(Nucleus::createNeg(val.value));
4086         }
4087
4088         RValue<Int> operator~(RValue<Int> val)
4089         {
4090                 return RValue<Int>(Nucleus::createNot(val.value));
4091         }
4092
4093         RValue<Int> operator++(Int &val, int)   // Post-increment
4094         {
4095                 RValue<Int> res = val;
4096                 val += 1;
4097                 return res;
4098         }
4099
4100         const Int &operator++(Int &val)   // Pre-increment
4101         {
4102                 val += 1;
4103                 return val;
4104         }
4105
4106         RValue<Int> operator--(Int &val, int)   // Post-decrement
4107         {
4108                 RValue<Int> res = val;
4109                 val -= 1;
4110                 return res;
4111         }
4112
4113         const Int &operator--(Int &val)   // Pre-decrement
4114         {
4115                 val -= 1;
4116                 return val;
4117         }
4118
4119         RValue<Bool> operator<(RValue<Int> lhs, RValue<Int> rhs)
4120         {
4121                 return RValue<Bool>(Nucleus::createICmpSLT(lhs.value, rhs.value));
4122         }
4123
4124         RValue<Bool> operator<=(RValue<Int> lhs, RValue<Int> rhs)
4125         {
4126                 return RValue<Bool>(Nucleus::createICmpSLE(lhs.value, rhs.value));
4127         }
4128
4129         RValue<Bool> operator>(RValue<Int> lhs, RValue<Int> rhs)
4130         {
4131                 return RValue<Bool>(Nucleus::createICmpSGT(lhs.value, rhs.value));
4132         }
4133
4134         RValue<Bool> operator>=(RValue<Int> lhs, RValue<Int> rhs)
4135         {
4136                 return RValue<Bool>(Nucleus::createICmpSGE(lhs.value, rhs.value));
4137         }
4138
4139         RValue<Bool> operator!=(RValue<Int> lhs, RValue<Int> rhs)
4140         {
4141                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
4142         }
4143
4144         RValue<Bool> operator==(RValue<Int> lhs, RValue<Int> rhs)
4145         {
4146                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
4147         }
4148
4149         RValue<Int> Max(RValue<Int> x, RValue<Int> y)
4150         {
4151                 return IfThenElse(x > y, x, y);
4152         }
4153
4154         RValue<Int> Min(RValue<Int> x, RValue<Int> y)
4155         {
4156                 return IfThenElse(x < y, x, y);
4157         }
4158
4159         RValue<Int> Clamp(RValue<Int> x, RValue<Int> min, RValue<Int> max)
4160         {
4161                 return Min(Max(x, min), max);
4162         }
4163
4164         RValue<Int> RoundInt(RValue<Float> cast)
4165         {
4166                 if(emulateIntrinsics)
4167                 {
4168                         // Push the fractional part off the mantissa. Accurate up to +/-2^22.
4169                         return Int((cast + Float(0x00C00000)) - Float(0x00C00000));
4170                 }
4171                 else
4172                 {
4173                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_i32);
4174                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Nearbyint, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
4175                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
4176                         auto nearbyint = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
4177                         nearbyint->addArg(cast.value);
4178                         ::basicBlock->appendInst(nearbyint);
4179
4180                         return RValue<Int>(V(result));
4181                 }
4182         }
4183
4184         Type *Int::getType()
4185         {
4186                 return T(Ice::IceType_i32);
4187         }
4188
4189         Long::Long(RValue<Int> cast)
4190         {
4191                 Value *integer = Nucleus::createSExt(cast.value, Long::getType());
4192
4193                 storeValue(integer);
4194         }
4195
4196         Long::Long(RValue<UInt> cast)
4197         {
4198                 Value *integer = Nucleus::createZExt(cast.value, Long::getType());
4199
4200                 storeValue(integer);
4201         }
4202
4203         Long::Long(RValue<Long> rhs)
4204         {
4205                 storeValue(rhs.value);
4206         }
4207
4208         RValue<Long> Long::operator=(int64_t rhs)
4209         {
4210                 return RValue<Long>(storeValue(Nucleus::createConstantLong(rhs)));
4211         }
4212
4213         RValue<Long> Long::operator=(RValue<Long> rhs)
4214         {
4215                 storeValue(rhs.value);
4216
4217                 return rhs;
4218         }
4219
4220         RValue<Long> Long::operator=(const Long &rhs)
4221         {
4222                 Value *value = rhs.loadValue();
4223                 storeValue(value);
4224
4225                 return RValue<Long>(value);
4226         }
4227
4228         RValue<Long> Long::operator=(const Reference<Long> &rhs)
4229         {
4230                 Value *value = rhs.loadValue();
4231                 storeValue(value);
4232
4233                 return RValue<Long>(value);
4234         }
4235
4236         RValue<Long> operator+(RValue<Long> lhs, RValue<Long> rhs)
4237         {
4238                 return RValue<Long>(Nucleus::createAdd(lhs.value, rhs.value));
4239         }
4240
4241         RValue<Long> operator-(RValue<Long> lhs, RValue<Long> rhs)
4242         {
4243                 return RValue<Long>(Nucleus::createSub(lhs.value, rhs.value));
4244         }
4245
4246         RValue<Long> operator+=(Long &lhs, RValue<Long> rhs)
4247         {
4248                 return lhs = lhs + rhs;
4249         }
4250
4251         RValue<Long> operator-=(Long &lhs, RValue<Long> rhs)
4252         {
4253                 return lhs = lhs - rhs;
4254         }
4255
4256         RValue<Long> AddAtomic(RValue<Pointer<Long> > x, RValue<Long> y)
4257         {
4258                 return RValue<Long>(Nucleus::createAtomicAdd(x.value, y.value));
4259         }
4260
4261         Type *Long::getType()
4262         {
4263                 return T(Ice::IceType_i64);
4264         }
4265
4266         UInt::UInt(Argument<UInt> argument)
4267         {
4268                 storeValue(argument.value);
4269         }
4270
4271         UInt::UInt(RValue<UShort> cast)
4272         {
4273                 Value *integer = Nucleus::createZExt(cast.value, UInt::getType());
4274
4275                 storeValue(integer);
4276         }
4277
4278         UInt::UInt(RValue<Long> cast)
4279         {
4280                 Value *integer = Nucleus::createTrunc(cast.value, UInt::getType());
4281
4282                 storeValue(integer);
4283         }
4284
4285         UInt::UInt(RValue<Float> cast)
4286         {
4287                 // Smallest positive value representable in UInt, but not in Int
4288                 const unsigned int ustart = 0x80000000u;
4289                 const float ustartf = float(ustart);
4290
4291                 // If the value is negative, store 0, otherwise store the result of the conversion
4292                 storeValue((~(As<Int>(cast) >> 31) &
4293                 // Check if the value can be represented as an Int
4294                         IfThenElse(cast >= ustartf,
4295                 // If the value is too large, subtract ustart and re-add it after conversion.
4296                                 As<Int>(As<UInt>(Int(cast - Float(ustartf))) + UInt(ustart)),
4297                 // Otherwise, just convert normally
4298                                 Int(cast))).value);
4299         }
4300
4301         UInt::UInt(int x)
4302         {
4303                 storeValue(Nucleus::createConstantInt(x));
4304         }
4305
4306         UInt::UInt(unsigned int x)
4307         {
4308                 storeValue(Nucleus::createConstantInt(x));
4309         }
4310
4311         UInt::UInt(RValue<UInt> rhs)
4312         {
4313                 storeValue(rhs.value);
4314         }
4315
4316         UInt::UInt(RValue<Int> rhs)
4317         {
4318                 storeValue(rhs.value);
4319         }
4320
4321         UInt::UInt(const UInt &rhs)
4322         {
4323                 Value *value = rhs.loadValue();
4324                 storeValue(value);
4325         }
4326
4327         UInt::UInt(const Reference<UInt> &rhs)
4328         {
4329                 Value *value = rhs.loadValue();
4330                 storeValue(value);
4331         }
4332
4333         UInt::UInt(const Int &rhs)
4334         {
4335                 Value *value = rhs.loadValue();
4336                 storeValue(value);
4337         }
4338
4339         UInt::UInt(const Reference<Int> &rhs)
4340         {
4341                 Value *value = rhs.loadValue();
4342                 storeValue(value);
4343         }
4344
4345         RValue<UInt> UInt::operator=(unsigned int rhs)
4346         {
4347                 return RValue<UInt>(storeValue(Nucleus::createConstantInt(rhs)));
4348         }
4349
4350         RValue<UInt> UInt::operator=(RValue<UInt> rhs)
4351         {
4352                 storeValue(rhs.value);
4353
4354                 return rhs;
4355         }
4356
4357         RValue<UInt> UInt::operator=(RValue<Int> rhs)
4358         {
4359                 storeValue(rhs.value);
4360
4361                 return RValue<UInt>(rhs);
4362         }
4363
4364         RValue<UInt> UInt::operator=(const UInt &rhs)
4365         {
4366                 Value *value = rhs.loadValue();
4367                 storeValue(value);
4368
4369                 return RValue<UInt>(value);
4370         }
4371
4372         RValue<UInt> UInt::operator=(const Reference<UInt> &rhs)
4373         {
4374                 Value *value = rhs.loadValue();
4375                 storeValue(value);
4376
4377                 return RValue<UInt>(value);
4378         }
4379
4380         RValue<UInt> UInt::operator=(const Int &rhs)
4381         {
4382                 Value *value = rhs.loadValue();
4383                 storeValue(value);
4384
4385                 return RValue<UInt>(value);
4386         }
4387
4388         RValue<UInt> UInt::operator=(const Reference<Int> &rhs)
4389         {
4390                 Value *value = rhs.loadValue();
4391                 storeValue(value);
4392
4393                 return RValue<UInt>(value);
4394         }
4395
4396         RValue<UInt> operator+(RValue<UInt> lhs, RValue<UInt> rhs)
4397         {
4398                 return RValue<UInt>(Nucleus::createAdd(lhs.value, rhs.value));
4399         }
4400
4401         RValue<UInt> operator-(RValue<UInt> lhs, RValue<UInt> rhs)
4402         {
4403                 return RValue<UInt>(Nucleus::createSub(lhs.value, rhs.value));
4404         }
4405
4406         RValue<UInt> operator*(RValue<UInt> lhs, RValue<UInt> rhs)
4407         {
4408                 return RValue<UInt>(Nucleus::createMul(lhs.value, rhs.value));
4409         }
4410
4411         RValue<UInt> operator/(RValue<UInt> lhs, RValue<UInt> rhs)
4412         {
4413                 return RValue<UInt>(Nucleus::createUDiv(lhs.value, rhs.value));
4414         }
4415
4416         RValue<UInt> operator%(RValue<UInt> lhs, RValue<UInt> rhs)
4417         {
4418                 return RValue<UInt>(Nucleus::createURem(lhs.value, rhs.value));
4419         }
4420
4421         RValue<UInt> operator&(RValue<UInt> lhs, RValue<UInt> rhs)
4422         {
4423                 return RValue<UInt>(Nucleus::createAnd(lhs.value, rhs.value));
4424         }
4425
4426         RValue<UInt> operator|(RValue<UInt> lhs, RValue<UInt> rhs)
4427         {
4428                 return RValue<UInt>(Nucleus::createOr(lhs.value, rhs.value));
4429         }
4430
4431         RValue<UInt> operator^(RValue<UInt> lhs, RValue<UInt> rhs)
4432         {
4433                 return RValue<UInt>(Nucleus::createXor(lhs.value, rhs.value));
4434         }
4435
4436         RValue<UInt> operator<<(RValue<UInt> lhs, RValue<UInt> rhs)
4437         {
4438                 return RValue<UInt>(Nucleus::createShl(lhs.value, rhs.value));
4439         }
4440
4441         RValue<UInt> operator>>(RValue<UInt> lhs, RValue<UInt> rhs)
4442         {
4443                 return RValue<UInt>(Nucleus::createLShr(lhs.value, rhs.value));
4444         }
4445
4446         RValue<UInt> operator+=(UInt &lhs, RValue<UInt> rhs)
4447         {
4448                 return lhs = lhs + rhs;
4449         }
4450
4451         RValue<UInt> operator-=(UInt &lhs, RValue<UInt> rhs)
4452         {
4453                 return lhs = lhs - rhs;
4454         }
4455
4456         RValue<UInt> operator*=(UInt &lhs, RValue<UInt> rhs)
4457         {
4458                 return lhs = lhs * rhs;
4459         }
4460
4461         RValue<UInt> operator/=(UInt &lhs, RValue<UInt> rhs)
4462         {
4463                 return lhs = lhs / rhs;
4464         }
4465
4466         RValue<UInt> operator%=(UInt &lhs, RValue<UInt> rhs)
4467         {
4468                 return lhs = lhs % rhs;
4469         }
4470
4471         RValue<UInt> operator&=(UInt &lhs, RValue<UInt> rhs)
4472         {
4473                 return lhs = lhs & rhs;
4474         }
4475
4476         RValue<UInt> operator|=(UInt &lhs, RValue<UInt> rhs)
4477         {
4478                 return lhs = lhs | rhs;
4479         }
4480
4481         RValue<UInt> operator^=(UInt &lhs, RValue<UInt> rhs)
4482         {
4483                 return lhs = lhs ^ rhs;
4484         }
4485
4486         RValue<UInt> operator<<=(UInt &lhs, RValue<UInt> rhs)
4487         {
4488                 return lhs = lhs << rhs;
4489         }
4490
4491         RValue<UInt> operator>>=(UInt &lhs, RValue<UInt> rhs)
4492         {
4493                 return lhs = lhs >> rhs;
4494         }
4495
4496         RValue<UInt> operator+(RValue<UInt> val)
4497         {
4498                 return val;
4499         }
4500
4501         RValue<UInt> operator-(RValue<UInt> val)
4502         {
4503                 return RValue<UInt>(Nucleus::createNeg(val.value));
4504         }
4505
4506         RValue<UInt> operator~(RValue<UInt> val)
4507         {
4508                 return RValue<UInt>(Nucleus::createNot(val.value));
4509         }
4510
4511         RValue<UInt> operator++(UInt &val, int)   // Post-increment
4512         {
4513                 RValue<UInt> res = val;
4514                 val += 1;
4515                 return res;
4516         }
4517
4518         const UInt &operator++(UInt &val)   // Pre-increment
4519         {
4520                 val += 1;
4521                 return val;
4522         }
4523
4524         RValue<UInt> operator--(UInt &val, int)   // Post-decrement
4525         {
4526                 RValue<UInt> res = val;
4527                 val -= 1;
4528                 return res;
4529         }
4530
4531         const UInt &operator--(UInt &val)   // Pre-decrement
4532         {
4533                 val -= 1;
4534                 return val;
4535         }
4536
4537         RValue<UInt> Max(RValue<UInt> x, RValue<UInt> y)
4538         {
4539                 return IfThenElse(x > y, x, y);
4540         }
4541
4542         RValue<UInt> Min(RValue<UInt> x, RValue<UInt> y)
4543         {
4544                 return IfThenElse(x < y, x, y);
4545         }
4546
4547         RValue<UInt> Clamp(RValue<UInt> x, RValue<UInt> min, RValue<UInt> max)
4548         {
4549                 return Min(Max(x, min), max);
4550         }
4551
4552         RValue<Bool> operator<(RValue<UInt> lhs, RValue<UInt> rhs)
4553         {
4554                 return RValue<Bool>(Nucleus::createICmpULT(lhs.value, rhs.value));
4555         }
4556
4557         RValue<Bool> operator<=(RValue<UInt> lhs, RValue<UInt> rhs)
4558         {
4559                 return RValue<Bool>(Nucleus::createICmpULE(lhs.value, rhs.value));
4560         }
4561
4562         RValue<Bool> operator>(RValue<UInt> lhs, RValue<UInt> rhs)
4563         {
4564                 return RValue<Bool>(Nucleus::createICmpUGT(lhs.value, rhs.value));
4565         }
4566
4567         RValue<Bool> operator>=(RValue<UInt> lhs, RValue<UInt> rhs)
4568         {
4569                 return RValue<Bool>(Nucleus::createICmpUGE(lhs.value, rhs.value));
4570         }
4571
4572         RValue<Bool> operator!=(RValue<UInt> lhs, RValue<UInt> rhs)
4573         {
4574                 return RValue<Bool>(Nucleus::createICmpNE(lhs.value, rhs.value));
4575         }
4576
4577         RValue<Bool> operator==(RValue<UInt> lhs, RValue<UInt> rhs)
4578         {
4579                 return RValue<Bool>(Nucleus::createICmpEQ(lhs.value, rhs.value));
4580         }
4581
4582 //      RValue<UInt> RoundUInt(RValue<Float> cast)
4583 //      {
4584 //              assert(false && "UNIMPLEMENTED"); return RValue<UInt>(V(nullptr));
4585 //      }
4586
4587         Type *UInt::getType()
4588         {
4589                 return T(Ice::IceType_i32);
4590         }
4591
4592 //      Int2::Int2(RValue<Int> cast)
4593 //      {
4594 //              Value *extend = Nucleus::createZExt(cast.value, Long::getType());
4595 //              Value *vector = Nucleus::createBitCast(extend, Int2::getType());
4596 //
4597 //              Constant *shuffle[2];
4598 //              shuffle[0] = Nucleus::createConstantInt(0);
4599 //              shuffle[1] = Nucleus::createConstantInt(0);
4600 //
4601 //              Value *replicate = Nucleus::createShuffleVector(vector, UndefValue::get(Int2::getType()), Nucleus::createConstantVector(shuffle, 2));
4602 //
4603 //              storeValue(replicate);
4604 //      }
4605
4606         Int2::Int2(RValue<Int4> cast)
4607         {
4608                 storeValue(Nucleus::createBitCast(cast.value, getType()));
4609         }
4610
4611         Int2::Int2(int x, int y)
4612         {
4613                 int64_t constantVector[2] = {x, y};
4614                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
4615         }
4616
4617         Int2::Int2(RValue<Int2> rhs)
4618         {
4619                 storeValue(rhs.value);
4620         }
4621
4622         Int2::Int2(const Int2 &rhs)
4623         {
4624                 Value *value = rhs.loadValue();
4625                 storeValue(value);
4626         }
4627
4628         Int2::Int2(const Reference<Int2> &rhs)
4629         {
4630                 Value *value = rhs.loadValue();
4631                 storeValue(value);
4632         }
4633
4634         Int2::Int2(RValue<Int> lo, RValue<Int> hi)
4635         {
4636                 int shuffle[4] = {0, 4, 1, 5};
4637                 Value *packed = Nucleus::createShuffleVector(Int4(lo).loadValue(), Int4(hi).loadValue(), shuffle);
4638
4639                 storeValue(Nucleus::createBitCast(packed, Int2::getType()));
4640         }
4641
4642         RValue<Int2> Int2::operator=(RValue<Int2> rhs)
4643         {
4644                 storeValue(rhs.value);
4645
4646                 return rhs;
4647         }
4648
4649         RValue<Int2> Int2::operator=(const Int2 &rhs)
4650         {
4651                 Value *value = rhs.loadValue();
4652                 storeValue(value);
4653
4654                 return RValue<Int2>(value);
4655         }
4656
4657         RValue<Int2> Int2::operator=(const Reference<Int2> &rhs)
4658         {
4659                 Value *value = rhs.loadValue();
4660                 storeValue(value);
4661
4662                 return RValue<Int2>(value);
4663         }
4664
4665         RValue<Int2> operator+(RValue<Int2> lhs, RValue<Int2> rhs)
4666         {
4667                 return RValue<Int2>(Nucleus::createAdd(lhs.value, rhs.value));
4668         }
4669
4670         RValue<Int2> operator-(RValue<Int2> lhs, RValue<Int2> rhs)
4671         {
4672                 return RValue<Int2>(Nucleus::createSub(lhs.value, rhs.value));
4673         }
4674
4675 //      RValue<Int2> operator*(RValue<Int2> lhs, RValue<Int2> rhs)
4676 //      {
4677 //              return RValue<Int2>(Nucleus::createMul(lhs.value, rhs.value));
4678 //      }
4679
4680 //      RValue<Int2> operator/(RValue<Int2> lhs, RValue<Int2> rhs)
4681 //      {
4682 //              return RValue<Int2>(Nucleus::createSDiv(lhs.value, rhs.value));
4683 //      }
4684
4685 //      RValue<Int2> operator%(RValue<Int2> lhs, RValue<Int2> rhs)
4686 //      {
4687 //              return RValue<Int2>(Nucleus::createSRem(lhs.value, rhs.value));
4688 //      }
4689
4690         RValue<Int2> operator&(RValue<Int2> lhs, RValue<Int2> rhs)
4691         {
4692                 return RValue<Int2>(Nucleus::createAnd(lhs.value, rhs.value));
4693         }
4694
4695         RValue<Int2> operator|(RValue<Int2> lhs, RValue<Int2> rhs)
4696         {
4697                 return RValue<Int2>(Nucleus::createOr(lhs.value, rhs.value));
4698         }
4699
4700         RValue<Int2> operator^(RValue<Int2> lhs, RValue<Int2> rhs)
4701         {
4702                 return RValue<Int2>(Nucleus::createXor(lhs.value, rhs.value));
4703         }
4704
4705         RValue<Int2> operator<<(RValue<Int2> lhs, unsigned char rhs)
4706         {
4707                 return RValue<Int2>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
4708         }
4709
4710         RValue<Int2> operator>>(RValue<Int2> lhs, unsigned char rhs)
4711         {
4712                 return RValue<Int2>(Nucleus::createAShr(lhs.value, V(::context->getConstantInt32(rhs))));
4713         }
4714
4715         RValue<Int2> operator+=(Int2 &lhs, RValue<Int2> rhs)
4716         {
4717                 return lhs = lhs + rhs;
4718         }
4719
4720         RValue<Int2> operator-=(Int2 &lhs, RValue<Int2> rhs)
4721         {
4722                 return lhs = lhs - rhs;
4723         }
4724
4725 //      RValue<Int2> operator*=(Int2 &lhs, RValue<Int2> rhs)
4726 //      {
4727 //              return lhs = lhs * rhs;
4728 //      }
4729
4730 //      RValue<Int2> operator/=(Int2 &lhs, RValue<Int2> rhs)
4731 //      {
4732 //              return lhs = lhs / rhs;
4733 //      }
4734
4735 //      RValue<Int2> operator%=(Int2 &lhs, RValue<Int2> rhs)
4736 //      {
4737 //              return lhs = lhs % rhs;
4738 //      }
4739
4740         RValue<Int2> operator&=(Int2 &lhs, RValue<Int2> rhs)
4741         {
4742                 return lhs = lhs & rhs;
4743         }
4744
4745         RValue<Int2> operator|=(Int2 &lhs, RValue<Int2> rhs)
4746         {
4747                 return lhs = lhs | rhs;
4748         }
4749
4750         RValue<Int2> operator^=(Int2 &lhs, RValue<Int2> rhs)
4751         {
4752                 return lhs = lhs ^ rhs;
4753         }
4754
4755         RValue<Int2> operator<<=(Int2 &lhs, unsigned char rhs)
4756         {
4757                 return lhs = lhs << rhs;
4758         }
4759
4760         RValue<Int2> operator>>=(Int2 &lhs, unsigned char rhs)
4761         {
4762                 return lhs = lhs >> rhs;
4763         }
4764
4765 //      RValue<Int2> operator+(RValue<Int2> val)
4766 //      {
4767 //              return val;
4768 //      }
4769
4770 //      RValue<Int2> operator-(RValue<Int2> val)
4771 //      {
4772 //              return RValue<Int2>(Nucleus::createNeg(val.value));
4773 //      }
4774
4775         RValue<Int2> operator~(RValue<Int2> val)
4776         {
4777                 return RValue<Int2>(Nucleus::createNot(val.value));
4778         }
4779
4780         RValue<Short4> UnpackLow(RValue<Int2> x, RValue<Int2> y)
4781         {
4782                 int shuffle[4] = {0, 4, 1, 5};   // Real type is v4i32
4783                 return As<Short4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
4784         }
4785
4786         RValue<Short4> UnpackHigh(RValue<Int2> x, RValue<Int2> y)
4787         {
4788                 int shuffle[4] = {0, 4, 1, 5};   // Real type is v4i32
4789                 auto lowHigh = RValue<Int4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
4790                 return As<Short4>(Swizzle(lowHigh, 0xEE));
4791         }
4792
4793         RValue<Int> Extract(RValue<Int2> val, int i)
4794         {
4795                 return RValue<Int>(Nucleus::createExtractElement(val.value, Int::getType(), i));
4796         }
4797
4798         RValue<Int2> Insert(RValue<Int2> val, RValue<Int> element, int i)
4799         {
4800                 return RValue<Int2>(Nucleus::createInsertElement(val.value, element.value, i));
4801         }
4802
4803         Type *Int2::getType()
4804         {
4805                 return T(Type_v2i32);
4806         }
4807
4808         UInt2::UInt2(unsigned int x, unsigned int y)
4809         {
4810                 int64_t constantVector[2] = {x, y};
4811                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
4812         }
4813
4814         UInt2::UInt2(RValue<UInt2> rhs)
4815         {
4816                 storeValue(rhs.value);
4817         }
4818
4819         UInt2::UInt2(const UInt2 &rhs)
4820         {
4821                 Value *value = rhs.loadValue();
4822                 storeValue(value);
4823         }
4824
4825         UInt2::UInt2(const Reference<UInt2> &rhs)
4826         {
4827                 Value *value = rhs.loadValue();
4828                 storeValue(value);
4829         }
4830
4831         RValue<UInt2> UInt2::operator=(RValue<UInt2> rhs)
4832         {
4833                 storeValue(rhs.value);
4834
4835                 return rhs;
4836         }
4837
4838         RValue<UInt2> UInt2::operator=(const UInt2 &rhs)
4839         {
4840                 Value *value = rhs.loadValue();
4841                 storeValue(value);
4842
4843                 return RValue<UInt2>(value);
4844         }
4845
4846         RValue<UInt2> UInt2::operator=(const Reference<UInt2> &rhs)
4847         {
4848                 Value *value = rhs.loadValue();
4849                 storeValue(value);
4850
4851                 return RValue<UInt2>(value);
4852         }
4853
4854         RValue<UInt2> operator+(RValue<UInt2> lhs, RValue<UInt2> rhs)
4855         {
4856                 return RValue<UInt2>(Nucleus::createAdd(lhs.value, rhs.value));
4857         }
4858
4859         RValue<UInt2> operator-(RValue<UInt2> lhs, RValue<UInt2> rhs)
4860         {
4861                 return RValue<UInt2>(Nucleus::createSub(lhs.value, rhs.value));
4862         }
4863
4864 //      RValue<UInt2> operator*(RValue<UInt2> lhs, RValue<UInt2> rhs)
4865 //      {
4866 //              return RValue<UInt2>(Nucleus::createMul(lhs.value, rhs.value));
4867 //      }
4868
4869 //      RValue<UInt2> operator/(RValue<UInt2> lhs, RValue<UInt2> rhs)
4870 //      {
4871 //              return RValue<UInt2>(Nucleus::createUDiv(lhs.value, rhs.value));
4872 //      }
4873
4874 //      RValue<UInt2> operator%(RValue<UInt2> lhs, RValue<UInt2> rhs)
4875 //      {
4876 //              return RValue<UInt2>(Nucleus::createURem(lhs.value, rhs.value));
4877 //      }
4878
4879         RValue<UInt2> operator&(RValue<UInt2> lhs, RValue<UInt2> rhs)
4880         {
4881                 return RValue<UInt2>(Nucleus::createAnd(lhs.value, rhs.value));
4882         }
4883
4884         RValue<UInt2> operator|(RValue<UInt2> lhs, RValue<UInt2> rhs)
4885         {
4886                 return RValue<UInt2>(Nucleus::createOr(lhs.value, rhs.value));
4887         }
4888
4889         RValue<UInt2> operator^(RValue<UInt2> lhs, RValue<UInt2> rhs)
4890         {
4891                 return RValue<UInt2>(Nucleus::createXor(lhs.value, rhs.value));
4892         }
4893
4894         RValue<UInt2> operator<<(RValue<UInt2> lhs, unsigned char rhs)
4895         {
4896                 return RValue<UInt2>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
4897         }
4898
4899         RValue<UInt2> operator>>(RValue<UInt2> lhs, unsigned char rhs)
4900         {
4901                 return RValue<UInt2>(Nucleus::createLShr(lhs.value, V(::context->getConstantInt32(rhs))));
4902         }
4903
4904         RValue<UInt2> operator+=(UInt2 &lhs, RValue<UInt2> rhs)
4905         {
4906                 return lhs = lhs + rhs;
4907         }
4908
4909         RValue<UInt2> operator-=(UInt2 &lhs, RValue<UInt2> rhs)
4910         {
4911                 return lhs = lhs - rhs;
4912         }
4913
4914 //      RValue<UInt2> operator*=(UInt2 &lhs, RValue<UInt2> rhs)
4915 //      {
4916 //              return lhs = lhs * rhs;
4917 //      }
4918
4919 //      RValue<UInt2> operator/=(UInt2 &lhs, RValue<UInt2> rhs)
4920 //      {
4921 //              return lhs = lhs / rhs;
4922 //      }
4923
4924 //      RValue<UInt2> operator%=(UInt2 &lhs, RValue<UInt2> rhs)
4925 //      {
4926 //              return lhs = lhs % rhs;
4927 //      }
4928
4929         RValue<UInt2> operator&=(UInt2 &lhs, RValue<UInt2> rhs)
4930         {
4931                 return lhs = lhs & rhs;
4932         }
4933
4934         RValue<UInt2> operator|=(UInt2 &lhs, RValue<UInt2> rhs)
4935         {
4936                 return lhs = lhs | rhs;
4937         }
4938
4939         RValue<UInt2> operator^=(UInt2 &lhs, RValue<UInt2> rhs)
4940         {
4941                 return lhs = lhs ^ rhs;
4942         }
4943
4944         RValue<UInt2> operator<<=(UInt2 &lhs, unsigned char rhs)
4945         {
4946                 return lhs = lhs << rhs;
4947         }
4948
4949         RValue<UInt2> operator>>=(UInt2 &lhs, unsigned char rhs)
4950         {
4951                 return lhs = lhs >> rhs;
4952         }
4953
4954 //      RValue<UInt2> operator+(RValue<UInt2> val)
4955 //      {
4956 //              return val;
4957 //      }
4958
4959 //      RValue<UInt2> operator-(RValue<UInt2> val)
4960 //      {
4961 //              return RValue<UInt2>(Nucleus::createNeg(val.value));
4962 //      }
4963
4964         RValue<UInt2> operator~(RValue<UInt2> val)
4965         {
4966                 return RValue<UInt2>(Nucleus::createNot(val.value));
4967         }
4968
4969         Type *UInt2::getType()
4970         {
4971                 return T(Type_v2i32);
4972         }
4973
4974         Int4::Int4(RValue<Byte4> cast)
4975         {
4976                 Value *x = Nucleus::createBitCast(cast.value, Int::getType());
4977                 Value *a = Nucleus::createInsertElement(loadValue(), x, 0);
4978
4979                 Value *e;
4980                 int swizzle[16] = {0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23};
4981                 Value *b = Nucleus::createBitCast(a, Byte16::getType());
4982                 Value *c = Nucleus::createShuffleVector(b, V(Nucleus::createNullValue(Byte16::getType())), swizzle);
4983
4984                 int swizzle2[8] = {0, 8, 1, 9, 2, 10, 3, 11};
4985                 Value *d = Nucleus::createBitCast(c, Short8::getType());
4986                 e = Nucleus::createShuffleVector(d, V(Nucleus::createNullValue(Short8::getType())), swizzle2);
4987
4988                 Value *f = Nucleus::createBitCast(e, Int4::getType());
4989                 storeValue(f);
4990         }
4991
4992         Int4::Int4(RValue<SByte4> cast)
4993         {
4994                 Value *x = Nucleus::createBitCast(cast.value, Int::getType());
4995                 Value *a = Nucleus::createInsertElement(loadValue(), x, 0);
4996
4997                 Value *e;
4998                 int swizzle[16] = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7};
4999                 Value *b = Nucleus::createBitCast(a, Byte16::getType());
5000                 Value *c = Nucleus::createShuffleVector(b, b, swizzle);
5001
5002                 int swizzle2[8] = {0, 0, 1, 1, 2, 2, 3, 3};
5003                 Value *d = Nucleus::createBitCast(c, Short8::getType());
5004                 e = Nucleus::createShuffleVector(d, d, swizzle2);
5005
5006                 Value *f = Nucleus::createBitCast(e, Int4::getType());
5007                 Value *g = Nucleus::createAShr(f, V(::context->getConstantInt32(24)));
5008                 storeValue(g);
5009         }
5010
5011         Int4::Int4(RValue<Float4> cast)
5012         {
5013                 Value *xyzw = Nucleus::createFPToSI(cast.value, Int4::getType());
5014
5015                 storeValue(xyzw);
5016         }
5017
5018         Int4::Int4(RValue<Short4> cast)
5019         {
5020                 int swizzle[8] = {0, 0, 1, 1, 2, 2, 3, 3};
5021                 Value *c = Nucleus::createShuffleVector(cast.value, cast.value, swizzle);
5022                 Value *d = Nucleus::createBitCast(c, Int4::getType());
5023                 Value *e = Nucleus::createAShr(d, V(::context->getConstantInt32(16)));
5024                 storeValue(e);
5025         }
5026
5027         Int4::Int4(RValue<UShort4> cast)
5028         {
5029                 int swizzle[8] = {0, 8, 1, 9, 2, 10, 3, 11};
5030                 Value *c = Nucleus::createShuffleVector(cast.value, Short8(0, 0, 0, 0, 0, 0, 0, 0).loadValue(), swizzle);
5031                 Value *d = Nucleus::createBitCast(c, Int4::getType());
5032                 storeValue(d);
5033         }
5034
5035         Int4::Int4(int xyzw)
5036         {
5037                 constant(xyzw, xyzw, xyzw, xyzw);
5038         }
5039
5040         Int4::Int4(int x, int yzw)
5041         {
5042                 constant(x, yzw, yzw, yzw);
5043         }
5044
5045         Int4::Int4(int x, int y, int zw)
5046         {
5047                 constant(x, y, zw, zw);
5048         }
5049
5050         Int4::Int4(int x, int y, int z, int w)
5051         {
5052                 constant(x, y, z, w);
5053         }
5054
5055         void Int4::constant(int x, int y, int z, int w)
5056         {
5057                 int64_t constantVector[4] = {x, y, z, w};
5058                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
5059         }
5060
5061         Int4::Int4(RValue<Int4> rhs)
5062         {
5063                 storeValue(rhs.value);
5064         }
5065
5066         Int4::Int4(const Int4 &rhs)
5067         {
5068                 Value *value = rhs.loadValue();
5069                 storeValue(value);
5070         }
5071
5072         Int4::Int4(const Reference<Int4> &rhs)
5073         {
5074                 Value *value = rhs.loadValue();
5075                 storeValue(value);
5076         }
5077
5078         Int4::Int4(RValue<UInt4> rhs)
5079         {
5080                 storeValue(rhs.value);
5081         }
5082
5083         Int4::Int4(const UInt4 &rhs)
5084         {
5085                 Value *value = rhs.loadValue();
5086                 storeValue(value);
5087         }
5088
5089         Int4::Int4(const Reference<UInt4> &rhs)
5090         {
5091                 Value *value = rhs.loadValue();
5092                 storeValue(value);
5093         }
5094
5095         Int4::Int4(RValue<Int2> lo, RValue<Int2> hi)
5096         {
5097                 int shuffle[4] = {0, 1, 4, 5};   // Real type is v4i32
5098                 Value *packed = Nucleus::createShuffleVector(lo.value, hi.value, shuffle);
5099
5100                 storeValue(packed);
5101         }
5102
5103         Int4::Int4(RValue<Int> rhs)
5104         {
5105                 Value *vector = Nucleus::createBitCast(rhs.value, Int4::getType());
5106
5107                 int swizzle[4] = {0, 0, 0, 0};
5108                 Value *replicate = Nucleus::createShuffleVector(vector, vector, swizzle);
5109
5110                 storeValue(replicate);
5111         }
5112
5113         Int4::Int4(const Int &rhs)
5114         {
5115                 *this = RValue<Int>(rhs.loadValue());
5116         }
5117
5118         Int4::Int4(const Reference<Int> &rhs)
5119         {
5120                 *this = RValue<Int>(rhs.loadValue());
5121         }
5122
5123         RValue<Int4> Int4::operator=(RValue<Int4> rhs)
5124         {
5125                 storeValue(rhs.value);
5126
5127                 return rhs;
5128         }
5129
5130         RValue<Int4> Int4::operator=(const Int4 &rhs)
5131         {
5132                 Value *value = rhs.loadValue();
5133                 storeValue(value);
5134
5135                 return RValue<Int4>(value);
5136         }
5137
5138         RValue<Int4> Int4::operator=(const Reference<Int4> &rhs)
5139         {
5140                 Value *value = rhs.loadValue();
5141                 storeValue(value);
5142
5143                 return RValue<Int4>(value);
5144         }
5145
5146         RValue<Int4> operator+(RValue<Int4> lhs, RValue<Int4> rhs)
5147         {
5148                 return RValue<Int4>(Nucleus::createAdd(lhs.value, rhs.value));
5149         }
5150
5151         RValue<Int4> operator-(RValue<Int4> lhs, RValue<Int4> rhs)
5152         {
5153                 return RValue<Int4>(Nucleus::createSub(lhs.value, rhs.value));
5154         }
5155
5156         RValue<Int4> operator*(RValue<Int4> lhs, RValue<Int4> rhs)
5157         {
5158                 return RValue<Int4>(Nucleus::createMul(lhs.value, rhs.value));
5159         }
5160
5161         RValue<Int4> operator/(RValue<Int4> lhs, RValue<Int4> rhs)
5162         {
5163                 return RValue<Int4>(Nucleus::createSDiv(lhs.value, rhs.value));
5164         }
5165
5166         RValue<Int4> operator%(RValue<Int4> lhs, RValue<Int4> rhs)
5167         {
5168                 return RValue<Int4>(Nucleus::createSRem(lhs.value, rhs.value));
5169         }
5170
5171         RValue<Int4> operator&(RValue<Int4> lhs, RValue<Int4> rhs)
5172         {
5173                 return RValue<Int4>(Nucleus::createAnd(lhs.value, rhs.value));
5174         }
5175
5176         RValue<Int4> operator|(RValue<Int4> lhs, RValue<Int4> rhs)
5177         {
5178                 return RValue<Int4>(Nucleus::createOr(lhs.value, rhs.value));
5179         }
5180
5181         RValue<Int4> operator^(RValue<Int4> lhs, RValue<Int4> rhs)
5182         {
5183                 return RValue<Int4>(Nucleus::createXor(lhs.value, rhs.value));
5184         }
5185
5186         RValue<Int4> operator<<(RValue<Int4> lhs, unsigned char rhs)
5187         {
5188                 return RValue<Int4>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
5189         }
5190
5191         RValue<Int4> operator>>(RValue<Int4> lhs, unsigned char rhs)
5192         {
5193                 return RValue<Int4>(Nucleus::createAShr(lhs.value, V(::context->getConstantInt32(rhs))));
5194         }
5195
5196         RValue<Int4> operator<<(RValue<Int4> lhs, RValue<Int4> rhs)
5197         {
5198                 return RValue<Int4>(Nucleus::createShl(lhs.value, rhs.value));
5199         }
5200
5201         RValue<Int4> operator>>(RValue<Int4> lhs, RValue<Int4> rhs)
5202         {
5203                 return RValue<Int4>(Nucleus::createAShr(lhs.value, rhs.value));
5204         }
5205
5206         RValue<Int4> operator+=(Int4 &lhs, RValue<Int4> rhs)
5207         {
5208                 return lhs = lhs + rhs;
5209         }
5210
5211         RValue<Int4> operator-=(Int4 &lhs, RValue<Int4> rhs)
5212         {
5213                 return lhs = lhs - rhs;
5214         }
5215
5216         RValue<Int4> operator*=(Int4 &lhs, RValue<Int4> rhs)
5217         {
5218                 return lhs = lhs * rhs;
5219         }
5220
5221 //      RValue<Int4> operator/=(Int4 &lhs, RValue<Int4> rhs)
5222 //      {
5223 //              return lhs = lhs / rhs;
5224 //      }
5225
5226 //      RValue<Int4> operator%=(Int4 &lhs, RValue<Int4> rhs)
5227 //      {
5228 //              return lhs = lhs % rhs;
5229 //      }
5230
5231         RValue<Int4> operator&=(Int4 &lhs, RValue<Int4> rhs)
5232         {
5233                 return lhs = lhs & rhs;
5234         }
5235
5236         RValue<Int4> operator|=(Int4 &lhs, RValue<Int4> rhs)
5237         {
5238                 return lhs = lhs | rhs;
5239         }
5240
5241         RValue<Int4> operator^=(Int4 &lhs, RValue<Int4> rhs)
5242         {
5243                 return lhs = lhs ^ rhs;
5244         }
5245
5246         RValue<Int4> operator<<=(Int4 &lhs, unsigned char rhs)
5247         {
5248                 return lhs = lhs << rhs;
5249         }
5250
5251         RValue<Int4> operator>>=(Int4 &lhs, unsigned char rhs)
5252         {
5253                 return lhs = lhs >> rhs;
5254         }
5255
5256         RValue<Int4> operator+(RValue<Int4> val)
5257         {
5258                 return val;
5259         }
5260
5261         RValue<Int4> operator-(RValue<Int4> val)
5262         {
5263                 return RValue<Int4>(Nucleus::createNeg(val.value));
5264         }
5265
5266         RValue<Int4> operator~(RValue<Int4> val)
5267         {
5268                 return RValue<Int4>(Nucleus::createNot(val.value));
5269         }
5270
5271         RValue<Int4> CmpEQ(RValue<Int4> x, RValue<Int4> y)
5272         {
5273                 return RValue<Int4>(Nucleus::createICmpEQ(x.value, y.value));
5274         }
5275
5276         RValue<Int4> CmpLT(RValue<Int4> x, RValue<Int4> y)
5277         {
5278                 return RValue<Int4>(Nucleus::createICmpSLT(x.value, y.value));
5279         }
5280
5281         RValue<Int4> CmpLE(RValue<Int4> x, RValue<Int4> y)
5282         {
5283                 return RValue<Int4>(Nucleus::createICmpSLE(x.value, y.value));
5284         }
5285
5286         RValue<Int4> CmpNEQ(RValue<Int4> x, RValue<Int4> y)
5287         {
5288                 return RValue<Int4>(Nucleus::createICmpNE(x.value, y.value));
5289         }
5290
5291         RValue<Int4> CmpNLT(RValue<Int4> x, RValue<Int4> y)
5292         {
5293                 return RValue<Int4>(Nucleus::createICmpSGE(x.value, y.value));
5294         }
5295
5296         RValue<Int4> CmpNLE(RValue<Int4> x, RValue<Int4> y)
5297         {
5298                 return RValue<Int4>(Nucleus::createICmpSGT(x.value, y.value));
5299         }
5300
5301         RValue<Int4> Max(RValue<Int4> x, RValue<Int4> y)
5302         {
5303                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v4i1);
5304                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Sle, condition, x.value, y.value);
5305                 ::basicBlock->appendInst(cmp);
5306
5307                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4i32);
5308                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
5309                 ::basicBlock->appendInst(select);
5310
5311                 return RValue<Int4>(V(result));
5312         }
5313
5314         RValue<Int4> Min(RValue<Int4> x, RValue<Int4> y)
5315         {
5316                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v4i1);
5317                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Sgt, condition, x.value, y.value);
5318                 ::basicBlock->appendInst(cmp);
5319
5320                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4i32);
5321                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
5322                 ::basicBlock->appendInst(select);
5323
5324                 return RValue<Int4>(V(result));
5325         }
5326
5327         RValue<Int4> RoundInt(RValue<Float4> cast)
5328         {
5329                 if(emulateIntrinsics)
5330                 {
5331                         // Push the fractional part off the mantissa. Accurate up to +/-2^22.
5332                         return Int4((cast + Float4(0x00C00000)) - Float4(0x00C00000));
5333                 }
5334                 else
5335                 {
5336                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4i32);
5337                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Nearbyint, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
5338                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
5339                         auto nearbyint = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
5340                         nearbyint->addArg(cast.value);
5341                         ::basicBlock->appendInst(nearbyint);
5342
5343                         return RValue<Int4>(V(result));
5344                 }
5345         }
5346
5347         RValue<Short8> Pack(RValue<Int4> x, RValue<Int4> y)
5348         {
5349                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
5350                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::VectorPackSigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
5351                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
5352                 auto pack = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
5353                 pack->addArg(x.value);
5354                 pack->addArg(y.value);
5355                 ::basicBlock->appendInst(pack);
5356
5357                 return RValue<Short8>(V(result));
5358         }
5359
5360         RValue<Int> Extract(RValue<Int4> x, int i)
5361         {
5362                 return RValue<Int>(Nucleus::createExtractElement(x.value, Int::getType(), i));
5363         }
5364
5365         RValue<Int4> Insert(RValue<Int4> x, RValue<Int> element, int i)
5366         {
5367                 return RValue<Int4>(Nucleus::createInsertElement(x.value, element.value, i));
5368         }
5369
5370         RValue<Int> SignMask(RValue<Int4> x)
5371         {
5372                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_i32);
5373                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SignMask, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
5374                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
5375                 auto movmsk = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
5376                 movmsk->addArg(x.value);
5377                 ::basicBlock->appendInst(movmsk);
5378
5379                 return RValue<Int>(V(result));
5380         }
5381
5382         RValue<Int4> Swizzle(RValue<Int4> x, unsigned char select)
5383         {
5384                 return RValue<Int4>(createSwizzle4(x.value, select));
5385         }
5386
5387         Type *Int4::getType()
5388         {
5389                 return T(Ice::IceType_v4i32);
5390         }
5391
5392         UInt4::UInt4(RValue<Float4> cast)
5393         {
5394                 // Smallest positive value representable in UInt, but not in Int
5395                 const unsigned int ustart = 0x80000000u;
5396                 const float ustartf = float(ustart);
5397
5398                 // Check if the value can be represented as an Int
5399                 Int4 uiValue = CmpNLT(cast, Float4(ustartf));
5400                 // If the value is too large, subtract ustart and re-add it after conversion.
5401                 uiValue = (uiValue & As<Int4>(As<UInt4>(Int4(cast - Float4(ustartf))) + UInt4(ustart))) |
5402                 // Otherwise, just convert normally
5403                           (~uiValue & Int4(cast));
5404                 // If the value is negative, store 0, otherwise store the result of the conversion
5405                 storeValue((~(As<Int4>(cast) >> 31) & uiValue).value);
5406         }
5407
5408         UInt4::UInt4(int xyzw)
5409         {
5410                 constant(xyzw, xyzw, xyzw, xyzw);
5411         }
5412
5413         UInt4::UInt4(int x, int yzw)
5414         {
5415                 constant(x, yzw, yzw, yzw);
5416         }
5417
5418         UInt4::UInt4(int x, int y, int zw)
5419         {
5420                 constant(x, y, zw, zw);
5421         }
5422
5423         UInt4::UInt4(int x, int y, int z, int w)
5424         {
5425                 constant(x, y, z, w);
5426         }
5427
5428         void UInt4::constant(int x, int y, int z, int w)
5429         {
5430                 int64_t constantVector[4] = {x, y, z, w};
5431                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
5432         }
5433
5434         UInt4::UInt4(RValue<UInt4> rhs)
5435         {
5436                 storeValue(rhs.value);
5437         }
5438
5439         UInt4::UInt4(const UInt4 &rhs)
5440         {
5441                 Value *value = rhs.loadValue();
5442                 storeValue(value);
5443         }
5444
5445         UInt4::UInt4(const Reference<UInt4> &rhs)
5446         {
5447                 Value *value = rhs.loadValue();
5448                 storeValue(value);
5449         }
5450
5451         UInt4::UInt4(RValue<Int4> rhs)
5452         {
5453                 storeValue(rhs.value);
5454         }
5455
5456         UInt4::UInt4(const Int4 &rhs)
5457         {
5458                 Value *value = rhs.loadValue();
5459                 storeValue(value);
5460         }
5461
5462         UInt4::UInt4(const Reference<Int4> &rhs)
5463         {
5464                 Value *value = rhs.loadValue();
5465                 storeValue(value);
5466         }
5467
5468         UInt4::UInt4(RValue<UInt2> lo, RValue<UInt2> hi)
5469         {
5470                 int shuffle[4] = {0, 1, 4, 5};   // Real type is v4i32
5471                 Value *packed = Nucleus::createShuffleVector(lo.value, hi.value, shuffle);
5472
5473                 storeValue(packed);
5474         }
5475
5476         RValue<UInt4> UInt4::operator=(RValue<UInt4> rhs)
5477         {
5478                 storeValue(rhs.value);
5479
5480                 return rhs;
5481         }
5482
5483         RValue<UInt4> UInt4::operator=(const UInt4 &rhs)
5484         {
5485                 Value *value = rhs.loadValue();
5486                 storeValue(value);
5487
5488                 return RValue<UInt4>(value);
5489         }
5490
5491         RValue<UInt4> UInt4::operator=(const Reference<UInt4> &rhs)
5492         {
5493                 Value *value = rhs.loadValue();
5494                 storeValue(value);
5495
5496                 return RValue<UInt4>(value);
5497         }
5498
5499         RValue<UInt4> operator+(RValue<UInt4> lhs, RValue<UInt4> rhs)
5500         {
5501                 return RValue<UInt4>(Nucleus::createAdd(lhs.value, rhs.value));
5502         }
5503
5504         RValue<UInt4> operator-(RValue<UInt4> lhs, RValue<UInt4> rhs)
5505         {
5506                 return RValue<UInt4>(Nucleus::createSub(lhs.value, rhs.value));
5507         }
5508
5509         RValue<UInt4> operator*(RValue<UInt4> lhs, RValue<UInt4> rhs)
5510         {
5511                 return RValue<UInt4>(Nucleus::createMul(lhs.value, rhs.value));
5512         }
5513
5514         RValue<UInt4> operator/(RValue<UInt4> lhs, RValue<UInt4> rhs)
5515         {
5516                 return RValue<UInt4>(Nucleus::createUDiv(lhs.value, rhs.value));
5517         }
5518
5519         RValue<UInt4> operator%(RValue<UInt4> lhs, RValue<UInt4> rhs)
5520         {
5521                 return RValue<UInt4>(Nucleus::createURem(lhs.value, rhs.value));
5522         }
5523
5524         RValue<UInt4> operator&(RValue<UInt4> lhs, RValue<UInt4> rhs)
5525         {
5526                 return RValue<UInt4>(Nucleus::createAnd(lhs.value, rhs.value));
5527         }
5528
5529         RValue<UInt4> operator|(RValue<UInt4> lhs, RValue<UInt4> rhs)
5530         {
5531                 return RValue<UInt4>(Nucleus::createOr(lhs.value, rhs.value));
5532         }
5533
5534         RValue<UInt4> operator^(RValue<UInt4> lhs, RValue<UInt4> rhs)
5535         {
5536                 return RValue<UInt4>(Nucleus::createXor(lhs.value, rhs.value));
5537         }
5538
5539         RValue<UInt4> operator<<(RValue<UInt4> lhs, unsigned char rhs)
5540         {
5541                 return RValue<UInt4>(Nucleus::createShl(lhs.value, V(::context->getConstantInt32(rhs))));
5542         }
5543
5544         RValue<UInt4> operator>>(RValue<UInt4> lhs, unsigned char rhs)
5545         {
5546                 return RValue<UInt4>(Nucleus::createLShr(lhs.value, V(::context->getConstantInt32(rhs))));
5547         }
5548
5549         RValue<UInt4> operator<<(RValue<UInt4> lhs, RValue<UInt4> rhs)
5550         {
5551                 return RValue<UInt4>(Nucleus::createShl(lhs.value, rhs.value));
5552         }
5553
5554         RValue<UInt4> operator>>(RValue<UInt4> lhs, RValue<UInt4> rhs)
5555         {
5556                 return RValue<UInt4>(Nucleus::createLShr(lhs.value, rhs.value));
5557         }
5558
5559         RValue<UInt4> operator+=(UInt4 &lhs, RValue<UInt4> rhs)
5560         {
5561                 return lhs = lhs + rhs;
5562         }
5563
5564         RValue<UInt4> operator-=(UInt4 &lhs, RValue<UInt4> rhs)
5565         {
5566                 return lhs = lhs - rhs;
5567         }
5568
5569         RValue<UInt4> operator*=(UInt4 &lhs, RValue<UInt4> rhs)
5570         {
5571                 return lhs = lhs * rhs;
5572         }
5573
5574 //      RValue<UInt4> operator/=(UInt4 &lhs, RValue<UInt4> rhs)
5575 //      {
5576 //              return lhs = lhs / rhs;
5577 //      }
5578
5579 //      RValue<UInt4> operator%=(UInt4 &lhs, RValue<UInt4> rhs)
5580 //      {
5581 //              return lhs = lhs % rhs;
5582 //      }
5583
5584         RValue<UInt4> operator&=(UInt4 &lhs, RValue<UInt4> rhs)
5585         {
5586                 return lhs = lhs & rhs;
5587         }
5588
5589         RValue<UInt4> operator|=(UInt4 &lhs, RValue<UInt4> rhs)
5590         {
5591                 return lhs = lhs | rhs;
5592         }
5593
5594         RValue<UInt4> operator^=(UInt4 &lhs, RValue<UInt4> rhs)
5595         {
5596                 return lhs = lhs ^ rhs;
5597         }
5598
5599         RValue<UInt4> operator<<=(UInt4 &lhs, unsigned char rhs)
5600         {
5601                 return lhs = lhs << rhs;
5602         }
5603
5604         RValue<UInt4> operator>>=(UInt4 &lhs, unsigned char rhs)
5605         {
5606                 return lhs = lhs >> rhs;
5607         }
5608
5609         RValue<UInt4> operator+(RValue<UInt4> val)
5610         {
5611                 return val;
5612         }
5613
5614         RValue<UInt4> operator-(RValue<UInt4> val)
5615         {
5616                 return RValue<UInt4>(Nucleus::createNeg(val.value));
5617         }
5618
5619         RValue<UInt4> operator~(RValue<UInt4> val)
5620         {
5621                 return RValue<UInt4>(Nucleus::createNot(val.value));
5622         }
5623
5624         RValue<UInt4> CmpEQ(RValue<UInt4> x, RValue<UInt4> y)
5625         {
5626                 return RValue<UInt4>(Nucleus::createICmpEQ(x.value, y.value));
5627         }
5628
5629         RValue<UInt4> CmpLT(RValue<UInt4> x, RValue<UInt4> y)
5630         {
5631                 return RValue<UInt4>(Nucleus::createICmpULT(x.value, y.value));
5632         }
5633
5634         RValue<UInt4> CmpLE(RValue<UInt4> x, RValue<UInt4> y)
5635         {
5636                 return RValue<UInt4>(Nucleus::createICmpULE(x.value, y.value));
5637         }
5638
5639         RValue<UInt4> CmpNEQ(RValue<UInt4> x, RValue<UInt4> y)
5640         {
5641                 return RValue<UInt4>(Nucleus::createICmpNE(x.value, y.value));
5642         }
5643
5644         RValue<UInt4> CmpNLT(RValue<UInt4> x, RValue<UInt4> y)
5645         {
5646                 return RValue<UInt4>(Nucleus::createICmpUGE(x.value, y.value));
5647         }
5648
5649         RValue<UInt4> CmpNLE(RValue<UInt4> x, RValue<UInt4> y)
5650         {
5651                 return RValue<UInt4>(Nucleus::createICmpUGT(x.value, y.value));
5652         }
5653
5654         RValue<UInt4> Max(RValue<UInt4> x, RValue<UInt4> y)
5655         {
5656                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v4i1);
5657                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Ule, condition, x.value, y.value);
5658                 ::basicBlock->appendInst(cmp);
5659
5660                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4i32);
5661                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
5662                 ::basicBlock->appendInst(select);
5663
5664                 return RValue<UInt4>(V(result));
5665         }
5666
5667         RValue<UInt4> Min(RValue<UInt4> x, RValue<UInt4> y)
5668         {
5669                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v4i1);
5670                 auto cmp = Ice::InstIcmp::create(::function, Ice::InstIcmp::Ugt, condition, x.value, y.value);
5671                 ::basicBlock->appendInst(cmp);
5672
5673                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4i32);
5674                 auto select = Ice::InstSelect::create(::function, result, condition, y.value, x.value);
5675                 ::basicBlock->appendInst(select);
5676
5677                 return RValue<UInt4>(V(result));
5678         }
5679
5680         RValue<UShort8> Pack(RValue<UInt4> x, RValue<UInt4> y)
5681         {
5682                 if(CPUID::SSE4_1)
5683                 {
5684                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v8i16);
5685                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::VectorPackUnsigned, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
5686                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
5687                         auto pack = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
5688                         pack->addArg(x.value);
5689                         pack->addArg(y.value);
5690                         ::basicBlock->appendInst(pack);
5691
5692                         return RValue<UShort8>(V(result));
5693                 }
5694                 else
5695                 {
5696                         RValue<Int4> sx = As<Int4>(x);
5697                         RValue<Int4> bx = (sx & ~(sx >> 31)) - Int4(0x8000);
5698
5699                         RValue<Int4> sy = As<Int4>(y);
5700                         RValue<Int4> by = (sy & ~(sy >> 31)) - Int4(0x8000);
5701
5702                         return As<UShort8>(Pack(bx, by) + Short8(0x8000u));
5703                 }
5704         }
5705
5706         Type *UInt4::getType()
5707         {
5708                 return T(Ice::IceType_v4i32);
5709         }
5710
5711         Float::Float(RValue<Int> cast)
5712         {
5713                 Value *integer = Nucleus::createSIToFP(cast.value, Float::getType());
5714
5715                 storeValue(integer);
5716         }
5717
5718         Float::Float(RValue<UInt> cast)
5719         {
5720                 RValue<Float> result = Float(Int(cast & UInt(0x7FFFFFFF))) +
5721                                        As<Float>((As<Int>(cast) >> 31) & As<Int>(Float(0x80000000u)));
5722
5723                 storeValue(result.value);
5724         }
5725
5726         Float::Float(float x)
5727         {
5728                 storeValue(Nucleus::createConstantFloat(x));
5729         }
5730
5731         Float::Float(RValue<Float> rhs)
5732         {
5733                 storeValue(rhs.value);
5734         }
5735
5736         Float::Float(const Float &rhs)
5737         {
5738                 Value *value = rhs.loadValue();
5739                 storeValue(value);
5740         }
5741
5742         Float::Float(const Reference<Float> &rhs)
5743         {
5744                 Value *value = rhs.loadValue();
5745                 storeValue(value);
5746         }
5747
5748         RValue<Float> Float::operator=(RValue<Float> rhs)
5749         {
5750                 storeValue(rhs.value);
5751
5752                 return rhs;
5753         }
5754
5755         RValue<Float> Float::operator=(const Float &rhs)
5756         {
5757                 Value *value = rhs.loadValue();
5758                 storeValue(value);
5759
5760                 return RValue<Float>(value);
5761         }
5762
5763         RValue<Float> Float::operator=(const Reference<Float> &rhs)
5764         {
5765                 Value *value = rhs.loadValue();
5766                 storeValue(value);
5767
5768                 return RValue<Float>(value);
5769         }
5770
5771         RValue<Float> operator+(RValue<Float> lhs, RValue<Float> rhs)
5772         {
5773                 return RValue<Float>(Nucleus::createFAdd(lhs.value, rhs.value));
5774         }
5775
5776         RValue<Float> operator-(RValue<Float> lhs, RValue<Float> rhs)
5777         {
5778                 return RValue<Float>(Nucleus::createFSub(lhs.value, rhs.value));
5779         }
5780
5781         RValue<Float> operator*(RValue<Float> lhs, RValue<Float> rhs)
5782         {
5783                 return RValue<Float>(Nucleus::createFMul(lhs.value, rhs.value));
5784         }
5785
5786         RValue<Float> operator/(RValue<Float> lhs, RValue<Float> rhs)
5787         {
5788                 return RValue<Float>(Nucleus::createFDiv(lhs.value, rhs.value));
5789         }
5790
5791         RValue<Float> operator+=(Float &lhs, RValue<Float> rhs)
5792         {
5793                 return lhs = lhs + rhs;
5794         }
5795
5796         RValue<Float> operator-=(Float &lhs, RValue<Float> rhs)
5797         {
5798                 return lhs = lhs - rhs;
5799         }
5800
5801         RValue<Float> operator*=(Float &lhs, RValue<Float> rhs)
5802         {
5803                 return lhs = lhs * rhs;
5804         }
5805
5806         RValue<Float> operator/=(Float &lhs, RValue<Float> rhs)
5807         {
5808                 return lhs = lhs / rhs;
5809         }
5810
5811         RValue<Float> operator+(RValue<Float> val)
5812         {
5813                 return val;
5814         }
5815
5816         RValue<Float> operator-(RValue<Float> val)
5817         {
5818                 return RValue<Float>(Nucleus::createFNeg(val.value));
5819         }
5820
5821         RValue<Bool> operator<(RValue<Float> lhs, RValue<Float> rhs)
5822         {
5823                 return RValue<Bool>(Nucleus::createFCmpOLT(lhs.value, rhs.value));
5824         }
5825
5826         RValue<Bool> operator<=(RValue<Float> lhs, RValue<Float> rhs)
5827         {
5828                 return RValue<Bool>(Nucleus::createFCmpOLE(lhs.value, rhs.value));
5829         }
5830
5831         RValue<Bool> operator>(RValue<Float> lhs, RValue<Float> rhs)
5832         {
5833                 return RValue<Bool>(Nucleus::createFCmpOGT(lhs.value, rhs.value));
5834         }
5835
5836         RValue<Bool> operator>=(RValue<Float> lhs, RValue<Float> rhs)
5837         {
5838                 return RValue<Bool>(Nucleus::createFCmpOGE(lhs.value, rhs.value));
5839         }
5840
5841         RValue<Bool> operator!=(RValue<Float> lhs, RValue<Float> rhs)
5842         {
5843                 return RValue<Bool>(Nucleus::createFCmpONE(lhs.value, rhs.value));
5844         }
5845
5846         RValue<Bool> operator==(RValue<Float> lhs, RValue<Float> rhs)
5847         {
5848                 return RValue<Bool>(Nucleus::createFCmpOEQ(lhs.value, rhs.value));
5849         }
5850
5851         RValue<Float> Abs(RValue<Float> x)
5852         {
5853                 return IfThenElse(x > 0.0f, x, -x);
5854         }
5855
5856         RValue<Float> Max(RValue<Float> x, RValue<Float> y)
5857         {
5858                 return IfThenElse(x > y, x, y);
5859         }
5860
5861         RValue<Float> Min(RValue<Float> x, RValue<Float> y)
5862         {
5863                 return IfThenElse(x < y, x, y);
5864         }
5865
5866         RValue<Float> Rcp_pp(RValue<Float> x, bool exactAtPow2)
5867         {
5868                 return 1.0f / x;
5869         }
5870
5871         RValue<Float> RcpSqrt_pp(RValue<Float> x)
5872         {
5873                 return Rcp_pp(Sqrt(x));
5874         }
5875
5876         RValue<Float> Sqrt(RValue<Float> x)
5877         {
5878                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_f32);
5879                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Sqrt, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
5880                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
5881                 auto sqrt = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
5882                 sqrt->addArg(x.value);
5883                 ::basicBlock->appendInst(sqrt);
5884
5885                 return RValue<Float>(V(result));
5886         }
5887
5888         RValue<Float> Round(RValue<Float> x)
5889         {
5890                 return Float4(Round(Float4(x))).x;
5891         }
5892
5893         RValue<Float> Trunc(RValue<Float> x)
5894         {
5895                 return Float4(Trunc(Float4(x))).x;
5896         }
5897
5898         RValue<Float> Frac(RValue<Float> x)
5899         {
5900                 return Float4(Frac(Float4(x))).x;
5901         }
5902
5903         RValue<Float> Floor(RValue<Float> x)
5904         {
5905                 return Float4(Floor(Float4(x))).x;
5906         }
5907
5908         RValue<Float> Ceil(RValue<Float> x)
5909         {
5910                 return Float4(Ceil(Float4(x))).x;
5911         }
5912
5913         Type *Float::getType()
5914         {
5915                 return T(Ice::IceType_f32);
5916         }
5917
5918         Float2::Float2(RValue<Float4> cast)
5919         {
5920                 storeValue(Nucleus::createBitCast(cast.value, getType()));
5921         }
5922
5923         Type *Float2::getType()
5924         {
5925                 return T(Type_v2f32);
5926         }
5927
5928         Float4::Float4(RValue<Byte4> cast) : FloatXYZW(this)
5929         {
5930                 Value *a = Int4(cast).loadValue();
5931                 Value *xyzw = Nucleus::createSIToFP(a, Float4::getType());
5932
5933                 storeValue(xyzw);
5934         }
5935
5936         Float4::Float4(RValue<SByte4> cast) : FloatXYZW(this)
5937         {
5938                 Value *a = Int4(cast).loadValue();
5939                 Value *xyzw = Nucleus::createSIToFP(a, Float4::getType());
5940
5941                 storeValue(xyzw);
5942         }
5943
5944         Float4::Float4(RValue<Short4> cast) : FloatXYZW(this)
5945         {
5946                 Int4 c(cast);
5947                 storeValue(Nucleus::createSIToFP(RValue<Int4>(c).value, Float4::getType()));
5948         }
5949
5950         Float4::Float4(RValue<UShort4> cast) : FloatXYZW(this)
5951         {
5952                 Int4 c(cast);
5953                 storeValue(Nucleus::createSIToFP(RValue<Int4>(c).value, Float4::getType()));
5954         }
5955
5956         Float4::Float4(RValue<Int4> cast) : FloatXYZW(this)
5957         {
5958                 Value *xyzw = Nucleus::createSIToFP(cast.value, Float4::getType());
5959
5960                 storeValue(xyzw);
5961         }
5962
5963         Float4::Float4(RValue<UInt4> cast) : FloatXYZW(this)
5964         {
5965                 RValue<Float4> result = Float4(Int4(cast & UInt4(0x7FFFFFFF))) +
5966                                         As<Float4>((As<Int4>(cast) >> 31) & As<Int4>(Float4(0x80000000u)));
5967
5968                 storeValue(result.value);
5969         }
5970
5971         Float4::Float4() : FloatXYZW(this)
5972         {
5973         }
5974
5975         Float4::Float4(float xyzw) : FloatXYZW(this)
5976         {
5977                 constant(xyzw, xyzw, xyzw, xyzw);
5978         }
5979
5980         Float4::Float4(float x, float yzw) : FloatXYZW(this)
5981         {
5982                 constant(x, yzw, yzw, yzw);
5983         }
5984
5985         Float4::Float4(float x, float y, float zw) : FloatXYZW(this)
5986         {
5987                 constant(x, y, zw, zw);
5988         }
5989
5990         Float4::Float4(float x, float y, float z, float w) : FloatXYZW(this)
5991         {
5992                 constant(x, y, z, w);
5993         }
5994
5995         void Float4::constant(float x, float y, float z, float w)
5996         {
5997                 double constantVector[4] = {x, y, z, w};
5998                 storeValue(Nucleus::createConstantVector(constantVector, getType()));
5999         }
6000
6001         Float4::Float4(RValue<Float4> rhs) : FloatXYZW(this)
6002         {
6003                 storeValue(rhs.value);
6004         }
6005
6006         Float4::Float4(const Float4 &rhs) : FloatXYZW(this)
6007         {
6008                 Value *value = rhs.loadValue();
6009                 storeValue(value);
6010         }
6011
6012         Float4::Float4(const Reference<Float4> &rhs) : FloatXYZW(this)
6013         {
6014                 Value *value = rhs.loadValue();
6015                 storeValue(value);
6016         }
6017
6018         Float4::Float4(RValue<Float> rhs) : FloatXYZW(this)
6019         {
6020                 Value *vector = Nucleus::createBitCast(rhs.value, Float4::getType());
6021
6022                 int swizzle[4] = {0, 0, 0, 0};
6023                 Value *replicate = Nucleus::createShuffleVector(vector, vector, swizzle);
6024
6025                 storeValue(replicate);
6026         }
6027
6028         Float4::Float4(const Float &rhs) : FloatXYZW(this)
6029         {
6030                 *this = RValue<Float>(rhs.loadValue());
6031         }
6032
6033         Float4::Float4(const Reference<Float> &rhs) : FloatXYZW(this)
6034         {
6035                 *this = RValue<Float>(rhs.loadValue());
6036         }
6037
6038         RValue<Float4> Float4::operator=(float x)
6039         {
6040                 return *this = Float4(x, x, x, x);
6041         }
6042
6043         RValue<Float4> Float4::operator=(RValue<Float4> rhs)
6044         {
6045                 storeValue(rhs.value);
6046
6047                 return rhs;
6048         }
6049
6050         RValue<Float4> Float4::operator=(const Float4 &rhs)
6051         {
6052                 Value *value = rhs.loadValue();
6053                 storeValue(value);
6054
6055                 return RValue<Float4>(value);
6056         }
6057
6058         RValue<Float4> Float4::operator=(const Reference<Float4> &rhs)
6059         {
6060                 Value *value = rhs.loadValue();
6061                 storeValue(value);
6062
6063                 return RValue<Float4>(value);
6064         }
6065
6066         RValue<Float4> Float4::operator=(RValue<Float> rhs)
6067         {
6068                 return *this = Float4(rhs);
6069         }
6070
6071         RValue<Float4> Float4::operator=(const Float &rhs)
6072         {
6073                 return *this = Float4(rhs);
6074         }
6075
6076         RValue<Float4> Float4::operator=(const Reference<Float> &rhs)
6077         {
6078                 return *this = Float4(rhs);
6079         }
6080
6081         RValue<Float4> operator+(RValue<Float4> lhs, RValue<Float4> rhs)
6082         {
6083                 return RValue<Float4>(Nucleus::createFAdd(lhs.value, rhs.value));
6084         }
6085
6086         RValue<Float4> operator-(RValue<Float4> lhs, RValue<Float4> rhs)
6087         {
6088                 return RValue<Float4>(Nucleus::createFSub(lhs.value, rhs.value));
6089         }
6090
6091         RValue<Float4> operator*(RValue<Float4> lhs, RValue<Float4> rhs)
6092         {
6093                 return RValue<Float4>(Nucleus::createFMul(lhs.value, rhs.value));
6094         }
6095
6096         RValue<Float4> operator/(RValue<Float4> lhs, RValue<Float4> rhs)
6097         {
6098                 return RValue<Float4>(Nucleus::createFDiv(lhs.value, rhs.value));
6099         }
6100
6101         RValue<Float4> operator%(RValue<Float4> lhs, RValue<Float4> rhs)
6102         {
6103                 return RValue<Float4>(Nucleus::createFRem(lhs.value, rhs.value));
6104         }
6105
6106         RValue<Float4> operator+=(Float4 &lhs, RValue<Float4> rhs)
6107         {
6108                 return lhs = lhs + rhs;
6109         }
6110
6111         RValue<Float4> operator-=(Float4 &lhs, RValue<Float4> rhs)
6112         {
6113                 return lhs = lhs - rhs;
6114         }
6115
6116         RValue<Float4> operator*=(Float4 &lhs, RValue<Float4> rhs)
6117         {
6118                 return lhs = lhs * rhs;
6119         }
6120
6121         RValue<Float4> operator/=(Float4 &lhs, RValue<Float4> rhs)
6122         {
6123                 return lhs = lhs / rhs;
6124         }
6125
6126         RValue<Float4> operator%=(Float4 &lhs, RValue<Float4> rhs)
6127         {
6128                 return lhs = lhs % rhs;
6129         }
6130
6131         RValue<Float4> operator+(RValue<Float4> val)
6132         {
6133                 return val;
6134         }
6135
6136         RValue<Float4> operator-(RValue<Float4> val)
6137         {
6138                 return RValue<Float4>(Nucleus::createFNeg(val.value));
6139         }
6140
6141         RValue<Float4> Abs(RValue<Float4> x)
6142         {
6143                 Value *vector = Nucleus::createBitCast(x.value, Int4::getType());
6144                 int64_t constantVector[4] = {0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF};
6145                 Value *result = Nucleus::createAnd(vector, V(Nucleus::createConstantVector(constantVector, Int4::getType())));
6146
6147                 return As<Float4>(result);
6148         }
6149
6150         RValue<Float4> Max(RValue<Float4> x, RValue<Float4> y)
6151         {
6152                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v4i1);
6153                 auto cmp = Ice::InstFcmp::create(::function, Ice::InstFcmp::Ogt, condition, x.value, y.value);
6154                 ::basicBlock->appendInst(cmp);
6155
6156                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
6157                 auto select = Ice::InstSelect::create(::function, result, condition, x.value, y.value);
6158                 ::basicBlock->appendInst(select);
6159
6160                 return RValue<Float4>(V(result));
6161         }
6162
6163         RValue<Float4> Min(RValue<Float4> x, RValue<Float4> y)
6164         {
6165                 Ice::Variable *condition = ::function->makeVariable(Ice::IceType_v4i1);
6166                 auto cmp = Ice::InstFcmp::create(::function, Ice::InstFcmp::Olt, condition, x.value, y.value);
6167                 ::basicBlock->appendInst(cmp);
6168
6169                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
6170                 auto select = Ice::InstSelect::create(::function, result, condition, x.value, y.value);
6171                 ::basicBlock->appendInst(select);
6172
6173                 return RValue<Float4>(V(result));
6174         }
6175
6176         RValue<Float4> Rcp_pp(RValue<Float4> x, bool exactAtPow2)
6177         {
6178                 return Float4(1.0f) / x;
6179         }
6180
6181         RValue<Float4> RcpSqrt_pp(RValue<Float4> x)
6182         {
6183                 return Rcp_pp(Sqrt(x));
6184         }
6185
6186         RValue<Float4> Sqrt(RValue<Float4> x)
6187         {
6188                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
6189                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Sqrt, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
6190                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
6191                 auto sqrt = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
6192                 sqrt->addArg(x.value);
6193                 ::basicBlock->appendInst(sqrt);
6194
6195                 return RValue<Float4>(V(result));
6196         }
6197
6198         RValue<Float4> Insert(RValue<Float4> x, RValue<Float> element, int i)
6199         {
6200                 return RValue<Float4>(Nucleus::createInsertElement(x.value, element.value, i));
6201         }
6202
6203         RValue<Float> Extract(RValue<Float4> x, int i)
6204         {
6205                 return RValue<Float>(Nucleus::createExtractElement(x.value, Float::getType(), i));
6206         }
6207
6208         RValue<Float4> Swizzle(RValue<Float4> x, unsigned char select)
6209         {
6210                 return RValue<Float4>(createSwizzle4(x.value, select));
6211         }
6212
6213         RValue<Float4> ShuffleLowHigh(RValue<Float4> x, RValue<Float4> y, unsigned char imm)
6214         {
6215                 int shuffle[4] =
6216                 {
6217                         ((imm >> 0) & 0x03) + 0,
6218                         ((imm >> 2) & 0x03) + 0,
6219                         ((imm >> 4) & 0x03) + 4,
6220                         ((imm >> 6) & 0x03) + 4,
6221                 };
6222
6223                 return RValue<Float4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
6224         }
6225
6226         RValue<Float4> UnpackLow(RValue<Float4> x, RValue<Float4> y)
6227         {
6228                 int shuffle[4] = {0, 4, 1, 5};
6229                 return RValue<Float4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
6230         }
6231
6232         RValue<Float4> UnpackHigh(RValue<Float4> x, RValue<Float4> y)
6233         {
6234                 int shuffle[4] = {2, 6, 3, 7};
6235                 return RValue<Float4>(Nucleus::createShuffleVector(x.value, y.value, shuffle));
6236         }
6237
6238         RValue<Float4> Mask(Float4 &lhs, RValue<Float4> rhs, unsigned char select)
6239         {
6240                 Value *vector = lhs.loadValue();
6241                 Value *result = createMask4(vector, rhs.value, select);
6242                 lhs.storeValue(result);
6243
6244                 return RValue<Float4>(result);
6245         }
6246
6247         RValue<Int> SignMask(RValue<Float4> x)
6248         {
6249                 Ice::Variable *result = ::function->makeVariable(Ice::IceType_i32);
6250                 const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::SignMask, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
6251                 auto target = ::context->getConstantUndef(Ice::IceType_i32);
6252                 auto movmsk = Ice::InstIntrinsicCall::create(::function, 1, result, target, intrinsic);
6253                 movmsk->addArg(x.value);
6254                 ::basicBlock->appendInst(movmsk);
6255
6256                 return RValue<Int>(V(result));
6257         }
6258
6259         RValue<Int4> CmpEQ(RValue<Float4> x, RValue<Float4> y)
6260         {
6261                 return RValue<Int4>(Nucleus::createFCmpOEQ(x.value, y.value));
6262         }
6263
6264         RValue<Int4> CmpLT(RValue<Float4> x, RValue<Float4> y)
6265         {
6266                 return RValue<Int4>(Nucleus::createFCmpOLT(x.value, y.value));
6267         }
6268
6269         RValue<Int4> CmpLE(RValue<Float4> x, RValue<Float4> y)
6270         {
6271                 return RValue<Int4>(Nucleus::createFCmpOLE(x.value, y.value));
6272         }
6273
6274         RValue<Int4> CmpNEQ(RValue<Float4> x, RValue<Float4> y)
6275         {
6276                 return RValue<Int4>(Nucleus::createFCmpONE(x.value, y.value));
6277         }
6278
6279         RValue<Int4> CmpNLT(RValue<Float4> x, RValue<Float4> y)
6280         {
6281                 return RValue<Int4>(Nucleus::createFCmpOGE(x.value, y.value));
6282         }
6283
6284         RValue<Int4> CmpNLE(RValue<Float4> x, RValue<Float4> y)
6285         {
6286                 return RValue<Int4>(Nucleus::createFCmpOGT(x.value, y.value));
6287         }
6288
6289         RValue<Float4> Round(RValue<Float4> x)
6290         {
6291                 if(emulateIntrinsics)
6292                 {
6293                         // Push the fractional part off the mantissa. Accurate up to +/-2^22.
6294                         return (x + Float4(0x00C00000)) - Float4(0x00C00000);
6295                 }
6296                 else if(CPUID::SSE4_1)
6297                 {
6298                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
6299                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Round, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
6300                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
6301                         auto round = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
6302                         round->addArg(x.value);
6303                         round->addArg(::context->getConstantInt32(0));
6304                         ::basicBlock->appendInst(round);
6305
6306                         return RValue<Float4>(V(result));
6307                 }
6308                 else
6309                 {
6310                         return Float4(RoundInt(x));
6311                 }
6312         }
6313
6314         RValue<Float4> Trunc(RValue<Float4> x)
6315         {
6316                 if(CPUID::SSE4_1)
6317                 {
6318                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
6319                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Round, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
6320                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
6321                         auto round = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
6322                         round->addArg(x.value);
6323                         round->addArg(::context->getConstantInt32(3));
6324                         ::basicBlock->appendInst(round);
6325
6326                         return RValue<Float4>(V(result));
6327                 }
6328                 else
6329                 {
6330                         return Float4(Int4(x));
6331                 }
6332         }
6333
6334         RValue<Float4> Frac(RValue<Float4> x)
6335         {
6336                 Float4 frc;
6337
6338                 if(CPUID::SSE4_1)
6339                 {
6340                         frc = x - Floor(x);
6341                 }
6342                 else
6343                 {
6344                         frc = x - Float4(Int4(x));   // Signed fractional part.
6345
6346                         frc += As<Float4>(As<Int4>(CmpNLE(Float4(0.0f), frc)) & As<Int4>(Float4(1, 1, 1, 1)));   // Add 1.0 if negative.
6347                 }
6348
6349                 // x - floor(x) can be 1.0 for very small negative x.
6350                 // Clamp against the value just below 1.0.
6351                 return Min(frc, As<Float4>(Int4(0x3F7FFFFF)));
6352         }
6353
6354         RValue<Float4> Floor(RValue<Float4> x)
6355         {
6356                 if(CPUID::SSE4_1)
6357                 {
6358                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
6359                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Round, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
6360                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
6361                         auto round = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
6362                         round->addArg(x.value);
6363                         round->addArg(::context->getConstantInt32(1));
6364                         ::basicBlock->appendInst(round);
6365
6366                         return RValue<Float4>(V(result));
6367                 }
6368                 else
6369                 {
6370                         return x - Frac(x);
6371                 }
6372         }
6373
6374         RValue<Float4> Ceil(RValue<Float4> x)
6375         {
6376                 if(CPUID::SSE4_1)
6377                 {
6378                         Ice::Variable *result = ::function->makeVariable(Ice::IceType_v4f32);
6379                         const Ice::Intrinsics::IntrinsicInfo intrinsic = {Ice::Intrinsics::Round, Ice::Intrinsics::SideEffects_F, Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F};
6380                         auto target = ::context->getConstantUndef(Ice::IceType_i32);
6381                         auto round = Ice::InstIntrinsicCall::create(::function, 2, result, target, intrinsic);
6382                         round->addArg(x.value);
6383                         round->addArg(::context->getConstantInt32(2));
6384                         ::basicBlock->appendInst(round);
6385
6386                         return RValue<Float4>(V(result));
6387                 }
6388                 else
6389                 {
6390                         return -Floor(-x);
6391                 }
6392         }
6393
6394         Type *Float4::getType()
6395         {
6396                 return T(Ice::IceType_v4f32);
6397         }
6398
6399         RValue<Pointer<Byte>> operator+(RValue<Pointer<Byte>> lhs, int offset)
6400         {
6401                 return lhs + RValue<Int>(Nucleus::createConstantInt(offset));
6402         }
6403
6404         RValue<Pointer<Byte>> operator+(RValue<Pointer<Byte>> lhs, RValue<Int> offset)
6405         {
6406                 return RValue<Pointer<Byte>>(Nucleus::createGEP(lhs.value, Byte::getType(), offset.value, false));
6407         }
6408
6409         RValue<Pointer<Byte>> operator+(RValue<Pointer<Byte>> lhs, RValue<UInt> offset)
6410         {
6411                 return RValue<Pointer<Byte>>(Nucleus::createGEP(lhs.value, Byte::getType(), offset.value, true));
6412         }
6413
6414         RValue<Pointer<Byte>> operator+=(Pointer<Byte> &lhs, int offset)
6415         {
6416                 return lhs = lhs + offset;
6417         }
6418
6419         RValue<Pointer<Byte>> operator+=(Pointer<Byte> &lhs, RValue<Int> offset)
6420         {
6421                 return lhs = lhs + offset;
6422         }
6423
6424         RValue<Pointer<Byte>> operator+=(Pointer<Byte> &lhs, RValue<UInt> offset)
6425         {
6426                 return lhs = lhs + offset;
6427         }
6428
6429         RValue<Pointer<Byte>> operator-(RValue<Pointer<Byte>> lhs, int offset)
6430         {
6431                 return lhs + -offset;
6432         }
6433
6434         RValue<Pointer<Byte>> operator-(RValue<Pointer<Byte>> lhs, RValue<Int> offset)
6435         {
6436                 return lhs + -offset;
6437         }
6438
6439         RValue<Pointer<Byte>> operator-(RValue<Pointer<Byte>> lhs, RValue<UInt> offset)
6440         {
6441                 return lhs + -offset;
6442         }
6443
6444         RValue<Pointer<Byte>> operator-=(Pointer<Byte> &lhs, int offset)
6445         {
6446                 return lhs = lhs - offset;
6447         }
6448
6449         RValue<Pointer<Byte>> operator-=(Pointer<Byte> &lhs, RValue<Int> offset)
6450         {
6451                 return lhs = lhs - offset;
6452         }
6453
6454         RValue<Pointer<Byte>> operator-=(Pointer<Byte> &lhs, RValue<UInt> offset)
6455         {
6456                 return lhs = lhs - offset;
6457         }
6458
6459         void Return()
6460         {
6461                 Nucleus::createRetVoid();
6462                 Nucleus::setInsertBlock(Nucleus::createBasicBlock());
6463                 Nucleus::createUnreachable();
6464         }
6465
6466         void Return(RValue<Int> ret)
6467         {
6468                 Nucleus::createRet(ret.value);
6469                 Nucleus::setInsertBlock(Nucleus::createBasicBlock());
6470                 Nucleus::createUnreachable();
6471         }
6472
6473         void branch(RValue<Bool> cmp, BasicBlock *bodyBB, BasicBlock *endBB)
6474         {
6475                 Nucleus::createCondBr(cmp.value, bodyBB, endBB);
6476                 Nucleus::setInsertBlock(bodyBB);
6477         }
6478
6479         RValue<Long> Ticks()
6480         {
6481                 assert(false && "UNIMPLEMENTED"); return RValue<Long>(V(nullptr));
6482         }
6483 }