/* -*- Mode: c++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup" -*-
 *
 * Copyright (C) Opera Software ASA  2011
 *
 * @author Fredrik Ohrn
 */

#include "core/pch.h"

#include "modules/ecmascript/carakan/src/es_pch.h"

#include "modules/ecmascript/carakan/src/compiler/es_native.h"
#include "modules/ecmascript/carakan/src/util/es_codegenerator_mips.h"

#ifdef ES_NATIVE_SUPPORT
#ifdef ARCHITECTURE_MIPS

#ifndef ES_VALUES_AS_NANS
#error "Unsupported configuration."
#endif // ES_VALUES_AS_NANS

#ifdef __LP64__
#error "64-bit pointers not supported."
#endif // __LP64__

#define D ES_CodeGenerator::D // FPU double
#define S ES_CodeGenerator::S // FPU single
#define W ES_CodeGenerator::W // FPU int32

#define EQ ES_CodeGenerator::EQ // FPU condition

// Registers V0 to S4 are used by the arithmetic block register allocator.

#define REG_Z  ES_CodeGenerator::REG_ZERO
#define REG_AT ES_CodeGenerator::REG_AT // Scratch register shared with ES_CodeGenerator.
#define REG_V0 ES_CodeGenerator::REG_V0
#define REG_V1 ES_CodeGenerator::REG_V1
#define REG_A0 ES_CodeGenerator::REG_A0
#define REG_A1 ES_CodeGenerator::REG_A1 // Also used as argument count parameter.
#define REG_A2 ES_CodeGenerator::REG_A2
#define REG_A3 ES_CodeGenerator::REG_A3
#define REG_T0 ES_CodeGenerator::REG_T0
#define REG_T1 ES_CodeGenerator::REG_T1
#define REG_T2 ES_CodeGenerator::REG_T2
#define REG_T3 ES_CodeGenerator::REG_T3
#define REG_T4 ES_CodeGenerator::REG_T4
#define REG_T5 ES_CodeGenerator::REG_T5
#define REG_T6 ES_CodeGenerator::REG_T6
#define REG_T7 ES_CodeGenerator::REG_T7 // Used as value transfer pointer.
#define REG_S0 ES_CodeGenerator::REG_S0
#define REG_S1 ES_CodeGenerator::REG_S1
#define REG_S2 ES_CodeGenerator::REG_S2
#define REG_S3 ES_CodeGenerator::REG_S3
#define REG_S4 ES_CodeGenerator::REG_S4
#define REG_S5 ES_CodeGenerator::REG_S5 // Used to save return address in lightweight functions.
#define REG_S6 ES_CodeGenerator::REG_S6 // Used as constant data section pointer.
#define REG_S7 ES_CodeGenerator::REG_S7 // Used as ES_Execution_Context pointer.
#define REG_T8 ES_CodeGenerator::REG_T8 // Scratch register.
#define REG_T9 ES_CodeGenerator::REG_T9 // Scratch register.
#define REG_SP ES_CodeGenerator::REG_SP
#define REG_FP ES_CodeGenerator::REG_FP // Used as register frame pointer
#define REG_RA ES_CodeGenerator::REG_RA

#define REG_F28 ES_CodeGenerator::REG_F28 // Scratch register
#define REG_F30 ES_CodeGenerator::REG_F30 // Scratch register
#define REG_F31 ES_CodeGenerator::REG_F31 // Scratch register

#define REG_FEXR ES_CodeGenerator::REG_FEXR

/* These registers are used as output from the code generated by AllocateObject(). */
#define ALLOCATED_OBJECT_REGISTER ES_CodeGenerator::REG_S0
#define ALLOCATED_OBJECT_NAMED_PROPERTIES_REGISTER ES_CodeGenerator::REG_S1
#define ALLOCATED_OBJECT_INDEXED_PROPERTIES_REGISTER ES_CodeGenerator::REG_S2

#define VALUE_TRANSFER_REGISTER REG_T7
#define VALUE_TRANSFER_NATIVE_REGISTER (NR(VALUE_TRANSFER_REGISTER - REG_V0))

#define MIPS_STACK_ALIGNMENT 8
#define MIPS_STACK_ARG_AREA 16

#define SP_OFFSET_RETURN_ADDRESS(frame_size)    (MIPS_STACK_ARG_AREA + (frame_size) - sizeof(void*) * 1)
#define SP_OFFSET_NEXT_STACK_FRAME(frame_size)  (MIPS_STACK_ARG_AREA + (frame_size) - sizeof(void*) * 2)
#define SP_OFFSET_REGISTER_FRAME(frame_size)    (MIPS_STACK_ARG_AREA + (frame_size) - sizeof(void*) * 3)
#define SP_OFFSET_CODE(frame_size)              (MIPS_STACK_ARG_AREA + (frame_size) - sizeof(void*) * 4)
#define SP_OFFSET_ARGUMENTS_OBJECT(frame_size)  (MIPS_STACK_ARG_AREA + (frame_size) - sizeof(void*) * 5)
#define SP_OFFSET_ARGUMENT_COUNT(frame_size)    (MIPS_STACK_ARG_AREA + (frame_size) - sizeof(void*) * 6)
#define SP_OFFSET_VARIABLE_OBJECT(frame_size)   (MIPS_STACK_ARG_AREA + (frame_size) - sizeof(void*) * 7)

#define private_type other.type
#define private_value other.value

#define DECLARE_NOTHING() ES_DECLARE_NOTHING()

typedef ES_CodeGenerator::Constant Constant;
typedef ES_CodeGenerator::Register Register;
typedef ES_CodeGenerator::FPURegister FPURegister;
typedef ES_Native::NativeRegister NativeRegister;
typedef ES_Native::VirtualRegister VirtualRegister;

static int
REGISTER_OFFSET(unsigned index)
{
    int o = index * sizeof(ES_Value_Internal);

    OP_ASSERT(o >= SHRT_MIN && o <= SHRT_MAX);
    return o;
}

static int
REGISTER_OFFSET(VirtualRegister *vr)
{
    OP_ASSERT(vr->stack_frame_offset == INT_MAX);
    return REGISTER_OFFSET(vr->index);
}

static int
DVALUE_OFFSET(unsigned index)
{
    return REGISTER_OFFSET(index);
}

static int
DVALUE_OFFSET(VirtualRegister *vr)
{
    if (vr->stack_frame_offset == INT_MAX)
        return REGISTER_OFFSET(vr->index);
    else
    {
        OP_ASSERT(vr->stack_frame_type == ESTYPE_DOUBLE);
        int o = -(8 + vr->stack_frame_offset);
        OP_ASSERT(o >= SHRT_MIN && o <= 0);
        return o;
    }
}

static int
TYPE_OFFSET(unsigned index)
{
    DECLARE_NOTHING();

    int o = index * sizeof(ES_Value_Internal) + ES_OFFSETOF(ES_Value_Internal, private_type);

    OP_ASSERT(o >= SHRT_MIN && o <= SHRT_MAX);
    return o;
}

static int
TYPE_OFFSET(VirtualRegister *vr)
{
    OP_ASSERT(vr->stack_frame_offset == INT_MAX);
    return TYPE_OFFSET(vr->index);
}

static int
IVALUE_OFFSET(unsigned index)
{
    DECLARE_NOTHING();

    int o = index * sizeof(ES_Value_Internal) + ES_OFFSETOF(ES_Value_Internal, private_value.i32);

    OP_ASSERT(o >= SHRT_MIN && o <= SHRT_MAX);
    return o;
}

static int
IVALUE_OFFSET(VirtualRegister *vr)
{
    if (vr->stack_frame_offset == INT_MAX)
        return IVALUE_OFFSET(vr->index);
    else
    {
        OP_ASSERT(vr->stack_frame_type != ESTYPE_DOUBLE);
        int o = -(4 + vr->stack_frame_offset);
        OP_ASSERT(o >= SHRT_MIN && o <= 0);
        return o;
    }
}

static Register
BASE_REGISTER(VirtualRegister *vr)
{
    return vr->stack_frame_offset == INT_MAX ? REG_FP : REG_SP;
}

static Register
INTEGER_REGISTER(NativeRegister *nr)
{
    OP_ASSERT(nr->type == NativeRegister::TYPE_INTEGER);
    return static_cast<Register>(nr->register_code);
}

static Register
INTEGER_REGISTER(const ES_Native::Operand &op)
{
    return INTEGER_REGISTER(op.native_register);
}

static FPURegister
DOUBLE_REGISTER(NativeRegister *nr)
{
    OP_ASSERT(nr->type == NativeRegister::TYPE_DOUBLE);
    return static_cast<FPURegister>(nr->register_code);
}

static FPURegister
DOUBLE_REGISTER(const ES_Native::Operand &op)
{
    return DOUBLE_REGISTER(op.native_register);
}

static Constant*
GetTypeConst(ES_Native &ns, ES_ValueType value_type)
{
    switch (value_type)
    {
    case ESTYPE_DOUBLE:
        return ns.GetConst(CONST_ESTYPE_DOUBLE);
    case ESTYPE_INT32_OR_DOUBLE:
        return ns.GetConst(CONST_ESTYPE_INT32_OR_DOUBLE);
    case ESTYPE_INT32:
        return ns.GetConst(CONST_ESTYPE_INT32);
    case ESTYPE_UNDEFINED:
        return ns.GetConst(CONST_ESTYPE_UNDEFINED);
    case ESTYPE_NULL:
        return ns.GetConst(CONST_ESTYPE_NULL);
    case ESTYPE_BOOLEAN:
        return ns.GetConst(CONST_ESTYPE_BOOLEAN);
    case ESTYPE_BOXED:
        return ns.GetConst(CONST_ESTYPE_BOXED);
    case ESTYPE_STRING:
        return ns.GetConst(CONST_ESTYPE_STRING);
    case ESTYPE_OBJECT:
        return ns.GetConst(CONST_ESTYPE_OBJECT);
    default:
        OP_ASSERT(FALSE);
        return NULL;
    }
}

static void
LoadValue(ES_CodeGenerator &cg, Register src_base, int src_offset, Register dst_value, Register dst_type)
{
    if(src_offset >= SHRT_MIN && (src_offset + 4) <= SHRT_MAX)
    {
        OP_ASSERT(dst_value != src_base);

        cg.LW(dst_value, src_offset + VALUE_IVALUE_OFFSET, src_base);
        cg.LW(dst_type, src_offset + VALUE_TYPE_OFFSET, src_base);
    }
    else
    {
        OP_ASSERT(dst_value != REG_AT);

        cg.Add(REG_AT, src_base, src_offset);

        cg.LW(dst_value, VALUE_IVALUE_OFFSET, REG_AT);
        cg.LW(dst_type, VALUE_TYPE_OFFSET, REG_AT);
    }
}

static void
LoadValue(ES_CodeGenerator &cg, VirtualRegister *source, Register dst_value, Register dst_type)
{
    OP_ASSERT(source->stack_frame_offset == INT_MAX);

    cg.LW(dst_value, IVALUE_OFFSET(source), REG_FP);
    cg.LW(dst_type, TYPE_OFFSET(source), REG_FP);
}

static void
StoreValue(ES_CodeGenerator &cg, Register dst_base, int dst_offset, Register src_value, Register src_type)
{
    if(dst_offset >= SHRT_MIN && (dst_offset + 4) <= SHRT_MAX)
    {
        cg.SW(src_value, dst_offset + VALUE_IVALUE_OFFSET, dst_base);
        cg.SW(src_type, dst_offset + VALUE_TYPE_OFFSET, dst_base);
    }
    else
    {
        OP_ASSERT(src_value != REG_AT && src_type != REG_AT);

        cg.Add(REG_AT, dst_base, dst_offset);

        cg.SW(src_value, VALUE_IVALUE_OFFSET, REG_AT);
        cg.SW(src_type, VALUE_TYPE_OFFSET, REG_AT);
    }
}

static void
StoreValue(ES_CodeGenerator &cg, VirtualRegister *dest, Register src_value, Register src_type)
{
    OP_ASSERT(dest->stack_frame_offset == INT_MAX);

    cg.SW(src_value, IVALUE_OFFSET(dest), REG_FP);
    cg.SW(src_type, TYPE_OFFSET(dest), REG_FP);
}

static void
CopyValue(ES_CodeGenerator &cg, Register src_base, int src_offset, Register dst_base, int dst_offset)
{
    OP_ASSERT((dst_base != REG_T8) && (dst_base != REG_T9));

    LoadValue(cg, src_base, src_offset, REG_T8, REG_T9);
    StoreValue(cg, dst_base, dst_offset, REG_T8, REG_T9);
}

static void
CopyValue(ES_CodeGenerator &cg, VirtualRegister *src, Register dst_base, int dst_offset)
{
    OP_ASSERT((dst_base != REG_T8) && (dst_base != REG_T9));

    LoadValue(cg, src, REG_T8, REG_T9);
    StoreValue(cg, dst_base, dst_offset, REG_T8, REG_T9);
}

static void
CopyValue(ES_CodeGenerator &cg, Register src_base, int src_offset, VirtualRegister *dst)
{
    LoadValue(cg, src_base, src_offset, REG_T8, REG_T9);
    StoreValue(cg, dst, REG_T8, REG_T9);
}

static void
CopyValue(ES_CodeGenerator &cg, VirtualRegister *src, VirtualRegister *dst)
{
    LoadValue(cg, src, REG_T8, REG_T9);
    StoreValue(cg, dst, REG_T8, REG_T9);
}

static void
CopyDataToValue(ES_Native &n, Register reg_source, Register source_type, VirtualRegister *target)
{
    ES_CodeGenerator &cg = n.cg;

    ES_CodeGenerator::OutOfOrderBlock *handle_untyped = cg.StartOutOfOrderBlock();
    CopyValue(cg, reg_source, 0, target);
    cg.EndOutOfOrderBlock();

    cg.Load(REG_T8, n.GetConst(CONST_COPY_DATA_HANDLED));
    cg.AND(REG_T8, source_type, REG_T8);
    cg.JumpEQ(handle_untyped->GetJumpTarget(), REG_T8, REG_Z);

    ES_CodeGenerator::JumpTarget *not_null_target = cg.ForwardJump();
    ES_CodeGenerator::JumpTarget *store_target = cg.ForwardJump();

    cg.And(REG_T9, source_type, ES_STORAGE_NULL_MASK);
    cg.LW(REG_T8, 0, reg_source);

    cg.JumpEQ(not_null_target, REG_T9, REG_Z, TRUE);
    cg.JumpNE(not_null_target, REG_T8, REG_Z);

    cg.Load(REG_T9, n.GetConst(CONST_ESTYPE_NULL));
    cg.Jump(store_target, ES_NATIVE_UNCONDITIONAL, TRUE);

    cg.SetJumpTarget(not_null_target);
    cg.Load(REG_T9, n.GetConst(CONST_ES_VALUE_TYPE_INIT_MASK));
    cg.OR(REG_T9, REG_T9, source_type);

    cg.SetJumpTarget(store_target);
    StoreValue(cg, target, REG_T8, REG_T9);

    cg.SetOutOfOrderContinuationPoint(handle_untyped);
}

static void
CopyTypedDataToValue(ES_Native &n, Register reg_source, unsigned source_offset, ES_StorageType source_type, VirtualRegister *target)
{
    ES_CodeGenerator &cg = n.cg;

    if (source_type == ES_STORAGE_WHATEVER || source_type == ES_STORAGE_DOUBLE)
        CopyValue(cg, reg_source, source_offset, target);
    else
    {
        cg.Load(REG_T8, source_offset, reg_source);

        ES_CodeGenerator::OutOfOrderBlock *null_block = NULL;

        if (ES_Layout_Info::IsNullable(source_type))
        {
            null_block = cg.StartOutOfOrderBlock();
            cg.Load(REG_T9, n.GetConst(CONST_ESTYPE_NULL));
            cg.SW(REG_T9, TYPE_OFFSET(target), BASE_REGISTER(target));
            cg.EndOutOfOrderBlock();

            cg.JumpEQ(null_block->GetJumpTarget(), REG_T8, REG_Z);
        }

        cg.Load(REG_T9, GetTypeConst(n, ES_Value_Internal::ConvertToValueType(source_type)));
        StoreValue(cg, target, REG_T8, REG_T9);

        if (null_block)
            cg.SetOutOfOrderContinuationPoint(null_block);
    }
}

static void
JumpToSize(ES_CodeGenerator &cg, unsigned size, ES_CodeGenerator::JumpTarget *&size_4_target, ES_CodeGenerator::JumpTarget *&size_8_target)
{
    if (size == 4)
    {
        if (!size_4_target)
            size_4_target = cg.ForwardJump();
        cg.Jump(size_4_target);
    }
    else
    {
        OP_ASSERT(size == 8);
        if (!size_8_target)
            size_8_target = cg.ForwardJump();
        cg.Jump(size_8_target);
    }
}

static void
CopyValueToData2(ES_CodeGenerator &cg, Register reg_source, Register reg_target, unsigned target_offset, ES_CodeGenerator::JumpTarget *size_4_target, ES_CodeGenerator::JumpTarget *size_8_target)
{
    OP_ASSERT(size_4_target || size_8_target);

#ifdef OPERA_BIG_ENDIAN
    ES_CodeGenerator::JumpTarget *jt_done = NULL;

    if (size_8_target)
    {
        cg.SetJumpTarget(size_8_target);
        CopyValue(cg, reg_source, 0, reg_target, target_offset);

        if (size_4_target)
        {
            jt_done = cg.ForwardJump();
            cg.Jump(jt_done);
        }
    }

    if (size_4_target)
    {
        cg.SetJumpTarget(size_4_target);

        cg.LW(REG_T8, VALUE_IVALUE_OFFSET, reg_source);
        cg.Store(REG_T8, target_offset, reg_target);

        if (jt_done)
            cg.SetJumpTarget(jt_done);
    }
#else // OPERA_BIG_ENDIAN
    if (size_8_target)
    {
        cg.SetJumpTarget(size_8_target);
        if (!size_4_target)
        {
            CopyValue(cg, reg_source, 0, reg_target, target_offset);
            return;
        }
        else
        {
            cg.LW(REG_T8, 4, reg_source);
            cg.Store(REG_T8, target_offset + 4, reg_target);
        }
    }
    if (size_4_target || size_8_target)
    {
        if (size_4_target)
            cg.SetJumpTarget(size_4_target);

        cg.LW(REG_T8, 0, reg_source);
        cg.Store(REG_T8, target_offset, reg_target);
    }
#endif // OPERA_BIG_ENDIAN
}

static void
CopyValueToData(ES_Native &n, VirtualRegister *source, Register reg_target, unsigned target_offset, Register reg_target_type, ES_CodeGenerator::JumpTarget *slow_case_target)
{
    ES_CodeGenerator &cg = n.cg;

    ES_CodeGenerator::OutOfOrderBlock *handle_untyped = cg.StartOutOfOrderBlock();
    CopyValue(cg, source, reg_target, target_offset);
    cg.EndOutOfOrderBlock();

    cg.Load(REG_T9, n.GetConst(CONST_COPY_DATA_HANDLED));
    cg.AND(REG_T8, reg_target_type, REG_T9);
    cg.JumpEQ(handle_untyped->GetJumpTarget(), REG_T8, REG_Z);

    ES_CodeGenerator::JumpTarget *not_null_target = cg.ForwardJump();
    ES_CodeGenerator::JumpTarget *done_target = cg.ForwardJump();

    cg.And(REG_T8, reg_target_type, ES_STORAGE_NULL_MASK);
    cg.LW(REG_T9, TYPE_OFFSET(source->index), BASE_REGISTER(source));

    cg.JumpEQ(not_null_target, REG_T8, REG_Z, TRUE);

    cg.Load(REG_T8, n.GetConst(CONST_ESTYPE_NULL));
    cg.JumpNE(not_null_target, REG_T8, REG_T9, TRUE);

    cg.Store(REG_Z, target_offset, reg_target);
    cg.Jump(done_target, ES_NATIVE_UNCONDITIONAL, TRUE);

    cg.SetJumpTarget(not_null_target);
    cg.Load(REG_T8, n.GetConst(CONST_ES_VALUE_TYPE_INIT_MASK));
    cg.OR(REG_T8, REG_T8, reg_target_type);
    cg.JumpNE(slow_case_target, REG_T8, REG_T9);

    cg.LW(REG_T8, IVALUE_OFFSET(source->index), BASE_REGISTER(source));
    cg.Store(REG_T8, target_offset, reg_target);

    cg.SetJumpTarget(done_target);
    cg.SetOutOfOrderContinuationPoint(handle_untyped);
}

static void
CopyValueToTypedData(ES_Native &n, VirtualRegister *source, Register reg_target, unsigned target_offset, ES_StorageType target_type, ES_CodeGenerator::JumpTarget *type_check_fail)
{
    ES_CodeGenerator &cg = n.cg;

    if (target_type == ES_STORAGE_WHATEVER || target_type == ES_STORAGE_DOUBLE)
    {
        LoadValue(cg, source, REG_T8, REG_T9);
        if (target_type == ES_STORAGE_DOUBLE && type_check_fail)
        {
            cg.Load(REG_AT, n.GetConst(CONST_ESTYPE_DOUBLE));
            cg.JumpGT(type_check_fail, REG_T9, REG_AT);
        }
        StoreValue(cg, reg_target, target_offset, REG_T8, REG_T9);
    }
    else if (type_check_fail)
    {
        ES_ValueType value_type = ES_Value_Internal::ConvertToValueType(target_type);

        LoadValue(cg, source, REG_T8, REG_T9);

        if (ES_Layout_Info::IsNullable(target_type))
        {
            ES_CodeGenerator::JumpTarget *not_null_target = cg.ForwardJump();
            ES_CodeGenerator::JumpTarget *done_target = cg.ForwardJump();

            cg.Load(REG_AT, n.GetConst(CONST_ESTYPE_NULL));
            cg.JumpNE(not_null_target, REG_T9, REG_AT);

            cg.Store(REG_Z, target_offset, reg_target);
            cg.Jump(done_target, ES_NATIVE_UNCONDITIONAL, TRUE);

            cg.SetJumpTarget(not_null_target);

            cg.Load(REG_AT, GetTypeConst(n, value_type));
            cg.JumpNE(type_check_fail, REG_T9, REG_AT);
            cg.Store(REG_T8, target_offset, reg_target);

            cg.SetJumpTarget(done_target);
        }
        else
        {
            cg.Load(REG_AT, GetTypeConst(n, value_type));
            cg.JumpNE(type_check_fail, REG_T9, REG_AT);
            cg.Store(REG_T8, target_offset, reg_target);
        }
    }
    else if (target_type == ES_STORAGE_NULL)
    {
        cg.Store(REG_Z, target_offset, reg_target);
    }
    else if (target_type != ES_STORAGE_UNDEFINED)
    {
        OP_ASSERT(ES_Layout_Info::SizeOf(target_type) == 4);
        cg.LW(REG_T8, IVALUE_OFFSET(source->index), BASE_REGISTER(source));
        cg.Store(REG_T8, target_offset, reg_target);
    }
}

void
ES_Native::EmitTypeConversionHandlers(VirtualRegister *source, Register reg_properties, unsigned offset, ES_CodeGenerator::JumpTarget *null_target, ES_CodeGenerator::JumpTarget *int_to_double_target)
{
    ES_CodeGenerator::OutOfOrderBlock *int_to_double_block = NULL;

    if (int_to_double_target)
    {
        int_to_double_block = cg.StartOutOfOrderBlock();

        cg.SetJumpTarget(int_to_double_target);

        const Register reg_source = REG_S3;

        OP_ASSERT(reg_source != reg_properties);

        cg.Add(reg_source, REG_FP, REGISTER_OFFSET(source));

        if (fpu_type != ES_CodeGenerator::FPU_NONE)
        {
            cg.LWC1(REG_F28, VALUE_IVALUE_OFFSET, reg_source);
            cg.CVT(D, W, REG_F30, REG_F28);

            if (offset <= SHRT_MAX)
                cg.SDC1(REG_F30, offset, reg_properties);
            else
            {
                cg.Add(REG_AT, reg_properties, offset);
                cg.SDC1(REG_F30, 0, REG_AT);
            }
        }
        else
        {
            cg.Add(REG_A1, reg_properties, offset);
            cg.Load(REG_T9, GetConst(CONST_ESN_STOREINTASDOUBLE));
            cg.JALR(REG_T9);
            cg.LW(REG_A0, VALUE_IVALUE_OFFSET, reg_source);
        }
        cg.EndOutOfOrderBlock();
    }

    ES_CodeGenerator::OutOfOrderBlock *null_block = NULL;

    if (null_target)
    {
        null_block = cg.StartOutOfOrderBlock();
        cg.SetJumpTarget(null_target);
        cg.Store(REG_Z, offset, reg_properties);
        cg.EndOutOfOrderBlock();
    }

    if (int_to_double_block)
        cg.SetOutOfOrderContinuationPoint(int_to_double_block);

    if (null_block)
        cg.SetOutOfOrderContinuationPoint(null_block);
}

static ES_CodeGenerator::OutOfOrderBlock*
RecoverFromFailedPropertyValueTransfer(ES_Native &native, VirtualRegister *target, Register reg_transfer, ES_StorageType storage_type)
{
    if (!native.is_light_weight)
    {
        ES_CodeGenerator &cg = native.cg;
        ES_CodeGenerator::OutOfOrderBlock *recover = cg.StartOutOfOrderBlock();

        ANNOTATE("RecoverFromFailedPropertyValueTransfer");

        cg.SetOutOfOrderContinuationPoint(native.current_slow_case);
        native.current_slow_case = NULL;

        native.property_value_fail = cg.ForwardJump();
        native.EmitRegisterTypeCheck(target, ESTYPE_OBJECT, native.property_value_fail);

        int target_offset = storage_type == ES_STORAGE_WHATEVER ? REGISTER_OFFSET(target) : IVALUE_OFFSET(target);
        int register_offset = target_offset - native.property_value_offset;

        cg.Add(reg_transfer, REG_FP, register_offset);

        cg.EndOutOfOrderBlock();

        return recover;
    }
    else
        return NULL;
}

static void
CallContextFunc(ES_CodeGenerator &cg, Constant *func)
{
    cg.Load(REG_T9, func);
    cg.JALR(REG_T9);
    cg.Move(REG_A0, REG_S7);
}

static void
CallDispatcherFunc(ES_CodeGenerator &cg, Constant *func)
{
    ANNOTATE("\nCallDispatcherFunc");
    ES_CodeGenerator::JumpTarget *end = cg.ForwardJump();

    CallContextFunc(cg, func);

    cg.Jump(end, ES_NATIVE_CONDITION_EQZ(REG_V0));

    cg.JR(REG_V0);
    cg.NOP();

    cg.SetJumpTarget(end);
}

static int
CreateNativeStackFrame(ES_Native &ns)
{
    DECLARE_NOTHING();

    ES_CodeGenerator &cg = ns.cg;
    ES_Code *code = ns.code;

    ANNOTATE("CreateNativeStackFrame");

    unsigned frame_size = 0;

    // JIT functions do not require the 16 byte reserved area for C calls.
    // The new stack frame is written at a positive SP offset so that there
    // is no gap between consecutive JIT frames.

    // Push previous frame pointer and record new one.
    cg.LW(REG_T8, ES_OFFSETOF(ES_Execution_Context, native_stack_frame), REG_S7);

    cg.SW(REG_RA, 12, REG_SP);
    cg.SW(REG_T8,  8, REG_SP);

    cg.ADDIU(REG_T9, REG_SP, 8);
    cg.SW(REG_T9, ES_OFFSETOF(ES_Execution_Context, native_stack_frame), REG_S7);

    // Store register frame and ES_Code.
    cg.Load(REG_T8, ns.GetConst(CONST_CODE));
    cg.SW(REG_FP, 4, REG_SP);
    cg.SW(REG_T8, 0, REG_SP);

    frame_size += 4 * sizeof(void *);

    if (code->type == ES_Code::TYPE_FUNCTION)
    {
        cg.SW(REG_Z, -4, REG_SP); // Arguments object
        cg.SW(REG_A1, -8, REG_SP); // Argument count

        frame_size += 2 * sizeof(void *);

        if (code->CanHaveVariableObject())
        {
            cg.SW(REG_Z, -12, REG_SP); // Variable object
            frame_size += sizeof(void *);
        }
    }

    frame_size = (frame_size + MIPS_STACK_ALIGNMENT - 1) & ~(MIPS_STACK_ALIGNMENT - 1);

    OP_ASSERT(frame_size == ES_NativeStackFrame::GetFrameSize(code, TRUE));

    cg.Sub(REG_SP, frame_size);

    return frame_size;
}

static void
CheckFPUException(ES_CodeGenerator &cg, ES_CodeGenerator::JumpTarget *slow_case)
{
    OP_ASSERT(slow_case);

    cg.CFC1(REG_T8, REG_FEXR);
    cg.And(REG_T8, 1 << 16); // Check invalid operation bit.
    cg.Jump(slow_case, ES_NATIVE_CONDITION_NEZ(REG_T8));
}


/* The code vectors below where generated from es_native_mips_trampolines.s */
const unsigned cv_BytecodeToNativeTrampoline[] =
{
    0x27bdffb0, // addiu   sp,sp,-80
    0xafb00028, // sw      s0,40(sp)
    0xafb1002c, // sw      s1,44(sp)
    0xafb20030, // sw      s2,48(sp)
    0xafb30034, // sw      s3,52(sp)
    0xafb40038, // sw      s4,56(sp)
    0xafb5003c, // sw      s5,60(sp)
    0xafb60040, // sw      s6,64(sp)
    0xafb70044, // sw      s7,68(sp)
    0xafbe0048, // sw      s8,72(sp)
    0xafbf004c, // sw      ra,76(sp)
    0xafa40050, // sw      a0,80(sp)
    0x8c960024, // lw      s6,36(a0)
    0x8c970000, // lw      s7,0(a0)
    0x8c9e0010, // lw      s8,16(a0)
    0x8c88000c, // lw      t0,12(a0)
    0x8d090000, // lw      t1,0(t0)
    0xafa90020, // sw      t1,32(sp)
    0xad1d0000, // sw      sp,0(t0)
    0xafa0001c, // sw      zero,28(sp)
    0xafa00018, // sw      zero,24(sp)
    0x8c880004, // lw      t0,4(a0)
    0xafa80014, // sw      t0,20(sp)
    0xafa00010, // sw      zero,16(sp)
    0x8c880008, // lw      t0,8(a0)
    0x27a90018, // addiu   t1,sp,24
    0xad090000, // sw      t1,0(t0)
    0x8c990014, // lw      t9,20(a0)
    0x0320f809, // jalr    t9
    0x00000000, // nop
    0x8fa40050, // lw      a0,80(sp)
    0x8c880008, // lw      t0,8(a0)
    0xad000000, // sw      zero,0(t0)
    0x8c88000c, // lw      t0,12(a0)
    0x8fa90020, // lw      t1,32(sp)
    0xad090000, // sw      t1,0(t0)
    0x8fb00028, // lw      s0,40(sp)
    0x8fb1002c, // lw      s1,44(sp)
    0x8fb20030, // lw      s2,48(sp)
    0x8fb30034, // lw      s3,52(sp)
    0x8fb40038, // lw      s4,56(sp)
    0x8fb5003c, // lw      s5,60(sp)
    0x8fb60040, // lw      s6,64(sp)
    0x8fb70044, // lw      s7,68(sp)
    0x8fbe0048, // lw      s8,72(sp)
    0x8fbf004c, // lw      ra,76(sp)
    0x34020001, // li      v0,0x1
    0x03e00008, // jr      ra
    0x27bd0050  // addiu   sp,sp,80
};

#if _MIPS_FPSET == 32
const unsigned cv_BytecodeToNativeTrampoline_fpu[] =
{
    0x27bdff60, // addiu   sp,sp,-160
    0xafb00028, // sw      s0,40(sp)
    0xafb1002c, // sw      s1,44(sp)
    0xafb20030, // sw      s2,48(sp)
    0xafb30034, // sw      s3,52(sp)
    0xafb40038, // sw      s4,56(sp)
    0xafb5003c, // sw      s5,60(sp)
    0xafb60040, // sw      s6,64(sp)
    0xafb70044, // sw      s7,68(sp)
    0xafbe0048, // sw      s8,72(sp)
    0xafbf004c, // sw      ra,76(sp)
    0xf7b40050, // sdc1    $f20,80(sp)
    0xf7b50058, // sdc1    $f21,88(sp)
    0xf7b60060, // sdc1    $f22,96(sp)
    0xf7b70068, // sdc1    $f23,104(sp)
    0xf7b80070, // sdc1    $f24,112(sp)
    0xf7b90078, // sdc1    $f25,120(sp)
    0xf7ba0080, // sdc1    $f26,128(sp)
    0xf7bb0088, // sdc1    $f27,136(sp)
    0xf7bc0090, // sdc1    $f28,144(sp)
    0xf7be0098, // sdc1    $f30,152(sp)
    0xafa400a0, // sw      a0,160(sp)
    0x8c960024, // lw      s6,36(a0)
    0x8c970000, // lw      s7,0(a0)
    0x8c9e0010, // lw      s8,16(a0)
    0x8c88000c, // lw      t0,12(a0)
    0x8d090000, // lw      t1,0(t0)
    0xafa90020, // sw      t1,32(sp)
    0xad1d0000, // sw      sp,0(t0)
    0xafa0001c, // sw      zero,28(sp)
    0xafa00018, // sw      zero,24(sp)
    0x8c880004, // lw      t0,4(a0)
    0xafa80014, // sw      t0,20(sp)
    0xafa00010, // sw      zero,16(sp)
    0x8c880008, // lw      t0,8(a0)
    0x27a90018, // addiu   t1,sp,24
    0xad090000, // sw      t1,0(t0)
    0x8c990014, // lw      t9,20(a0)
    0x0320f809, // jalr    t9
    0x00000000, // nop
    0x8fa400a0, // lw      a0,160(sp)
    0x8c880008, // lw      t0,8(a0)
    0xad000000, // sw      zero,0(t0)
    0x8c88000c, // lw      t0,12(a0)
    0x8fa90020, // lw      t1,32(sp)
    0xad090000, // sw      t1,0(t0)
    0x8fb00028, // lw      s0,40(sp)
    0x8fb1002c, // lw      s1,44(sp)
    0x8fb20030, // lw      s2,48(sp)
    0x8fb30034, // lw      s3,52(sp)
    0x8fb40038, // lw      s4,56(sp)
    0x8fb5003c, // lw      s5,60(sp)
    0x8fb60040, // lw      s6,64(sp)
    0x8fb70044, // lw      s7,68(sp)
    0x8fbe0048, // lw      s8,72(sp)
    0x8fbf004c, // lw      ra,76(sp)
    0xd7b40050, // ldc1    $f20,80(sp)
    0xd7b50058, // ldc1    $f21,88(sp)
    0xd7b60060, // ldc1    $f22,96(sp)
    0xd7b70068, // ldc1    $f23,104(sp)
    0xd7b80070, // ldc1    $f24,112(sp)
    0xd7b90078, // ldc1    $f25,120(sp)
    0xd7ba0080, // ldc1    $f26,128(sp)
    0xd7bb0088, // ldc1    $f27,136(sp)
    0xd7bc0090, // ldc1    $f28,144(sp)
    0xf7be0098, // sdc1    $f30,152(sp)
    0x34020001, // li      v0,0x1
    0x03e00008, // jr      ra
    0x27bd00a0  // addiu   sp,sp,160
};
#else
const unsigned cv_BytecodeToNativeTrampoline_fpu[] =
{
    0x27bdff80, // addiu   sp,sp,-128
    0xafb00028, // sw      s0,40(sp)
    0xafb1002c, // sw      s1,44(sp)
    0xafb20030, // sw      s2,48(sp)
    0xafb30034, // sw      s3,52(sp)
    0xafb40038, // sw      s4,56(sp)
    0xafb5003c, // sw      s5,60(sp)
    0xafb60040, // sw      s6,64(sp)
    0xafb70044, // sw      s7,68(sp)
    0xafbe0048, // sw      s8,72(sp)
    0xafbf004c, // sw      ra,76(sp)
    0xf7b40050, // sdc1    $f20,80(sp)
    0xf7b60058, // sdc1    $f22,88(sp)
    0xf7b80060, // sdc1    $f24,96(sp)
    0xf7ba0068, // sdc1    $f26,104(sp)
    0xf7bc0070, // sdc1    $f28,112(sp)
    0xf7be0078, // sdc1    $f30,120(sp)
    0xafa40080, // sw      a0,128(sp)
    0x8c960024, // lw      s6,36(a0)
    0x8c970000, // lw      s7,0(a0)
    0x8c9e0010, // lw      s8,16(a0)
    0x8c88000c, // lw      t0,12(a0)
    0x8d090000, // lw      t1,0(t0)
    0xafa90020, // sw      t1,32(sp)
    0xad1d0000, // sw      sp,0(t0)
    0xafa0001c, // sw      zero,28(sp)
    0xafa00018, // sw      zero,24(sp)
    0x8c880004, // lw      t0,4(a0)
    0xafa80014, // sw      t0,20(sp)
    0xafa00010, // sw      zero,16(sp)
    0x8c880008, // lw      t0,8(a0)
    0x27a90018, // addiu   t1,sp,24
    0xad090000, // sw      t1,0(t0)
    0x8c990014, // lw      t9,20(a0)
    0x0320f809, // jalr    t9
    0x00000000, // nop
    0x8fa40080, // lw      a0,128(sp)
    0x8c880008, // lw      t0,8(a0)
    0xad000000, // sw      zero,0(t0)
    0x8c88000c, // lw      t0,12(a0)
    0x8fa90020, // lw      t1,32(sp)
    0xad090000, // sw      t1,0(t0)
    0x8fb00028, // lw      s0,40(sp)
    0x8fb1002c, // lw      s1,44(sp)
    0x8fb20030, // lw      s2,48(sp)
    0x8fb30034, // lw      s3,52(sp)
    0x8fb40038, // lw      s4,56(sp)
    0x8fb5003c, // lw      s5,60(sp)
    0x8fb60040, // lw      s6,64(sp)
    0x8fb70044, // lw      s7,68(sp)
    0x8fbe0048, // lw      s8,72(sp)
    0x8fbf004c, // lw      ra,76(sp)
    0xd7b40050, // ldc1    $f20,80(sp)
    0xd7b60058, // ldc1    $f22,88(sp)
    0xd7b80060, // ldc1    $f24,96(sp)
    0xd7ba0068, // ldc1    $f26,104(sp)
    0xd7bc0070, // ldc1    $f28,112(sp)
    0xd7be0078, // ldc1    $f30,120(sp)
    0x34020001, // li      v0,0x1
    0x03e00008, // jr      ra
    0x27bd0080  // addiu   sp,sp,128
};
#endif


/* static */ BOOL
(*ES_Native::GetBytecodeToNativeTrampoline())(void **, unsigned)
{
#ifdef CONSTANT_DATA_IS_EXECUTABLE
    return reinterpret_cast<BOOL (*)(void **, unsigned)>(ES_CodeGenerator::GetFPUType() == ES_CodeGenerator::FPU_NONE ? cv_BytecodeToNativeTrampoline : cv_BytecodeToNativeTrampoline_fpu);
#else // CONSTANT_DATA_IS_EXECUTABLE
    if (!g_ecma_bytecode_to_native_block)
    {
        if (ES_CodeGenerator::GetFPUType() == ES_CodeGenerator::FPU_NONE)
        {
            g_ecma_bytecode_to_native_block = g_executableMemory->AllocateL(sizeof(cv_BytecodeToNativeTrampoline));
            op_memcpy(g_ecma_bytecode_to_native_block->address, cv_BytecodeToNativeTrampoline, sizeof(cv_BytecodeToNativeTrampoline));
            OpExecMemoryManager::FinalizeL(g_ecma_bytecode_to_native_block);
        }
        else
        {
            g_ecma_bytecode_to_native_block = g_executableMemory->AllocateL(sizeof(cv_BytecodeToNativeTrampoline_fpu));
            op_memcpy(g_ecma_bytecode_to_native_block->address, cv_BytecodeToNativeTrampoline_fpu, sizeof(cv_BytecodeToNativeTrampoline_fpu));
            OpExecMemoryManager::FinalizeL(g_ecma_bytecode_to_native_block);
        }
    }

    return reinterpret_cast<BOOL (*)(void **, unsigned)>(g_ecma_bytecode_to_native_block->address);
#endif // CONSTANT_DATA_IS_EXECUTABLE
}

const unsigned cv_ReconstructNativeStackTrampoline1[] =
{
    0x00808021, // move    s0,a0
    0x00a08821, // move    s1,a1
    0x02e02021, // move    a0,s7
    0x27a5000c, // addiu   a1,sp,12
    0xacbf0000, // sw      ra,0(a1)
    0x8e1d0020, // lw      sp,32(s0)
    0x27bdfff0, // addiu   sp,sp,-16
    0x8e19001c, // lw      t9,28(s0)
    0x0320f809, // jalr    t9
    0x00000000, // nop
    0x02202821, // move    a1,s1
    0x8c5f0000, // lw      ra,0(v0)
    0x8e080018, // lw      t0,24(s0)
    0x01000008, // jr      t0
    0x245dfff4  // addiu   sp,v0,-12
};

const unsigned cv_ReconstructNativeStackTrampoline2[] =
{
    0x00808021, // move    s0,a0
    0x00a08821, // move    s1,a1
    0x02e02021, // move    a0,s7
    0x27a5000c, // addiu   a1,sp,12
    0xacbf0000, // sw      ra,0(a1)
    0x8e1d0020, // lw      sp,32(s0)
    0x27bdfff0, // addiu   sp,sp,-16
    0x8e19001c, // lw      t9,28(s0)
    0x0320f809, // jalr    t9
    0x00000000, // nop
    0x02202821, // move    a1,s1
    0x8e080018, // lw      t0,24(s0)
    0x01000008, // jr      t0
    0x245dfff0  // addiu   sp,v0,-16
};

/* static */ void*
ES_Native::GetReconstructNativeStackTrampoline(BOOL prologue_entry_point)
{
#ifdef CONSTANT_DATA_IS_EXECUTABLE
    if (prologue_entry_point)
        return (void*)(cv_ReconstructNativeStackTrampoline1);
    else
        return (void*)(cv_ReconstructNativeStackTrampoline2);
#else // CONSTANT_DATA_IS_EXECUTABLE
    if (!g_ecma_reconstruct_native_stack1_block)
    {
        g_ecma_reconstruct_native_stack1_block = g_executableMemory->AllocateL(sizeof(cv_ReconstructNativeStackTrampoline1));
        op_memcpy(g_ecma_reconstruct_native_stack1_block->address, cv_ReconstructNativeStackTrampoline1, sizeof(cv_ReconstructNativeStackTrampoline1));
        OpExecMemoryManager::FinalizeL(g_ecma_reconstruct_native_stack1_block);
    }

    if (!g_ecma_reconstruct_native_stack2_block)
    {
        g_ecma_reconstruct_native_stack2_block = g_executableMemory->AllocateL(sizeof(cv_ReconstructNativeStackTrampoline2));
        op_memcpy(g_ecma_reconstruct_native_stack2_block->address, cv_ReconstructNativeStackTrampoline2, sizeof(cv_ReconstructNativeStackTrampoline2));
        OpExecMemoryManager::FinalizeL(g_ecma_reconstruct_native_stack2_block);
    }

    if (prologue_entry_point)
        return g_ecma_reconstruct_native_stack1_block->address;
    else
        return g_ecma_reconstruct_native_stack2_block->address;
#endif // CONSTANT_DATA_IS_EXECUTABLE
}

const unsigned cv_ThrowFromMachineCode[] =
{
    0x8c880004, // lw      t0,4(a0)
    0xad000000, // sw      zero,0(t0)
    0x8c880008, // lw      t0,8(a0)
    0x8d1d0000, // lw      sp,0(t0)
    0x8fa90020, // lw      t1,32(sp)
    0xad090000, // sw      t1,0(t0)
    0x8fb00028, // lw      s0,40(sp)
    0x8fb1002c, // lw      s1,44(sp)
    0x8fb20030, // lw      s2,48(sp)
    0x8fb30034, // lw      s3,52(sp)
    0x8fb40038, // lw      s4,56(sp)
    0x8fb5003c, // lw      s5,60(sp)
    0x8fb60040, // lw      s6,64(sp)
    0x8fb70044, // lw      s7,68(sp)
    0x8fbe0048, // lw      s8,72(sp)
    0x8fbf004c, // lw      ra,76(sp)
    0x34020000, // li      v0,0x0
    0x03e00008, // jr      ra
    0x27bd0050  // addiu   sp,sp,80
};

#if _MIPS_FPSET == 32
const unsigned cv_ThrowFromMachineCode_fpu[] =
{
    0x8c880004, // lw      t0,4(a0)
    0xad000000, // sw      zero,0(t0)
    0x8c880008, // lw      t0,8(a0)
    0x8d1d0000, // lw      sp,0(t0)
    0x8fa90020, // lw      t1,32(sp)
    0xad090000, // sw      t1,0(t0)
    0x8fb00028, // lw      s0,40(sp)
    0x8fb1002c, // lw      s1,44(sp)
    0x8fb20030, // lw      s2,48(sp)
    0x8fb30034, // lw      s3,52(sp)
    0x8fb40038, // lw      s4,56(sp)
    0x8fb5003c, // lw      s5,60(sp)
    0x8fb60040, // lw      s6,64(sp)
    0x8fb70044, // lw      s7,68(sp)
    0x8fbe0048, // lw      s8,72(sp)
    0x8fbf004c, // lw      ra,76(sp)
    0xd7b40050, // ldc1    $f20,80(sp)
    0xd7b50058, // ldc1    $f21,88(sp)
    0xd7b60060, // ldc1    $f22,96(sp)
    0xd7b70068, // ldc1    $f23,104(sp)
    0xd7b80070, // ldc1    $f24,112(sp)
    0xd7b90078, // ldc1    $f25,120(sp)
    0xd7ba0080, // ldc1    $f26,128(sp)
    0xd7bb0088, // ldc1    $f27,136(sp)
    0xd7bc0090, // ldc1    $f28,144(sp)
    0xf7be0098, // sdc1    $f30,152(sp)
    0x34020000, // li      v0,0x0
    0x03e00008, // jr      ra
    0x27bd00a0, // addiu   sp,sp,160
    0x00000000  // nop
};
#else
const unsigned cv_ThrowFromMachineCode_fpu[] =
{
    0x8c880004, // lw      t0,4(a0)
    0xad000000, // sw      zero,0(t0)
    0x8c880008, // lw      t0,8(a0)
    0x8d1d0000, // lw      sp,0(t0)
    0x8fa90020, // lw      t1,32(sp)
    0xad090000, // sw      t1,0(t0)
    0x8fb00028, // lw      s0,40(sp)
    0x8fb1002c, // lw      s1,44(sp)
    0x8fb20030, // lw      s2,48(sp)
    0x8fb30034, // lw      s3,52(sp)
    0x8fb40038, // lw      s4,56(sp)
    0x8fb5003c, // lw      s5,60(sp)
    0x8fb60040, // lw      s6,64(sp)
    0x8fb70044, // lw      s7,68(sp)
    0x8fbe0048, // lw      s8,72(sp)
    0x8fbf004c, // lw      ra,76(sp)
    0xd7b40050, // ldc1    $f20,80(sp)
    0xd7b60058, // ldc1    $f22,88(sp)
    0xd7b80060, // ldc1    $f24,96(sp)
    0xd7ba0068, // ldc1    $f26,104(sp)
    0xd7bc0070, // ldc1    $f28,112(sp)
    0xd7be0078, // ldc1    $f30,120(sp)
    0x34020000, // li      v0,0x0
    0x03e00008, // jr      ra
    0x27bd0080, // addiu   sp,sp,128
    0x00000000  // nop
};
#endif


/* static */ void
(*ES_Native::GetThrowFromMachineCode())(void **)
{
#ifdef CONSTANT_DATA_IS_EXECUTABLE
    return reinterpret_cast<void (*)(void **)>(ES_CodeGenerator::GetFPUType() == ES_CodeGenerator::FPU_NONE ? cv_ThrowFromMachineCode : cv_ThrowFromMachineCode_fpu);
#else // CONSTANT_DATA_IS_EXECUTABLE
    if (!g_ecma_throw_from_machine_code_block)
    {
        if (ES_CodeGenerator::GetFPUType() == ES_CodeGenerator::FPU_NONE)
        {
            g_ecma_throw_from_machine_code_block = g_executableMemory->AllocateL(sizeof cv_ThrowFromMachineCode);
            op_memcpy(g_ecma_throw_from_machine_code_block->address, cv_ThrowFromMachineCode, sizeof cv_ThrowFromMachineCode);
            OpExecMemoryManager::FinalizeL(g_ecma_throw_from_machine_code_block);
        }
        else
        {
            g_ecma_throw_from_machine_code_block = g_executableMemory->AllocateL(sizeof cv_ThrowFromMachineCode_fpu);
            op_memcpy(g_ecma_throw_from_machine_code_block->address, cv_ThrowFromMachineCode_fpu, sizeof cv_ThrowFromMachineCode_fpu);
            OpExecMemoryManager::FinalizeL(g_ecma_throw_from_machine_code_block);
        }
    }

    return reinterpret_cast<void (*)(void **)>(g_ecma_throw_from_machine_code_block->address);
#endif // CONSTANT_DATA_IS_EXECUTABLE
}

#ifdef DEBUG_ENABLE_OPASSERT
/* static */ BOOL
ES_Native::IsAddressInBytecodeToNativeTrampoline(void *address)
{
#ifdef CONSTANT_DATA_IS_EXECUTABLE
    if (ES_CodeGenerator::GetFPUType() != ES_CodeGenerator::FPU_NONE)
        return reinterpret_cast<unsigned *>(address) >= cv_BytecodeToNativeTrampoline_fpu && reinterpret_cast<unsigned *>(address) < cv_BytecodeToNativeTrampoline_fpu + ARRAY_SIZE(cv_BytecodeToNativeTrampoline_fpu);
    else
        return reinterpret_cast<unsigned *>(address) >= cv_BytecodeToNativeTrampoline && reinterpret_cast<unsigned *>(address) < cv_BytecodeToNativeTrampoline + ARRAY_SIZE(cv_BytecodeToNativeTrampoline);
#else // CONSTANT_DATA_IS_EXECUTABLE
    return TRUE;
#endif // CONSTANT_DATA_IS_EXECUTABLE
}
#endif // DEBUG_ENABLE_OPASSERT

void
ES_Native::UpdateCode(ES_Code *new_code)
{
    pointer_const_template[CONST_CODE] = code = new_code;
    OP_ASSERT(pointer_constants[CONST_CODE] == NULL);
}

/* Inspect ES_Native::code and return FALSE if the architecture doesn't support JIT compiling that code. */
BOOL
ES_Native::CheckLimits()
{
    if (code->data->register_frame_size * sizeof(ES_Value_Internal) > SHRT_MAX)
        return FALSE;

    // Calculate the worst case data section size.

    unsigned data_section_size = 0;

    data_section_size += code->data->doubles_count * sizeof(double);
    data_section_size += NUMBER_OF_INT_CONSTANTS * sizeof(int);
    data_section_size += NUMBER_OF_FUNCTION_CONSTANTS * sizeof(void *);
    data_section_size += NUMBER_OF_POINTER_CONSTANTS * sizeof(void *);

    if (code->data->switch_tables_count)
    {
        ES_CodeStatic::SwitchTable *switch_tables = code->data->switch_tables;
        unsigned switch_entries_count = code->data->switch_tables_count;

        for (unsigned i = 0; i < code->data->switch_tables_count; i++)
            switch_entries_count += switch_tables[i].maximum - switch_tables[i].minimum + 1;

        data_section_size += switch_entries_count * sizeof(void *);
    }

    return data_section_size <= ES_DATA_SECTION_MAX_SIZE;
}

/* Set up register allocator appropriately by allocating and populating the 'native_registers' array. */
void
ES_Native::InitializeNativeRegisters()
{
    unsigned i, j, step = 1;

    integer_registers_count = 19;

    switch (fpu_type)
    {
    default:
    case ES_CodeGenerator::FPU_NONE:
        double_registers_count = 0;
        break;

    case ES_CodeGenerator::FPU_SINGLE:
        double_registers_count = 14;
        step = 2;
        break;

    case ES_CodeGenerator::FPU_DOUBLE:
        // TODO: Use registers 29 and 31 as well.
        double_registers_count = 28;
        break;
    }

    native_registers_count = integer_registers_count + double_registers_count;

    native_registers = OP_NEWGROA_L(NativeRegister, native_registers_count, Arena());

    memset(native_registers, 0, sizeof(NativeRegister) * native_registers_count);

    for (i = 0; i < integer_registers_count; i++)
    {
        native_registers[i].register_code = i + 2;
        native_registers[i].type = NativeRegister::TYPE_INTEGER;
    }

    for (i = integer_registers_count, j = 0; i < native_registers_count; i++, j += step)
    {
        native_registers[i].register_code = j;
        native_registers[i].type = NativeRegister::TYPE_DOUBLE;
    }
}

/* Emit a dynamic register type check, and call instruction handler if it fails. */
void
ES_Native::EmitRegisterTypeCheck(VirtualRegister *source, ES_ValueType value_type, ES_CodeGenerator::JumpTarget *slow_case, BOOL invert_result)
{
    ANNOTATE("EmitRegisterTypeCheck");

    OP_ASSERT((property_value_fail && property_value_fail == slow_case) || property_value_needs_type_check || property_value_read_vr == NULL || property_value_read_vr != source);
    OP_ASSERT(source->stack_frame_offset == INT_MAX);

    if (!slow_case)
    {
        if (!current_slow_case)
            EmitSlowCaseCall();

        slow_case = current_slow_case->GetJumpTarget();
    }

    if (property_value_read_vr == source && property_value_nr && !property_value_fail)
        cg.Load(REG_T8, property_value_offset + VALUE_TYPE_OFFSET, INTEGER_REGISTER(property_value_nr));
    else
        cg.LW(REG_T8, TYPE_OFFSET(source), BASE_REGISTER(source));

    if (value_type == ESTYPE_INT32_OR_DOUBLE)
    {
        cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));
        if (invert_result)
            cg.JumpLE(slow_case, REG_T8, REG_T9);
        else
            cg.JumpGT(slow_case, REG_T8, REG_T9);
    }
    else if (value_type == ESTYPE_DOUBLE)
    {
        cg.Load(REG_T9, GetConst(CONST_ESTYPE_DOUBLE));
        if (invert_result)
            cg.JumpLE(slow_case, REG_T8, REG_T9);
        else
            cg.JumpGT(slow_case, REG_T8, REG_T9);
    }
    else
    {
        cg.Load(REG_T9, GetTypeConst(*this, value_type));
        if (invert_result)
            cg.JumpEQ(slow_case, REG_T8, REG_T9);
        else
            cg.JumpNE(slow_case, REG_T8, REG_T9);
    }
}

/* Load a virtual register's value into a native register. */
void
ES_Native::EmitLoadRegisterValue(NativeRegister *target, VirtualRegister *source, ES_CodeGenerator::JumpTarget *type_check_fail)
{
    ANNOTATE("EmitLoadRegisterValue");

    if (source->stack_frame_offset != INT_MAX)
    {
        OP_ASSERT((target->type == NativeRegister::TYPE_INTEGER && source->stack_frame_type != ESTYPE_DOUBLE) ||
            (target->type == NativeRegister::TYPE_DOUBLE && source->stack_frame_type == ESTYPE_DOUBLE));

        type_check_fail = NULL;
    }

    if (target->type == NativeRegister::TYPE_INTEGER)
    {
        if (type_check_fail)
            EmitRegisterTypeCheck(source, ESTYPE_INT32, type_check_fail);

        cg.LW(INTEGER_REGISTER(target), IVALUE_OFFSET(source), BASE_REGISTER(source));
    }

    if (target->type == NativeRegister::TYPE_DOUBLE)
    {
        if (type_check_fail)
            EmitRegisterTypeCheck(source, ESTYPE_DOUBLE, type_check_fail);

        cg.LDC1(DOUBLE_REGISTER(target), DVALUE_OFFSET(source), BASE_REGISTER(source));
    }
}

/* Store a native register's value into a virtual register. */
void
ES_Native::EmitStoreRegisterValue(VirtualRegister *target, NativeRegister *source, BOOL write_type, BOOL saved_condition)
{
    ANNOTATE("EmitStoreRegisterValue");

    if (target->stack_frame_offset != INT_MAX)
    {
        OP_ASSERT((source->type == NativeRegister::TYPE_INTEGER && target->stack_frame_type != ESTYPE_DOUBLE) ||
            (source->type == NativeRegister::TYPE_DOUBLE && target->stack_frame_type == ESTYPE_DOUBLE));

        write_type = FALSE;
    }

    if (source->type == NativeRegister::TYPE_INTEGER)
    {
        if (write_type)
            cg.Load(REG_T8, GetConst(CONST_ESTYPE_INT32));

        cg.SW(INTEGER_REGISTER(source), IVALUE_OFFSET(target), BASE_REGISTER(target));

        if (write_type)
            cg.SW(REG_T8, TYPE_OFFSET(target), BASE_REGISTER(target));
    }

    if (source->type == NativeRegister::TYPE_DOUBLE)
    {
        // MIPS represents silent NaNs as 0x7ff7ffff... wich is < ESTYPE_DOUBLE so it's safe to omit the NaN check here.
        cg.SDC1(DOUBLE_REGISTER(source), DVALUE_OFFSET(target), BASE_REGISTER(target));
    }
}

void
ES_Native::EmitStoreGlobalObject(VirtualRegister *target)
{
    ANNOTATE("EmitStoreGlobalObject");

    cg.Load(REG_T8, GetConst(CONST_GLOBAL_OBJECT));

    if (target->stack_frame_offset == INT_MAX)
        cg.Load(REG_T9, GetConst(CONST_ESTYPE_OBJECT));

    cg.SW(REG_T8, IVALUE_OFFSET(target), BASE_REGISTER(target));

    if (target->stack_frame_offset == INT_MAX)
        cg.SW(REG_T9, TYPE_OFFSET(target), REG_FP);
}

void
ES_Native::EmitStoreConstantBoolean(VirtualRegister *target, BOOL value)
{
    ANNOTATE("EmitStoreConstantBoolean");

    if (target->stack_frame_offset == INT_MAX)
        cg.Load(REG_T9, GetConst(CONST_ESTYPE_BOOLEAN));

    if (value)
    {
        cg.Load(REG_T8, 1);
        cg.SW(REG_T8, IVALUE_OFFSET(target), BASE_REGISTER(target));
    }
    else
        cg.SW(REG_Z, IVALUE_OFFSET(target), BASE_REGISTER(target));

    if (target->stack_frame_offset == INT_MAX)
        cg.SW(REG_T9, TYPE_OFFSET(target), REG_FP);
}

#define TYPE_TEST(T) \
    ES_CodeGenerator::JumpTarget *skip = NULL; \
    \
    if (type_test && !(last == T && handled == possible)) \
    { \
        if (first != T) \
            cg.Sub(REG_T8, diff); \
    \
        if (last == T) \
            cg.Jump(slow_case, ES_NATIVE_CONDITION_GTZ(REG_T8)); \
        else \
        { \
            skip = cg.ForwardJump(); \
            cg.Jump(skip, ES_NATIVE_CONDITION_GTZ(REG_T8)); \
        } \
    }

#define JUMP_FINISHED(T) if (last != T) cg.Jump(finished, ES_NATIVE_UNCONDITIONAL, TRUE);

#define SKIP_TARGET if (skip) cg.SetJumpTarget(skip);

/* Load a virtual register's value into a native register, with type check and conversion. */
void
ES_Native::EmitConvertRegisterValue(NativeRegister *target, VirtualRegister *source, unsigned handled, unsigned possible, ES_CodeGenerator::JumpTarget *slow_case)
{
    ANNOTATE("EmitConvertRegisterValue");

    OP_ASSERT(handled);
    OP_ASSERT(handled == possible || slow_case);
    OP_ASSERT(source->stack_frame_offset == INT_MAX || (handled == possible && source->stack_frame_type == ES_Value_Internal::TypeFromBits(handled)));

    ES_CodeGenerator::JumpTarget *finished = cg.ForwardJump();

    ES_ValueType first, last;
    int diff = 2;

    if (handled & ESTYPE_BITS_DOUBLE)
        first = ESTYPE_DOUBLE;
    else if (handled & ESTYPE_BITS_INT32)
        first = ESTYPE_INT32;
    else if (handled & (ESTYPE_BITS_UNDEFINED | ESTYPE_BITS_NULL))
        first = ESTYPE_NULL;
    else if (handled & ESTYPE_BITS_BOOLEAN)
        first = ESTYPE_BOOLEAN;
    else
    {
        // TODO: Goto slow case?
        OP_ASSERT(FALSE);
        return;
    }

    if (handled & ESTYPE_BITS_BOOLEAN)
        last = ESTYPE_BOOLEAN;
    else if (handled & (ESTYPE_BITS_UNDEFINED | ESTYPE_BITS_NULL))
        last = ESTYPE_NULL;
    else if (handled & ESTYPE_BITS_INT32)
        last = ESTYPE_INT32;
    else if (handled & ESTYPE_BITS_DOUBLE)
        last = ESTYPE_DOUBLE;
    else
    {
        OP_ASSERT(FALSE);
        return;
    }

    BOOL type_test = (handled != possible) || (first != last);

    if (type_test)
    {
        cg.LW(REG_T8, TYPE_OFFSET(source), BASE_REGISTER(source));
        cg.Load(REG_T9, GetTypeConst(*this, first));

        if (first > ESTYPE_DOUBLE)
        {
            if (handled != possible)
                cg.JumpLT(slow_case, REG_T8, REG_T9);
            cg.Sub(REG_T8, REG_T9);
        }
    }

    if (target->type == NativeRegister::TYPE_INTEGER)
    {
        ANNOTATE("EmitConvertRegisterValue, integer target");

        if (handled & ESTYPE_BITS_DOUBLE)
        {
            ANNOTATE("EmitConvertRegisterValue, double source");

            ES_CodeGenerator::JumpTarget *skip = NULL;

            if (type_test && !(last == ESTYPE_DOUBLE && handled == possible))
            {
                if (last == ESTYPE_DOUBLE)
                    cg.JumpGT(slow_case, REG_T8, REG_T9);
                else
                {
                    skip = cg.ForwardJump();
                    cg.JumpGT(skip, REG_T8, REG_T9);
                }
            }

            cg.LDC1(REG_F28, DVALUE_OFFSET(source), BASE_REGISTER(source));
            cg.TRUNC(W, D, REG_F30, REG_F28);

            CheckFPUException(cg, slow_case);

            cg.MFC1(INTEGER_REGISTER(target), REG_F30);
            JUMP_FINISHED(ESTYPE_DOUBLE);

            if (skip)
            {
                cg.SetJumpTarget(skip);
                cg.Sub(REG_T8, REG_T9);
            }
        }

        if (handled & ESTYPE_BITS_INT32)
        {
            ANNOTATE("EmitConvertRegisterValue, int source");

            TYPE_TEST(ESTYPE_INT32);

            cg.LW(INTEGER_REGISTER(target), IVALUE_OFFSET(source), BASE_REGISTER(source));
            JUMP_FINISHED(ESTYPE_INT32);

            SKIP_TARGET;
        }
        else
            diff += 2;

        if (handled & (ESTYPE_BITS_UNDEFINED | ESTYPE_BITS_NULL))
        {
            ANNOTATE("EmitConvertRegisterValue, undefined or null source");

            diff = 1;

            TYPE_TEST(ESTYPE_NULL);

            cg.Move(INTEGER_REGISTER(target), REG_Z);
            JUMP_FINISHED(ESTYPE_NULL);

            SKIP_TARGET;
        }
        else
            diff++;

        if (handled & ESTYPE_BITS_BOOLEAN)
        {
            ANNOTATE("EmitConvertRegisterValue, boolean source");

            TYPE_TEST(ESTYPE_BOOLEAN);

            cg.LW(INTEGER_REGISTER(target), IVALUE_OFFSET(source), BASE_REGISTER(source));
            JUMP_FINISHED(ESTYPE_BOOLEAN);

            SKIP_TARGET;
        }
    }

    if (target->type == NativeRegister::TYPE_DOUBLE)
    {
        ANNOTATE("EmitConvertRegisterValue, double target");

        if (handled & ESTYPE_BITS_DOUBLE)
        {
            ANNOTATE("EmitConvertRegisterValue, double source");

            ES_CodeGenerator::JumpTarget *skip = NULL;

            if (type_test && !(last == ESTYPE_DOUBLE && handled == possible))
            {
                if (last == ESTYPE_DOUBLE)
                    cg.JumpGT(slow_case, REG_T8, REG_T9);
                else
                {
                    skip = cg.ForwardJump();
                    cg.JumpGT(skip, REG_T8, REG_T9);
                }
            }

            cg.LDC1(DOUBLE_REGISTER(target), DVALUE_OFFSET(source), BASE_REGISTER(source));
            JUMP_FINISHED(ESTYPE_DOUBLE);

            if (skip)
            {
                cg.SetJumpTarget(skip);
                cg.Sub(REG_T8, REG_T9);
            }
        }

        if (handled & ESTYPE_BITS_INT32)
        {
            ANNOTATE("EmitConvertRegisterValue, int source");

            TYPE_TEST(ESTYPE_INT32);

            cg.LWC1(REG_F28, IVALUE_OFFSET(source), BASE_REGISTER(source));

            cg.CVT(D, W, DOUBLE_REGISTER(target), REG_F28);
            JUMP_FINISHED(ESTYPE_INT32);

            SKIP_TARGET;
        }
        else
            diff += 2;

        if (handled & (ESTYPE_BITS_UNDEFINED | ESTYPE_BITS_NULL))
        {
            ANNOTATE("EmitConvertRegisterValue, undefined or null source");

            diff = 1;

            TYPE_TEST(ESTYPE_NULL);

            cg.MTC1(REG_Z, REG_F28);

            cg.CVT(D, W, DOUBLE_REGISTER(target), REG_F28);
            JUMP_FINISHED(ESTYPE_NULL);

            SKIP_TARGET;
        }
        else
            diff++;

        if (handled & ESTYPE_BITS_BOOLEAN)
        {
            ANNOTATE("EmitConvertRegisterValue, boolean source");

            TYPE_TEST(ESTYPE_BOOLEAN);

            cg.LWC1(REG_F28, IVALUE_OFFSET(source), BASE_REGISTER(source));

            cg.CVT(D, W, DOUBLE_REGISTER(target), REG_F28);
            JUMP_FINISHED(ESTYPE_BOOLEAN);

            SKIP_TARGET;
        }
    }

    cg.SetJumpTarget(finished);
}

#undef TYPE_TEST
#undef JUMP_FINISHED
#undef SKIP_TARGET

/* Set only a virtual register's type.  If 'type' is ESTYPE_UNDEFINED or
   ESTYPE_NULL this is of course sort of equivalent to setting its value as well. */
void
ES_Native::EmitSetRegisterType(VirtualRegister *target, ES_ValueType type)
{
    ANNOTATE("EmitSetRegisterType");

    OP_ASSERT(target->stack_frame_offset == INT_MAX);

    if (type != ESTYPE_DOUBLE)
    {
        cg.Load(REG_T8, GetTypeConst(*this, type));
        cg.SW(REG_T8, TYPE_OFFSET(target), BASE_REGISTER(target));
    }
}

/* Load a constant value directly into a virtual register. */
void
ES_Native::EmitSetRegisterValue(VirtualRegister *target, const ES_Value_Internal &value, BOOL write_type, BOOL saved_condition)
{
    ANNOTATE("EmitSetRegisterValue");

    if (target->stack_frame_offset == INT_MAX)
    {
        cg.Load(REG_T8, value.private_value.i32);
        if (value.IsDouble())
            cg.Load(REG_T9, value.private_type);
        else
            cg.Load(REG_T9, GetTypeConst(*this, value.private_type));
        cg.SW(REG_T8, IVALUE_OFFSET(target), REG_FP);
        cg.SW(REG_T9, TYPE_OFFSET(target), REG_FP);
    }
    else
    {
        if (value.IsDouble())
        {
            Constant *constant = cg.NewConstant(value.GetDouble());
            cg.Load(REG_F28, constant);
            cg.SDC1(REG_F28, DVALUE_OFFSET(target), REG_SP);
        }
        else
        {
            cg.Load(REG_T8, value.private_value.i32);
            cg.SW(REG_T8, IVALUE_OFFSET(target), REG_SP);
        }
    }
}

void
ES_Native::EmitSetRegisterValueFromStackFrameStorage(VirtualRegister *target)
{
    ANNOTATE("EmitSetRegisterValueFromStackFrameStorage");

    OP_ASSERT(target->stack_frame_offset != INT_MAX);

    if (target->stack_frame_type == ESTYPE_DOUBLE)
    {
        cg.LW(REG_T8, DVALUE_OFFSET(target), REG_SP);
        cg.LW(REG_T9, DVALUE_OFFSET(target) + 4, REG_SP);

        target->stack_frame_offset = INT_MAX;

        cg.SW(REG_T8, DVALUE_OFFSET(target), REG_FP);
        cg.SW(REG_T9, DVALUE_OFFSET(target) + 4, REG_FP);
    }
    else
    {
        cg.LW(REG_T8, IVALUE_OFFSET(target), REG_SP);
        cg.Load(REG_T9, GetTypeConst(*this, target->stack_frame_type));

        target->stack_frame_offset = INT_MAX;

        cg.SW(REG_T8, IVALUE_OFFSET(target), REG_FP);
        cg.SW(REG_T9, TYPE_OFFSET(target), REG_FP);
    }
}

/* Returns TRUE if storing the native register to the given virtual register may side-effect the condition flags. */
BOOL
ES_Native::IsComplexStore(VirtualRegister *virtual_register, NativeRegister *native_register)
{
    return FALSE;
}

/* Save away processor condition flags from within a block. */
void
ES_Native::EmitSaveCondition()
{
    OP_ASSERT(FALSE);
}

/* Restore the condition flags (EFLAGS) previously saved by EmitSaveCondition(). */
void
ES_Native::EmitRestoreCondition()
{
    OP_ASSERT(FALSE);
}

/* Copy value from 'source' to 'target'.  If one is an integer register
   and the other is a double register, the value should be converted
   (trivial from integer to double, as by ToInt32() from double to
   integer.)*/
void
ES_Native::EmitRegisterCopy(const Operand &source, const Operand &target, ES_CodeGenerator::JumpTarget *slow_case)
{
    ANNOTATE("EmitRegisterCopy");

    if (source.native_register && target.virtual_register)
        EmitStoreRegisterValue(target.virtual_register, source.native_register, TRUE);
    else if (source.virtual_register && target.native_register)
    {
        if (source.codeword)
        {
            OP_ASSERT(target.native_register->type == NativeRegister::TYPE_INTEGER);
            cg.Load(INTEGER_REGISTER(target), source.codeword->immediate);
        }
        else
            EmitLoadRegisterValue(target.native_register, source.virtual_register, NULL);
    }
    else if (source.codeword)
        EmitRegisterInt32Assign(source, target);
    else if (source.native_register && target.native_register)
    {
        if (target.native_register->type == NativeRegister::TYPE_DOUBLE)
        {
            if (source.native_register->type == NativeRegister::TYPE_INTEGER)
            {
                cg.MTC1(INTEGER_REGISTER(source.native_register), REG_F28);
                cg.CVT(D, W, DOUBLE_REGISTER(target.native_register), REG_F28);
            }
            else
                cg.MOV(D, DOUBLE_REGISTER(target.native_register), DOUBLE_REGISTER(source.native_register));
        }
        else
        {
            if (source.native_register->type == NativeRegister::TYPE_INTEGER)
                cg.Move(INTEGER_REGISTER(target.native_register), INTEGER_REGISTER(source.native_register));
            else
            {
                cg.TRUNC(W, D, REG_F28, DOUBLE_REGISTER(source.native_register));

                CheckFPUException(cg, slow_case);

                cg.MFC1(INTEGER_REGISTER(target.native_register), REG_F28);
            }
        }
    }
    else
        OP_ASSERT(FALSE);
}

void
ES_Native::EmitRegisterCopy(VirtualRegister *source, VirtualRegister *target)
{
    ANNOTATE("EmitRegisterCopy");

    CopyValue(cg, source, target);
}

void
ES_Native::EmitRegisterInt32Copy(VirtualRegister *source, VirtualRegister *target)
{
    ANNOTATE("EmitRegisterInt32Copy");

    cg.LW(REG_T8, IVALUE_OFFSET(source), BASE_REGISTER(source));

    if (target->stack_frame_offset == INT_MAX)
        cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));

    cg.SW(REG_T8, IVALUE_OFFSET(target), BASE_REGISTER(target));

    if (target->stack_frame_offset == INT_MAX)
        cg.SW(REG_T9, TYPE_OFFSET(target), REG_FP);
}

void
ES_Native::EmitRegisterInt32Assign(const Operand &source, const Operand &target)
{
    ANNOTATE("EmitRegisterInt32Assign");

    OP_ASSERT(source.codeword);

    Register itarget;

    if (target.native_register && target.native_register->type == NativeRegister::TYPE_INTEGER)
        itarget = INTEGER_REGISTER(target);
    else
        itarget = REG_T8;

    cg.Load(itarget, source.codeword->immediate);

    if (target.native_register)
        if (target.native_register->type == NativeRegister::TYPE_INTEGER)
            return;
        else
        {
            cg.MTC1(itarget, REG_F28);
            cg.CVT(D, W, DOUBLE_REGISTER(target), REG_F28);
        }
    else
    {
        cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));
        cg.SW(itarget, IVALUE_OFFSET(target.virtual_register), BASE_REGISTER(target.virtual_register));
        cg.SW(REG_T9, TYPE_OFFSET(target.virtual_register), BASE_REGISTER(target.virtual_register));
    }
}

void
ES_Native::EmitRegisterInt32Assign(int source, NativeRegister *target)
{
    ANNOTATE("EmitRegisterInt32Assign");

    cg.Load(INTEGER_REGISTER(target), source);
}

void
ES_Native::EmitRegisterDoubleAssign(const double *value, const Operand &target)
{
    ANNOTATE("EmitRegisterDoubleAssign");

    OP_ASSERT(target.native_register || target.virtual_register);

    if (target.native_register)
    {
        Constant *constant = cg.NewConstant(*value);
        cg.Load(DOUBLE_REGISTER(target), constant);
    }
    else if (target.virtual_register->stack_frame_offset == INT_MAX && op_isnan(*value))
    {
        cg.Load(REG_T8, GetConst(CONST_ESTYPE_DOUBLE));
        cg.SW(REG_T8, TYPE_OFFSET(target.virtual_register), BASE_REGISTER(target.virtual_register));
    }
    else
    {
        Constant *constant = cg.NewConstant(*value);

        cg.Load(REG_F28, constant);
        cg.SDC1(REG_F28, DVALUE_OFFSET(target.virtual_register), BASE_REGISTER(target.virtual_register));
    }
}

void
ES_Native::EmitRegisterStringAssign(JString *value, const Operand &target)
{
    ANNOTATE("EmitRegisterStringAssign");

    OP_ASSERT(target.native_register || target.virtual_register);

    if (target.native_register)
        cg.Load(INTEGER_REGISTER(target), value);
    else
    {
        cg.Load(REG_T8, GetConst(CONST_ESTYPE_STRING));
        cg.Load(REG_T9, value);
        cg.SW(REG_T8, TYPE_OFFSET(target.virtual_register), BASE_REGISTER(target.virtual_register));
        cg.SW(REG_T9, IVALUE_OFFSET(target.virtual_register), BASE_REGISTER(target.virtual_register));
    }
}

/* Emit a call to the instruction handler for ESI_TOPRIMITIVE(1) if the value in the register is an object. */
void
ES_Native::EmitToPrimitive(VirtualRegister *source)
{
    ANNOTATE("EmitToPrimitive");

    if (!current_slow_case)
        EmitSlowCaseCall();

    cg.LW(REG_T8, TYPE_OFFSET(source), BASE_REGISTER(source));
    cg.Load(REG_T9, GetConst(CONST_ESTYPE_OBJECT));
    cg.Jump(current_slow_case->GetJumpTarget(), ES_NATIVE_CONDITION_EQ(REG_T8, REG_T9));
}

/* Simply emit a call to an appropriate instruction handler (the instruction code to handle is 'word->instruction'.) */
void
ES_Native::EmitInstructionHandlerCall()
{
    DECLARE_NOTHING();

    ANNOTATE("EmitInstructionHandlerCall");

    if (is_light_weight)
        LEAVE(OpStatus::ERR);

    ES_CodeWord *word = CurrentCodeWord();
    ES_Instruction instruction = word->instruction;

    if (instruction == ESI_TOPRIMITIVE)
        instruction = ESI_TOPRIMITIVE1;

    if (word == entry_point_cw && !entry_point_jump_target)
        entry_point_jump_target = EmitEntryPoint();

    switch (instruction)
    {
    case ESI_EVAL:
    case ESI_CALL:
    case ESI_CONSTRUCT:

        cg.Load(REG_A1, cw_index + (constructor ? code->data->codewords_count : 0));

        if (word->instruction == ESI_EVAL || word->instruction == ESI_CALL)
            CallDispatcherFunc(cg, GetConst(CONST_EXC_CALLFROMMACHINECODE));
        else
            CallDispatcherFunc(cg, GetConst(CONST_EXC_CONSTRUCTFROMMACHINECODE));

        break;

    case ESI_GETN_IMM:
    case ESI_GET_LENGTH:
    case ESI_PUTN_IMM:
    case ESI_INIT_PROPERTY:

        cg.Load(REG_A1, cw_index + (constructor ? code->data->codewords_count : 0));

        if (instruction == ESI_GETN_IMM || instruction == ESI_GET_LENGTH)
            CallDispatcherFunc(cg, GetConst(CONST_EXC_GETNAMEDIMMEDIATE));
        else
            CallDispatcherFunc(cg, GetConst(CONST_EXC_PUTNAMEDIMMEDIATE));

        break;

    case ESI_GET_GLOBAL:
    case ESI_PUT_GLOBAL:

        cg.Load(REG_A1, cw_index + (constructor ? code->data->codewords_count : 0));

        if (instruction == ESI_GET_GLOBAL)
        {
            if (code->global_caches[word[3].index].class_id == ES_Class::GLOBAL_IMMEDIATE_CLASS_ID)
                CallDispatcherFunc(cg, GetConst(CONST_EXC_GETGLOBALIMMEDIATE));
            else
                CallDispatcherFunc(cg, GetConst(CONST_EXC_GETGLOBAL));
        }
        else
            CallDispatcherFunc(cg, GetConst(CONST_EXC_PUTGLOBAL));

        break;

    case ESI_REDIRECTED_CALL:
        if (constructor)
        {
            cg.Load(REG_T8, 1);
            cg.SW(REG_T8, ES_OFFSETOF(ES_Execution_Context, in_constructor), REG_S7);
        }
        else
            cg.SW(REG_Z, ES_OFFSETOF(ES_Execution_Context, in_constructor), REG_S7);

        // Fall through to default

    default:
        cg.Load(REG_A1, code->data->codewords + cw_index + 1);

        cg.Load(REG_T9, g_ecma_instruction_handlers[instruction]);
        cg.JALR(REG_T9);
        cg.Move(REG_A0, REG_S7);

        /* Clear the in_constructor flag set prior to call */
        if (instruction == ESI_REDIRECTED_CALL && constructor)
            cg.SW(REG_Z, ES_OFFSETOF(ES_Execution_Context, in_constructor), REG_S7);

        switch (word->instruction)
        {
        case ESI_GET:
        case ESI_GETI_IMM:
        case ESI_PUT:
        case ESI_PUTI_IMM:
            cg.Load(REG_A1, code->data->instruction_offsets[instruction_index + 1] + (constructor ? code->data->codewords_count : 0));
            CallDispatcherFunc(cg, GetConst(CONST_EXC_UPDATENATIVEDISPATCHER));
            break;
        }
        break;
    }
}

void
ES_Native::EmitSlowCaseCall(BOOL failed_inlined_function)
{
    DECLARE_NOTHING();

    if (is_light_weight)
    {
        if (!light_weight_failure)
            EmitLightWeightFailure();

        current_slow_case = light_weight_failure;
    }
    else
    {
        current_slow_case = cg.StartOutOfOrderBlock();

        ANNOTATE("EmitSlowCaseCall");

#ifdef ES_SLOW_CASE_PROFILING
        extern BOOL g_slow_case_summary;
        if (g_slow_case_summary)
        {
            cg.Load(REG_T8, context->rt_data->slow_case_calls + CurrentCodeWord()->instruction);
            cg.LW(REG_T9, 0, REG_T8);
            cg.Add(REG_T9, 1);
            cg.SW(REG_T9, 0, REG_T8);
        }
#endif // ES_SLOW_CASE_PROFILING

        if (property_value_read_vr && property_value_nr)
        {
            CopyTypedDataToValue(*this, INTEGER_REGISTER(property_value_nr), property_value_offset, property_value_needs_type_check ? ES_STORAGE_WHATEVER : ES_STORAGE_OBJECT, property_value_read_vr);

            current_slow_case_main = cg.Here();
        }
        else
            current_slow_case_main = NULL;

        if (failed_inlined_function)
        {
            cg.Load(REG_T8, GetConst(CONST_CODE));
            cg.Load(REG_T9, 1);
            cg.SW(REG_T9, ES_OFFSETOF(ES_Code, has_failed_inlined_function), REG_T8);
        }

        EmitInstructionHandlerCall();

        cg.EndOutOfOrderBlock();
    }

    if (property_value_fail)
    {
        cg.StartOutOfOrderBlock();
        cg.SetJumpTarget(property_value_fail);

        if (is_light_weight || current_slow_case_main == NULL)
            cg.Jump(current_slow_case->GetJumpTarget());
        else
            cg.Jump(current_slow_case_main);

        cg.EndOutOfOrderBlock(FALSE);
        property_value_fail = NULL;
    }
}

void
ES_Native::EmitLightWeightFailure()
{
    if (!light_weight_failure)
    {
        light_weight_failure = cg.StartOutOfOrderBlock();

        ANNOTATE("EmitLightWeightFailure");

        cg.Load(REG_A1, fncode->GetData()->formals_count);

        light_weight_wrong_argc = cg.Here();

        cg.Move(REG_RA, REG_S5);

        frame_size = CreateNativeStackFrame(*this);

        CallDispatcherFunc(cg, GetConst(CONST_EXC_LIGHTWEIGHTDISPATCHERFAILURE));

        cg.LW(REG_RA, SP_OFFSET_RETURN_ADDRESS(frame_size), REG_SP);
        cg.JR();
        cg.Add(REG_SP, frame_size);

        cg.EndOutOfOrderBlock(FALSE);
    }
}

void
ES_Native::EmitExecuteBytecode(unsigned start_instruction_index, unsigned end_instruction_index, BOOL last_in_slow_case)
{
    ANNOTATE("EmitExecuteBytecode");

    cg.Load(REG_A1, start_instruction_index);

    if (constructor)
        cg.Load(REG_A2, ~end_instruction_index);
    else
        cg.Load(REG_A2, end_instruction_index);

    CallDispatcherFunc(cg, GetConst(CONST_EXC_EXECUTEBYTECODE));
}

/* Emit int32 arithmetic instruction.  If the result isn't an int32 (arithmetic over- or underflow,) */
void
ES_Native::EmitInt32Arithmetic(Int32ArithmeticType type, const Operand &target, const Operand &lhs, const Operand &rhs, BOOL as_condition, ES_CodeGenerator::JumpTarget *overflow_target)
{
    ANNOTATE("EmitInt32Arithmetic");

    OP_ASSERT(target.native_register);
    OP_ASSERT(lhs.native_register || lhs.codeword);
    OP_ASSERT(rhs.native_register || rhs.codeword);
    OP_ASSERT(lhs.native_register || rhs.native_register);

    Register rtarget = INTEGER_REGISTER(target);
    Register rlhs, rrhs;

    if (lhs.native_register)
        rlhs = INTEGER_REGISTER(lhs);
    else if (!lhs.codeword)
    {
        UseInPlace(lhs.virtual_register);
        cg.LW(REG_T8, IVALUE_OFFSET(lhs.virtual_register), BASE_REGISTER(lhs.virtual_register));
        rlhs = REG_T8;
    }
    else
    {
        cg.Load(REG_T8, lhs.codeword->immediate);
        rlhs = REG_T8;
    }

    if (rhs.native_register)
        rrhs = INTEGER_REGISTER(rhs);
    else if (!rhs.codeword)
    {
        UseInPlace(rhs.virtual_register);
        cg.LW(REG_T9, IVALUE_OFFSET(rhs.virtual_register), BASE_REGISTER(rhs.virtual_register));
        rrhs = REG_T9;
    }
    else
    {
        cg.Load(REG_T9, rhs.codeword->immediate);
        rrhs = REG_T9;
    }

    if (as_condition)
        condition_register = rtarget;

    switch (type)
    {
    case INT32ARITHMETIC_LSHIFT:
        cg.SLLV(rtarget, rlhs, rrhs);
        break;

    case INT32ARITHMETIC_RSHIFT_SIGNED:
        cg.SRAV(rtarget, rlhs, rrhs);
        break;

    case INT32ARITHMETIC_RSHIFT_UNSIGNED:
        cg.SRLV(rtarget, rlhs, rrhs);

        if (overflow_target)
        {
            cg.ANDI(REG_T8, rrhs, 31);
            cg.SLT(REG_T9, rtarget, REG_Z);
            cg.SLT(REG_T8, REG_T8, REG_T9);
            cg.Jump(overflow_target, ES_NATIVE_CONDITION_NEZ(REG_T8));
        }
        break;

    case INT32ARITHMETIC_AND:
        cg.AND(rtarget, rlhs, rrhs);
        break;

    case INT32ARITHMETIC_OR:
        cg.OR(rtarget, rlhs, rrhs);
        break;

    case INT32ARITHMETIC_XOR:
        cg.XOR(rtarget, rlhs, rrhs);
        break;

    case INT32ARITHMETIC_ADD:
        if (overflow_target)
        {
            ES_CodeGenerator::JumpTarget *safe = cg.ForwardJump();

            cg.XOR(REG_AT, rlhs, rrhs);

            cg.ADDU(rtarget, rlhs, rrhs);
            cg.Jump(safe, ES_NATIVE_CONDITION_LTZ(REG_AT), TRUE);

            cg.XOR(REG_AT, rtarget, rlhs);
            cg.Jump(overflow_target, ES_NATIVE_CONDITION_LTZ(REG_AT));

            cg.SetJumpTarget(safe);
        }
        else
            cg.ADDU(rtarget, rlhs, rrhs);

        break;

    case INT32ARITHMETIC_SUB:
        if (overflow_target)
        {
            ES_CodeGenerator::JumpTarget *safe = cg.ForwardJump();

            cg.XOR(REG_AT, rlhs, rrhs);

            cg.SUBU(rtarget, rlhs, rrhs);
            cg.Jump(safe, ES_NATIVE_CONDITION_GEZ(REG_AT), TRUE);

            cg.XOR(REG_AT, rtarget, rlhs);
            cg.Jump(overflow_target, ES_NATIVE_CONDITION_LTZ(REG_AT));

            cg.SetJumpTarget(safe);
        }
        else
            cg.SUBU(rtarget, rlhs, rrhs);

        break;

    case INT32ARITHMETIC_MUL:
        if (overflow_target)
        {
            if (lhs.IsImmediate() || rhs.IsImmediate())
            {
                int imm;
                Register reg_other;

                if (lhs.IsImmediate())
                {
                    imm = lhs.codeword->immediate;
                    reg_other = rrhs;
                }
                else
                {
                    imm = rhs.codeword->immediate;
                    reg_other = rlhs;
                }

                if (imm == 0)
                    cg.Jump(overflow_target, ES_NATIVE_CONDITION_LTZ(reg_other));
                else if (imm < 0)
                    cg.Jump(overflow_target, ES_NATIVE_CONDITION_EQZ(reg_other));
            }
            else
            {
                ES_CodeGenerator::OutOfOrderBlock *slow_case = cg.StartOutOfOrderBlock();
                ES_CodeGenerator::JumpTarget *lhs_is_negative = cg.ForwardJump();

                ANNOTATE("EmitInt32Arithmetic, check for potential -0 in multiplication");

                cg.Jump(lhs_is_negative, ES_NATIVE_CONDITION_LTZ(rlhs));

                // lhs is 0, rhs < 0 yields -0
                cg.Jump(overflow_target, ES_NATIVE_CONDITION_LTZ(rrhs));
                cg.Jump(slow_case->GetContinuationTarget());

                cg.SetJumpTarget(lhs_is_negative);

                // lhs < 0, rhs is 0 yields -0
                cg.Jump(overflow_target, ES_NATIVE_CONDITION_EQZ(rrhs));
                cg.EndOutOfOrderBlock();

                // If one operand is positive non-zero, the result can never be -0.
                cg.Jump(slow_case->GetJumpTarget(), ES_NATIVE_CONDITION_LEZ(rlhs));

                cg.SetOutOfOrderContinuationPoint(slow_case);
            }
        }

        cg.MULT(rlhs, rrhs);

        if (overflow_target)
        {
            cg.MFHI(REG_T8);
            cg.MFLO(REG_T9);
            cg.SRA(REG_T9, REG_T9, 31);

            cg.JumpNE(overflow_target, REG_T8, REG_T9);
        }

        cg.MFLO(rtarget);
        break;

    case INT32ARITHMETIC_DIV:
    case INT32ARITHMETIC_REM:
        OP_ASSERT(overflow_target);

        cg.Jump(overflow_target, ES_NATIVE_CONDITION_EQZ(rrhs));

        cg.DIV(rlhs, rrhs);

        if (type == INT32ARITHMETIC_DIV)
        {
            cg.MFHI(REG_T9);
            cg.MFLO(rtarget);
            cg.Jump(overflow_target, ES_NATIVE_CONDITION_NEZ(REG_T9));
        }
        else
            cg.MFHI(rtarget);

        break;
    }
}

/* Emit int32 one's complement negation. */
void
ES_Native::EmitInt32Complement(const Operand &target, const Operand &source, BOOL as_condition)
{
    ANNOTATE("EmitInt32Complement");

    OP_ASSERT(target.native_register && source.native_register);

    if (as_condition)
        condition_register = INTEGER_REGISTER(target);

    cg.NOR(INTEGER_REGISTER(target), REG_Z, INTEGER_REGISTER(source));
}

/* Emit int32 two's complement negation. */
void
ES_Native::EmitInt32Negate(const Operand &target, const Operand &source, BOOL as_condition, ES_CodeGenerator::JumpTarget *overflow_target)
{
    ANNOTATE("EmitInt32Negate");

    OP_ASSERT(target.native_register && source.native_register);
    OP_ASSERT(overflow_target);

    if (as_condition)
        condition_register = INTEGER_REGISTER(target);

    // Signal overflow if input is 0 (can't represent -0)
    // or sign bit didn't change (can't represent -INT_MIN)

    if (target.native_register == source.native_register)
    {
        cg.SUBU(REG_T8, REG_Z, INTEGER_REGISTER(source));
        cg.XOR(REG_T9, REG_T8, INTEGER_REGISTER(source));
        cg.Move(INTEGER_REGISTER(target), REG_T8);
        cg.Jump(overflow_target, ES_NATIVE_CONDITION_LEZ(REG_T9), TRUE);
    }
    else
    {
        cg.SUBU(INTEGER_REGISTER(target), REG_Z, INTEGER_REGISTER(source));
        cg.XOR(REG_T8, INTEGER_REGISTER(target), INTEGER_REGISTER(source));
        cg.Jump(overflow_target, ES_NATIVE_CONDITION_LEZ(REG_T8));
    }
}

/* Emit increment-by-one or decrement-by-one instruction.  If
   'check_overflow' is TRUE, check for overflow and if so call slow case
   handler instead without modifying the target virtual register. */
void
ES_Native::EmitInt32IncOrDec(BOOL inc, const Operand &target, ES_CodeGenerator::JumpTarget *overflow_target)
{
    ANNOTATE("EmitInt32IncOrDec");

    OP_ASSERT(target.native_register);

    if (overflow_target)
    {
        if (inc)
        {
            cg.Load(REG_T8, GetConst(CONST_INT_MAX));
            cg.JumpEQ(overflow_target, INTEGER_REGISTER(target), REG_T8);
        }
        else
        {
            cg.Load(REG_T8, INT_MIN);
            cg.JumpEQ(overflow_target, INTEGER_REGISTER(target), REG_T8);
        }

        condition_register = INTEGER_REGISTER(target);
    }

    cg.Add(INTEGER_REGISTER(target), inc ? 1 : -1);
}

/* Emit comparison between int32:s in 'lhs' and 'rhs'.  If
   'value_target' is non-NULL, store either true (zero) or false (one)
   there depending on the outcome.  Otherwise, jump to the non-NULL jump
   target if the result is true or false, respectively, and otherwise do
   nothing (that is, let execution continue forward.) */
void
ES_Native::EmitInt32Relational(RelationalType relational_type, const Operand &lhs, const Operand &rhs, const Operand *value_target, ES_CodeGenerator::JumpTarget *true_target, ES_CodeGenerator::JumpTarget *false_target, ArithmeticBlock *arithmetic_block)
{
    ANNOTATE("EmitInt32Relational");

    OP_ASSERT(value_target && (!true_target || !false_target) || (!true_target != !false_target));

    Register rlhs, rrhs;

    if (arithmetic_block && (true_target || false_target))
        FlushToVirtualRegisters(arithmetic_block);

    if (lhs.native_register)
        rlhs = INTEGER_REGISTER(lhs);
    else if (!lhs.codeword)
    {
        cg.LW(REG_T8, IVALUE_OFFSET(lhs.virtual_register), BASE_REGISTER(lhs.virtual_register));
        rlhs = REG_T8;
    }
    else
    {
        cg.Load(REG_T8, lhs.codeword->immediate);
        rlhs = REG_T8;
    }

    if (rhs.native_register)
        rrhs = INTEGER_REGISTER(rhs);
    else if (!rhs.codeword)
    {
        cg.LW(REG_T9, IVALUE_OFFSET(rhs.virtual_register), BASE_REGISTER(rhs.virtual_register));
        rrhs = REG_T9;
    }
    else
    {
        cg.Load(REG_T9, rhs.codeword->immediate);
        rrhs = REG_T9;
    }

    if (value_target)
    {
        switch (relational_type)
        {
        case RELATIONAL_LT:
        case RELATIONAL_GTE:
            cg.SLT(REG_AT, rlhs, rrhs);
            break;

        case RELATIONAL_GT:
        case RELATIONAL_LTE:
            cg.SLT(REG_AT, rrhs, rlhs);
            break;

        case RELATIONAL_EQ:
        case RELATIONAL_NEQ:
            cg.XOR(REG_AT, rlhs, rrhs);
            cg.SLTIU(REG_AT, REG_AT, 1);
            break;
        }

        switch (relational_type)
        {
        case RELATIONAL_GTE:
        case RELATIONAL_LTE:
        case RELATIONAL_NEQ:
            cg.SLTIU(REG_AT, REG_AT, 1);
        }

        if (value_target->virtual_register->stack_frame_offset == INT_MAX)
            cg.Load(REG_T9, GetConst(CONST_ESTYPE_BOOLEAN));

        cg.SW(REG_AT, IVALUE_OFFSET(value_target->virtual_register), BASE_REGISTER(value_target->virtual_register));

        if (value_target->virtual_register->stack_frame_offset == INT_MAX)
            cg.SW(REG_T9, TYPE_OFFSET(value_target->virtual_register), BASE_REGISTER(value_target->virtual_register));

        if (true_target)
            cg.Jump(true_target, ES_NATIVE_CONDITION_NEZ(REG_AT), TRUE);
        else if (false_target)
            cg.Jump(false_target, ES_NATIVE_CONDITION_EQZ(REG_AT), TRUE);

        return;
    }

    if (true_target)
    {
        switch (relational_type)
        {
        case RELATIONAL_EQ:  cg.JumpEQ(true_target, rlhs, rrhs); break;
        case RELATIONAL_NEQ: cg.JumpNE(true_target, rlhs, rrhs); break;
        case RELATIONAL_LT:  cg.JumpLT(true_target, rlhs, rrhs); break;
        case RELATIONAL_LTE: cg.JumpLE(true_target, rlhs, rrhs); break;
        case RELATIONAL_GT:  cg.JumpGT(true_target, rlhs, rrhs); break;
        case RELATIONAL_GTE: cg.JumpGE(true_target, rlhs, rrhs); break;
        }
    }
    else
    {
        switch (relational_type)
        {
        case RELATIONAL_EQ:  cg.JumpNE(false_target, rlhs, rrhs); break;
        case RELATIONAL_NEQ: cg.JumpEQ(false_target, rlhs, rrhs); break;
        case RELATIONAL_LT:  cg.JumpGE(false_target, rlhs, rrhs); break;
        case RELATIONAL_LTE: cg.JumpGT(false_target, rlhs, rrhs); break;
        case RELATIONAL_GT:  cg.JumpLE(false_target, rlhs, rrhs); break;
        case RELATIONAL_GTE: cg.JumpLT(false_target, rlhs, rrhs); break;
        }
    }
}

/* Emit double arithmetic instruction. */
void
ES_Native::EmitDoubleArithmetic(DoubleArithmeticType type, const Operand &target, const Operand &lhs, const Operand &rhs, BOOL as_condition)
{
    ANNOTATE("EmitDoubleArithmetic");

    OP_ASSERT(target.native_register);
    OP_ASSERT(lhs.native_register);
    OP_ASSERT(rhs.native_register);

    switch (type)
    {
    case DOUBLEARITHMETIC_ADD:
        cg.ADD(D, DOUBLE_REGISTER(target), DOUBLE_REGISTER(lhs), DOUBLE_REGISTER(rhs));
        break;

    case DOUBLEARITHMETIC_SUB:
        cg.SUB(D, DOUBLE_REGISTER(target), DOUBLE_REGISTER(lhs), DOUBLE_REGISTER(rhs));
        break;

    case DOUBLEARITHMETIC_MUL:
        cg.MUL(D, DOUBLE_REGISTER(target), DOUBLE_REGISTER(lhs), DOUBLE_REGISTER(rhs));
        break;

    case DOUBLEARITHMETIC_DIV:
        cg.DIV(D, DOUBLE_REGISTER(target), DOUBLE_REGISTER(lhs), DOUBLE_REGISTER(rhs));
    }

    if (as_condition)
    {
        condition_register = REG_Z;

        // Load 0 into FPU scratch register
        cg.MTC1(REG_Z, REG_F28);
        cg.CVT(D, W, REG_F30, REG_F28);

        cg.C(EQ, D, REG_F30, DOUBLE_REGISTER(target));
    }
}

/* Emit double arithmetic instruction. */
void
ES_Native::EmitDoubleRelational(RelationalType type, const Operand &lhs, const Operand &rhs, const Operand *value_target, ES_CodeGenerator::JumpTarget *true_target, ES_CodeGenerator::JumpTarget *false_target, ArithmeticBlock *arithmetic_block)
{
    ANNOTATE("EmitDoubleRelational");

    OP_ASSERT(value_target && (!true_target || !false_target) || (!true_target != !false_target));

    OP_ASSERT(lhs.native_register);
    OP_ASSERT(rhs.native_register);

    ES_CodeGenerator::FPUCondition cond;
    BOOL true_cond;

    if (arithmetic_block && (true_target || false_target))
        FlushToVirtualRegisters(arithmetic_block);

    switch (type)
    {
    case RELATIONAL_EQ:  cond = ES_CodeGenerator::EQ;  true_cond = TRUE; break;
    case RELATIONAL_NEQ: cond = ES_CodeGenerator::EQ;  true_cond = FALSE; break;
    case RELATIONAL_LT:  cond = ES_CodeGenerator::OLT; true_cond = TRUE; break;
    case RELATIONAL_LTE: cond = ES_CodeGenerator::OLE; true_cond = TRUE; break;
    case RELATIONAL_GT:  cond = ES_CodeGenerator::ULE; true_cond = FALSE; break;
    case RELATIONAL_GTE: cond = ES_CodeGenerator::ULT; true_cond = FALSE; break;
    default: OP_ASSERT(FALSE); return;
    }

    cg.C(cond, D, DOUBLE_REGISTER(lhs.native_register), DOUBLE_REGISTER(rhs.native_register));

    if (value_target || (type != RELATIONAL_EQ && type != RELATIONAL_NEQ))
    {
        cg.Load(REG_T8, 1);

        if (true_cond)
            cg.MOVF(REG_T8, REG_Z);
        else
            cg.MOVT(REG_T8, REG_Z);
    }

    if (value_target)
    {
        if (value_target->virtual_register->stack_frame_offset == INT_MAX)
            cg.Load(REG_T9, GetConst(CONST_ESTYPE_BOOLEAN));

        cg.SW(REG_T8, IVALUE_OFFSET(value_target->virtual_register), BASE_REGISTER(value_target->virtual_register));

        if (value_target->virtual_register->stack_frame_offset == INT_MAX)
            cg.SW(REG_T9, TYPE_OFFSET(value_target->virtual_register), REG_FP);
    }

    if (type != RELATIONAL_EQ && type != RELATIONAL_NEQ)
    {
        if (true_target)
            cg.Jump(true_target, ES_NATIVE_CONDITION_NEZ(REG_T8));
        else if (false_target)
            cg.Jump(false_target, ES_NATIVE_CONDITION_EQZ(REG_T8));
    }
    else
    {
        if (true_target)
        {
            if (true_cond)
                cg.Jump(true_target, ES_NATIVE_CONDITION_C1T(0));
            else
                cg.Jump(true_target, ES_NATIVE_CONDITION_C1F(0));
        }
        else if (false_target)
        {
            if (!true_cond)
                cg.Jump(false_target, ES_NATIVE_CONDITION_C1T(0));
            else
                cg.Jump(false_target, ES_NATIVE_CONDITION_C1F(0));
        }
    }
}

/* Emit double negation, with type check on 'source', and upgrade from ESTYPE_INT32 if necessary. */
void
ES_Native::EmitDoubleNegate(const Operand &target, const Operand &source, BOOL as_condition)
{
    ANNOTATE("EmitDoubleNegate");

    OP_ASSERT(target.native_register);
    OP_ASSERT(source.native_register);

    cg.NEG(D, DOUBLE_REGISTER(target), DOUBLE_REGISTER(source));

    if (as_condition)
    {
        condition_register = REG_Z;

        // Load 0 into FPU scratch register
        cg.MTC1(REG_Z, REG_F28);
        cg.CVT(D, W, REG_F30, REG_F28);

        cg.C(EQ, D, REG_F30, DOUBLE_REGISTER(target));
    }
}

void
ES_Native::EmitDoubleIncOrDec(BOOL inc, const Operand &target)
{
    ANNOTATE("EmitDoubleIncOrDec");

    OP_ASSERT(target.native_register);

    cg.Load(REG_T8, inc ? 1 : -1);
    cg.MTC1(REG_T8, REG_F28);
    cg.CVT(D, W, REG_F30, REG_F28);

    cg.ADD(D, DOUBLE_REGISTER(target), DOUBLE_REGISTER(target), REG_F30);
}

/* Emit equality or non-equality comparison to null or undefined.  (The
   result is the same whether the constant operand is null or undefined,
   since null and undefined compares equal.) */
void
ES_Native::EmitNullOrUndefinedComparison(BOOL eq, VirtualRegister *vr, const Operand *value_target, ES_CodeGenerator::JumpTarget *true_target, ES_CodeGenerator::JumpTarget *false_target)
{
    ANNOTATE("EmitNullOrUndefinedComparison");

    OP_ASSERT(value_target && (!true_target || !false_target) || (!true_target != !false_target));

    cg.LW(REG_T8, TYPE_OFFSET(vr), BASE_REGISTER(vr));

    cg.Load(REG_A0, GetConst(CONST_ESTYPE_UNDEFINED));
    cg.Add(REG_A1, REG_A0, ESTYPE_BOOLEAN - ESTYPE_UNDEFINED);

    cg.SLT(REG_A2, REG_T8, REG_A0);
    cg.SLT(REG_A3, REG_T8, REG_A1);

    cg.XOR(REG_A0, REG_A2, REG_A3);

    if (!eq)
        cg.SLTIU(REG_A0, REG_A0, 1);

    if (value_target)
    {
        OP_ASSERT(value_target->virtual_register);

        cg.SW(REG_A0, IVALUE_OFFSET(value_target->virtual_register), BASE_REGISTER(value_target->virtual_register));

        if (value_target->virtual_register->stack_frame_offset == INT_MAX)
            cg.SW(REG_A1, TYPE_OFFSET(value_target->virtual_register), REG_FP);
    }

    if (true_target)
        cg.Jump(true_target, ES_NATIVE_CONDITION_NEZ(REG_A0));
    else if (false_target)
        cg.Jump(false_target, ES_NATIVE_CONDITION_EQZ(REG_A0));
}

/* Emit strict equality or non-equality comparison to null or undefined. */
void
ES_Native::EmitStrictNullOrUndefinedComparison(BOOL eq, VirtualRegister *vr, ES_ValueType type, ES_CodeGenerator::JumpTarget *true_target, ES_CodeGenerator::JumpTarget *false_target)
{
    ANNOTATE("EmitStrictNullOrUndefinedComparison");

    OP_ASSERT(!true_target != !false_target);

    cg.LW(REG_T8, TYPE_OFFSET(vr), BASE_REGISTER(vr));
    cg.Load(REG_T9, GetTypeConst(*this, type));

    if (true_target)
        cg.Jump(true_target, eq ? ES_NATIVE_CONDITION_EQ(REG_T8, REG_T9) : ES_NATIVE_CONDITION_NE(REG_T8, REG_T9));
    else
        cg.Jump(false_target, eq ? ES_NATIVE_CONDITION_NE(REG_T8, REG_T9) : ES_NATIVE_CONDITION_EQ(REG_T8, REG_T9));
}

void
ES_Native::EmitComparison(BOOL eq, BOOL strict, VirtualRegister *lhs, VirtualRegister *rhs, unsigned handled_types, const Operand *value_target, ES_CodeGenerator::JumpTarget *true_target, ES_CodeGenerator::JumpTarget *false_target)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitComparison");

    OP_ASSERT(value_target && (!true_target || !false_target) || (!true_target != !false_target));

    ES_CodeGenerator::JumpTarget *jt_true = true_target;
    ES_CodeGenerator::JumpTarget *jt_false = false_target;

    if (!eq)
    {
        ES_CodeGenerator::JumpTarget *temporary = false_target;
        false_target = true_target;
        true_target = temporary;
    }

    const Register reg_lhs_value = REG_A0;
    const Register reg_rhs_value = REG_A1;
    const Register reg_lhs_type = REG_S0;
    const Register reg_rhs_type = REG_S1;

    LoadValue(cg, lhs, reg_lhs_value, reg_lhs_type);
    LoadValue(cg, rhs, reg_rhs_value, reg_rhs_type);

    ES_CodeGenerator::JumpTarget *jt_value = cg.ForwardJump();
    ES_CodeGenerator::JumpTarget *jt_continue = cg.ForwardJump();

    if (handled_types)
    {
        ES_CodeGenerator::JumpTarget *jt_update = cg.ForwardJump();
        ES_CodeGenerator::JumpTarget *jt_slow_case = cg.ForwardJump();

        cg.Jump(jt_slow_case, ES_NATIVE_CONDITION_NE(reg_lhs_type, reg_rhs_type));

        ES_ValueType first;
        int diff = 2;

        // TODO: Handle ESTYPE_DOUBLE

        if (handled_types & ESTYPE_BITS_INT32)
            first = ESTYPE_INT32;
        else if (handled_types & (ESTYPE_BITS_UNDEFINED | ESTYPE_BITS_NULL))
            first = ESTYPE_NULL;
        else if (handled_types & ESTYPE_BITS_BOOLEAN)
            first = ESTYPE_BOOLEAN;
        else if (handled_types & ESTYPE_BITS_STRING)
            first = ESTYPE_STRING;
        else if (handled_types & ESTYPE_BITS_OBJECT)
            first = ESTYPE_OBJECT;
        else
            goto update_comparison;

        cg.Load(REG_T9, GetTypeConst(*this, first));
        cg.JumpLT(jt_update, reg_lhs_type, REG_T9);

        cg.Sub(reg_lhs_type, REG_T9);

        if (handled_types & ESTYPE_BITS_INT32)
        {
            ANNOTATE("EmitComparison, test ESTYPE_INT32");

            ES_CodeGenerator::JumpTarget *not_int = cg.ForwardJump();
            cg.Jump(not_int, ES_NATIVE_CONDITION_GTZ(reg_lhs_type));

            if (value_target)
            {
                cg.XOR(REG_T8, reg_lhs_value, reg_rhs_value);
                cg.SLTIU(REG_T8, REG_T8, 1);
                cg.Jump(jt_value, ES_NATIVE_UNCONDITIONAL, TRUE);
            }
            else
            {
                if (true_target)
                    cg.Jump(true_target, ES_NATIVE_CONDITION_EQ(reg_lhs_value, reg_rhs_value));
                else
                    cg.Jump(false_target, ES_NATIVE_CONDITION_NE(reg_lhs_value, reg_rhs_value));

                cg.Jump(jt_continue, ES_NATIVE_UNCONDITIONAL);
            }

            cg.SetJumpTarget(not_int);
        }

        if (handled_types & (ESTYPE_BITS_UNDEFINED | ESTYPE_BITS_NULL))
        {
            ANNOTATE("EmitComparison, test ESTYPE_UNDEFINED and ESTYPE_NULL");

            ES_CodeGenerator::JumpTarget *not_null = cg.ForwardJump();

            if (first != ESTYPE_NULL)
                cg.Sub(reg_lhs_type, diff);

            diff = 1;

            cg.Jump(not_null, ES_NATIVE_CONDITION_GTZ(reg_lhs_type));

            if (value_target)
            {
                cg.ORI(REG_T8, REG_Z, 1);
                cg.Jump(jt_value, ES_NATIVE_UNCONDITIONAL, TRUE);
            }
            else
            {
                if (true_target)
                    cg.Jump(true_target, ES_NATIVE_UNCONDITIONAL);
                else
                    cg.Jump(jt_continue, ES_NATIVE_UNCONDITIONAL);
            }

            cg.SetJumpTarget(not_null);
        }
        else
            diff++;

        if (handled_types & ESTYPE_BITS_BOOLEAN)
        {
            ANNOTATE("EmitComparison, test ESTYPE_BOOLEAN");

            ES_CodeGenerator::JumpTarget *not_bool = cg.ForwardJump();

            if (first != ESTYPE_BOOLEAN)
                cg.Sub(reg_lhs_type, diff);

            diff = 2;

            cg.Jump(not_bool, ES_NATIVE_CONDITION_GTZ(reg_lhs_type));

            if (value_target)
            {
                cg.XOR(REG_T8, reg_lhs_value, reg_rhs_value);
                cg.SLTIU(REG_T8, REG_T8, 1);
                cg.Jump(jt_value, ES_NATIVE_UNCONDITIONAL, TRUE);
            }
            else
            {
                if (true_target)
                    cg.Jump(true_target, ES_NATIVE_CONDITION_EQ(reg_lhs_value, reg_rhs_value));
                else
                    cg.Jump(false_target, ES_NATIVE_CONDITION_NE(reg_lhs_value, reg_rhs_value));

                cg.Jump(jt_continue, ES_NATIVE_UNCONDITIONAL);
            }

            cg.SetJumpTarget(not_bool);
        }
        else
            diff += 2;

        if (handled_types & ESTYPE_BITS_STRING)
        {
            ANNOTATE("EmitComparison, test ESTYPE_STRING");

            ES_CodeGenerator::JumpTarget *not_string = cg.ForwardJump();

            if (first != ESTYPE_STRING)
            {
                cg.Sub(reg_lhs_type, diff);
                cg.Jump(jt_update, ES_NATIVE_CONDITION_LTZ(reg_lhs_type)); // Test for ESTYPE_BOXED
            }

            diff = 1;

            cg.Jump(not_string, ES_NATIVE_CONDITION_GTZ(reg_lhs_type));

            ES_CodeGenerator::JumpTarget *ptr_ne = cg.ForwardJump();
            cg.Jump(ptr_ne, ES_NATIVE_CONDITION_NE(reg_lhs_value, reg_rhs_value));

            if (value_target)
            {
                cg.Load(REG_T8, 1);
                cg.Jump(jt_value, ES_NATIVE_UNCONDITIONAL, TRUE);
            }
            else
            {
                if (true_target)
                    cg.Jump(true_target, ES_NATIVE_UNCONDITIONAL);
                else
                    cg.Jump(jt_continue, ES_NATIVE_UNCONDITIONAL);
            }

            cg.SetJumpTarget(ptr_ne);

            ES_CodeGenerator::JumpTarget *length_eq = cg.ForwardJump();

            cg.LW(REG_T8, ES_OFFSETOF(JString, length), reg_lhs_value);
            cg.LW(REG_T9, ES_OFFSETOF(JString, length), reg_rhs_value);
            cg.Jump(length_eq, ES_NATIVE_CONDITION_EQ(REG_T8, REG_T9));

            if (value_target)
            {
                cg.Move(REG_T8, REG_Z);
                cg.Jump(jt_value, ES_NATIVE_UNCONDITIONAL, TRUE);
            }
            else
            {
                if (false_target)
                    cg.Jump(false_target, ES_NATIVE_UNCONDITIONAL);
                else
                    cg.Jump(jt_continue, ES_NATIVE_UNCONDITIONAL);
            }

            cg.SetJumpTarget(length_eq);

            cg.Load(REG_T9, GetConst(CONST_EQUALS));
            cg.JALR(REG_T9);
            cg.NOP();

            if (value_target)
            {
                cg.Move(REG_T8, REG_V0);
                cg.Jump(jt_value, ES_NATIVE_UNCONDITIONAL, TRUE);
            }
            else
            {
                if (true_target)
                    cg.Jump(true_target, ES_NATIVE_CONDITION_NEZ(REG_V0));
                else
                    cg.Jump(false_target, ES_NATIVE_CONDITION_EQZ(REG_V0));

                cg.Jump(jt_continue, ES_NATIVE_UNCONDITIONAL);
            }

            cg.SetJumpTarget(not_string);
        }
        else
            diff++;

        if (handled_types & ESTYPE_BITS_OBJECT)
        {
            ANNOTATE("EmitComparison, test ESTYPE_OBJECT");

            if (first != ESTYPE_OBJECT)
                cg.Sub(reg_lhs_type, diff);

            cg.Jump(jt_update, ES_NATIVE_CONDITION_NEZ(reg_lhs_type));

            ES_CodeGenerator::JumpTarget *ptr_ne = cg.ForwardJump();
            cg.Jump(ptr_ne, ES_NATIVE_CONDITION_NE(reg_lhs_value, reg_rhs_value));

            if (value_target)
            {
                cg.Load(REG_T8, 1);
                cg.Jump(jt_value, ES_NATIVE_UNCONDITIONAL, TRUE);
            }
            else
            {
                if (true_target)
                    cg.Jump(true_target, ES_NATIVE_UNCONDITIONAL);
                else
                    cg.Jump(jt_continue, ES_NATIVE_UNCONDITIONAL);
            }

            cg.SetJumpTarget(ptr_ne);

            ES_CodeGenerator::OutOfOrderBlock *compare_shifty_objects = cg.StartOutOfOrderBlock();
            ANNOTATE("EmitComparison, compare shifty objects");

            cg.Load(REG_T9, GetConst(CONST_ESN_COMPARESHIFTYOBJECTS));
            // Compensate for this call having Context as first argument
            cg.Move(REG_A2, REG_A1);
            cg.Move(REG_A1, REG_A0);
            cg.JALR(REG_T9);
            cg.Move(REG_A0, REG_S7);

            if (value_target)
            {
                cg.Move(REG_T8, REG_V0);
                cg.Jump(jt_value, ES_NATIVE_UNCONDITIONAL, TRUE);
            }
            else
            {
                if (true_target)
                    cg.Jump(true_target, ES_NATIVE_CONDITION_NEZ(REG_V0));
                else
                    cg.Jump(false_target, ES_NATIVE_CONDITION_EQZ(REG_V0));

                cg.Jump(jt_continue, ES_NATIVE_UNCONDITIONAL);
            }

            cg.EndOutOfOrderBlock(FALSE);

            cg.LW(REG_T8, ES_OFFSETOF(ES_Object, object_bits), reg_lhs_value);
            cg.And(REG_T8, ES_Object::MASK_MULTIPLE_IDENTITIES);
            cg.Jump(compare_shifty_objects->GetJumpTarget(), ES_NATIVE_CONDITION_NEZ(REG_T8));

            cg.LW(REG_T9, ES_OFFSETOF(ES_Object, object_bits), reg_rhs_value);
            cg.And(REG_T9, ES_Object::MASK_MULTIPLE_IDENTITIES);
            cg.Jump(compare_shifty_objects->GetJumpTarget(), ES_NATIVE_CONDITION_NEZ(REG_T9));

            if (value_target)
            {
                cg.Move(REG_T8, REG_Z);
                cg.Jump(jt_value, ES_NATIVE_UNCONDITIONAL, TRUE);
            }
            else
            {
                if (false_target)
                    cg.Jump(false_target, ES_NATIVE_UNCONDITIONAL);
                else
                    cg.Jump(jt_continue, ES_NATIVE_UNCONDITIONAL);
            }
        }

update_comparison:
        ANNOTATE("EmitComparison, update comparison");

        cg.SetJumpTarget(jt_update);

        cg.Load(REG_A1, cw_index);
        CallDispatcherFunc(cg, GetConst(CONST_EXC_UPDATECOMPARISON));

        cg.SetJumpTarget(jt_slow_case);
    }

    ANNOTATE("EmitComparison, slow case");

    cg.Add(REG_A1, REG_FP, REGISTER_OFFSET(lhs));
    cg.Add(REG_A2, REG_FP, REGISTER_OFFSET(rhs));

    if (strict)
        CallContextFunc(cg, GetConst(CONST_EXC_EQSTRICT));
    else
    {
        cg.Load(REG_A3, cw_index);
        CallContextFunc(cg, GetConst(CONST_EXC_EQFROMMACHINECODE));
    }

    if (value_target)
    {
        cg.Move(REG_T8, REG_V0);

        cg.SetJumpTarget(jt_value);

        cg.Load(REG_T9, GetConst(CONST_ESTYPE_BOOLEAN));

        if (!eq)
            cg.SLTIU(REG_T8, REG_T8, 1);

        cg.SW(REG_T8, IVALUE_OFFSET(value_target->virtual_register), BASE_REGISTER(value_target->virtual_register));
        cg.SW(REG_T9, TYPE_OFFSET(value_target->virtual_register), BASE_REGISTER(value_target->virtual_register));

        if (jt_true)
            cg.Jump(jt_true, ES_NATIVE_CONDITION_NEZ(REG_T8));
        else if (jt_false)
            cg.Jump(jt_false, ES_NATIVE_CONDITION_EQZ(REG_T8));
    }
    else
    {
        if (true_target)
            cg.Jump(true_target, ES_NATIVE_CONDITION_NEZ(REG_V0));
        else
            cg.Jump(false_target, ES_NATIVE_CONDITION_EQZ(REG_V0));
    }

    cg.SetJumpTarget(jt_continue);
}

void
ES_Native::EmitInstanceOf(VirtualRegister *object, VirtualRegister *constructor, const Operand *value_target, ES_CodeGenerator::JumpTarget *true_target, ES_CodeGenerator::JumpTarget *false_target)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitInstanceOf");

    OP_ASSERT(value_target && (!true_target || !false_target) || (!true_target != !false_target));

    ES_CodeGenerator::JumpTarget *use_true_target = true_target;
    ES_CodeGenerator::JumpTarget *use_false_target = false_target;

    if (value_target || !true_target)
        use_true_target = cg.ForwardJump();
    if (value_target || !false_target)
        use_false_target = cg.ForwardJump();

    if (!current_slow_case)
        EmitSlowCaseCall();

    ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();

    const Register reg_constructor_type = REG_S0;
    const Register reg_constructor = REG_A2;

    /* right-hand side not an object => exception (because it must be a
    function, and ToObject() never produces a function) */
    if (!IsType(constructor, ESTYPE_OBJECT))
    {
        LoadValue(cg, constructor, reg_constructor, reg_constructor_type);

        cg.Load(REG_T8, GetConst(CONST_ESTYPE_OBJECT));
        cg.Jump(slow_case, ES_NATIVE_CONDITION_NE(reg_constructor_type, REG_T8));
    }
    else
        cg.LW(reg_constructor, IVALUE_OFFSET(constructor), BASE_REGISTER(constructor));

    ES_CodeGenerator::JumpTarget *bits_ok = cg.ForwardJump();
    const Register reg_constructor_bits = REG_S2;

    cg.LW(reg_constructor_bits, ES_OFFSETOF(ES_Object, hdr.header), reg_constructor);
    cg.And(reg_constructor_bits, ES_Header::MASK_GCTAG);

    cg.Load(REG_T8, GCTAG_ES_Object_Function);
    cg.Jump(bits_ok, ES_NATIVE_CONDITION_EQ(reg_constructor_bits, REG_T8));

    cg.Load(REG_T8, GCTAG_ES_Object_RegExp_CTOR);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_NE(reg_constructor_bits, REG_T8));

    cg.SetJumpTarget(bits_ok);

    const Register reg_object_type = REG_S3;
    const Register reg_object = REG_S4;

    if (!IsType(object, ESTYPE_OBJECT))
    {
        LoadValue(cg, object, reg_object, reg_object_type);

        cg.Load(REG_T8, GetConst(CONST_ESTYPE_OBJECT));
        cg.Jump(use_false_target, ES_NATIVE_CONDITION_NE(reg_object_type, REG_T8));
    }
    else
        cg.LW(reg_object, IVALUE_OFFSET(object), BASE_REGISTER(object));

    ES_CodeGenerator::JumpTarget *simple_case = cg.ForwardJump();

    const Register reg_constructor_class = REG_T5;
    const Register reg_constructor_class_id = REG_V0;

    cg.LW(reg_constructor_class, ES_OFFSETOF(ES_Object, klass), reg_constructor);
    cg.LW(reg_constructor_class_id, ES_OFFSETOF(ES_Class, class_id), reg_constructor_class);

    cg.Load(REG_T8, code->global_object->GetNativeFunctionWithPrototypeClass()->GetId(context));
    cg.Jump(simple_case, ES_NATIVE_CONDITION_EQ(reg_constructor_class_id, REG_T8));

    cg.Load(REG_T9, code->global_object->GetBuiltInConstructorClass()->GetId(context));
    cg.Jump(simple_case, ES_NATIVE_CONDITION_EQ(reg_constructor_class_id, REG_T9));

    cg.Move(REG_A1, reg_object);
    CallContextFunc(cg, GetConst(CONST_ESN_INSTANCEOF));

    cg.Jump(use_true_target, ES_NATIVE_CONDITION_NEZ(REG_V0));
    cg.Jump(use_false_target, ES_NATIVE_CONDITION_EQZ(REG_V0));

    cg.SetJumpTarget(simple_case);

    const Register reg_properties = REG_T0;
    const Register reg_prototype_type = REG_T1;
    const Register reg_prototype = REG_T2;

    cg.LW(reg_properties, ES_OFFSETOF(ES_Object, properties), reg_constructor);

    LoadValue(cg, reg_properties, ES_Function::GetPropertyOffset(ES_Function::PROTOTYPE), reg_prototype, reg_prototype_type);

    cg.Load(REG_T8, GetConst(CONST_ESTYPE_OBJECT));
    cg.Jump(slow_case, ES_NATIVE_CONDITION_NE(reg_prototype_type, REG_T8));

    ES_CodeGenerator::JumpTarget *loop = cg.Here();

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, klass), reg_object);
    cg.LW(REG_T8, ES_OFFSETOF(ES_Class, static_data), REG_T8);
    cg.LW(reg_object, ES_OFFSETOF(ES_Class_Data, prototype), REG_T8);

    cg.Jump(use_true_target, ES_NATIVE_CONDITION_EQ(reg_object, reg_prototype));
    cg.Jump(loop, ES_NATIVE_CONDITION_NEZ(reg_object));

    if (use_false_target != false_target)
    {
        cg.SetJumpTarget(use_false_target);

        if (value_target)
        {
            cg.Load(REG_T8, GetConst(CONST_ESTYPE_BOOLEAN));
            cg.SW(REG_Z, IVALUE_OFFSET(value_target->virtual_register), BASE_REGISTER(value_target->virtual_register));
            cg.SW(REG_T8, TYPE_OFFSET(value_target->virtual_register), BASE_REGISTER(value_target->virtual_register));
        }
    }

    ES_CodeGenerator::JumpTarget *finished = NULL;

    if (false_target)
        cg.Jump(false_target);
    else
        cg.Jump(finished = cg.ForwardJump());

    if (use_true_target != true_target)
    {
        cg.SetJumpTarget(use_true_target);

        if (value_target)
        {
            cg.Load(REG_T9, 1);
            cg.Load(REG_T8, GetConst(CONST_ESTYPE_BOOLEAN));
            cg.SW(REG_T9, IVALUE_OFFSET(value_target->virtual_register), BASE_REGISTER(value_target->virtual_register));
            cg.SW(REG_T8, TYPE_OFFSET(value_target->virtual_register), BASE_REGISTER(value_target->virtual_register));
        }

        if (true_target)
            cg.Jump(true_target);
    }

    if (finished)
        cg.SetJumpTarget(finished);
}

/* Emit conditional jump according to the current value of the implicit boolean register. */
void
ES_Native::EmitConditionalJump(ES_CodeGenerator::JumpTarget *true_target, ES_CodeGenerator::JumpTarget *false_target, BOOL check_implicit_boolean, ArithmeticBlock *arithmetic_block)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitConditionalJump");

    OP_ASSERT(!true_target != !false_target);

    if (arithmetic_block)
        FlushToVirtualRegisters(arithmetic_block);

    if (check_implicit_boolean)
    {
        cg.LW(REG_T8, ES_OFFSETOF(ES_Execution_Context, implicit_bool), REG_S7);
        condition_register = REG_T8;
    }

    if (condition_register == REG_Z)
    {
        if (true_target)
            cg.Jump(true_target, ES_NATIVE_CONDITION_C1F(0));
        else
            cg.Jump(false_target, ES_NATIVE_CONDITION_C1T(0));
    }
    else
    {
        if (true_target)
            cg.Jump(true_target, ES_NATIVE_CONDITION_NEZ(condition_register));
        else
            cg.Jump(false_target, ES_NATIVE_CONDITION_EQZ(condition_register));
    }
}

/* Emit conditional jump whose condition is ToBoolean() applied to the value pointed to by the current value in the 'value' register. */
void
ES_Native::EmitConditionalJumpOnValue(VirtualRegister *value, ES_CodeGenerator::JumpTarget *true_target, ES_CodeGenerator::JumpTarget *false_target)
{
    ANNOTATE("EmitConditionalJumpOnValue");

    OP_ASSERT(!true_target != !false_target);
    OP_ASSERT(value->stack_frame_offset == INT_MAX);

    ES_CodeGenerator::OutOfOrderBlock *slow_case = cg.StartOutOfOrderBlock();

    ANNOTATE("EmitConditionalJumpOnValue, slow case");

    cg.Load(REG_T9, GetConst(CONST_ES_VALUE_INTERNAL_RETURNASBOOLEAN));
    cg.JALR(REG_T9);
    cg.ADDIU(REG_A0, REG_FP, REGISTER_OFFSET(value));

    cg.EndOutOfOrderBlock();

    ES_CodeGenerator::JumpTarget *safe = cg.ForwardJump();

    cg.LW(REG_T8, TYPE_OFFSET(value), BASE_REGISTER(value));

    cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));
    cg.Jump(safe, ES_NATIVE_CONDITION_EQ(REG_T8, REG_T9));

    cg.Add(REG_T9, ESTYPE_BOOLEAN - ESTYPE_INT32);
    cg.Jump(slow_case->GetJumpTarget(), ES_NATIVE_CONDITION_NE(REG_T8, REG_T9));

    cg.SetJumpTarget(safe);
    cg.LW(REG_V0, IVALUE_OFFSET(value), BASE_REGISTER(value));

    cg.SetOutOfOrderContinuationPoint(slow_case);

    if (true_target)
        cg.Jump(true_target, ES_NATIVE_CONDITION_NEZ(REG_V0));
    else
        cg.Jump(false_target, ES_NATIVE_CONDITION_EQZ(REG_V0));
}

static ES_Boxed *
AllocateSimple(ES_Context *context, ES_Heap *heap, unsigned nbytes)
{
    return heap->AllocateSimple(context, nbytes);
}

static void
AllocateObject(ES_Native &self, ES_Class *actual_klass, ES_Class *final_klass, unsigned property_count, unsigned *nindexed, ES_Compact_Indexed_Properties *representation, ES_CodeGenerator::JumpTarget *slow_case)
{
    DECLARE_NOTHING();

    ES_CodeGenerator &cg = self.cg;
    ES_Native::ObjectAllocationData data;

    ANNOTATE("AllocateObject");

    OP_ASSERT(!actual_klass->HasHashTableProperties());

    self.GetObjectAllocationData(data, actual_klass, final_klass, property_count, nindexed, representation);
    OP_ASSERT(data.named_bytes >= sizeof(ES_Box) + (final_klass ? final_klass : actual_klass)->SizeOf(property_count));

    const Register reg_object = ALLOCATED_OBJECT_REGISTER;

    unsigned total_bytes = data.main_bytes + data.named_bytes + data.indexed_bytes;

    ES_CodeGenerator::OutOfOrderBlock *simple_allocation = cg.StartOutOfOrderBlock();

    cg.Load(REG_A2, total_bytes);
    CallContextFunc(cg, self.GetConst(CONST_ALLOCATESIMPLE));

    cg.JumpEQ(slow_case, REG_V0, REG_Z);
    cg.Move(reg_object, REG_V0);

    cg.EndOutOfOrderBlock();

    /* Carve a chunk off the heap's current free block, check that it was big
    enough and if so, update the heap's current_top pointer. */

    const Register reg_heap = REG_A1;
    const Register reg_current_top = REG_T2;
    const Register reg_current_limit = REG_T3;

    cg.LW(reg_heap, ES_OFFSETOF(ES_Execution_Context, heap), REG_S7);
    cg.LW(reg_object, ES_OFFSETOF(ES_Heap, current_top), reg_heap);
    cg.LW(reg_current_limit, ES_OFFSETOF(ES_Heap, current_limit), reg_heap);

    cg.Add(reg_current_top, reg_object, total_bytes);
    cg.JumpGE(simple_allocation->GetJumpTarget(), reg_current_top, reg_current_limit);

    cg.SW(reg_current_top, ES_OFFSETOF(ES_Heap, current_top), reg_heap);

    const Register reg_bytes_live = REG_T4;

    cg.LW(reg_bytes_live, ES_OFFSETOF(ES_Heap, bytes_live), reg_heap);
    cg.Add(reg_bytes_live, total_bytes);
    cg.SW(reg_bytes_live, ES_OFFSETOF(ES_Heap, bytes_live), reg_heap);

    cg.SetOutOfOrderContinuationPoint(simple_allocation);

    ANNOTATE("AllocateObject, initialize ES_Object");

    const Register reg_property_values = ALLOCATED_OBJECT_NAMED_PROPERTIES_REGISTER;
    const Register reg_indexed_properties = ALLOCATED_OBJECT_INDEXED_PROPERTIES_REGISTER;

    cg.Add(reg_property_values, reg_object, data.main_bytes + sizeof(ES_Box));

    if (nindexed)
        if (representation)
            cg.Load(reg_indexed_properties, representation);
        else
            cg.Add(reg_indexed_properties, reg_object, data.main_bytes + data.named_bytes);
    else
        cg.Move(reg_indexed_properties, REG_Z);

    cg.Load(REG_T8, data.gctag | (data.main_bytes << 16));
    cg.Load(REG_T9, data.object_bits);
    cg.Load(REG_AT, property_count);
    cg.SW(REG_T8, ES_OFFSETOF(ES_Object, hdr.header), reg_object);
    cg.Load(REG_T8, actual_klass);
    cg.SW(REG_T9, ES_OFFSETOF(ES_Object, object_bits), reg_object);
    cg.SW(REG_AT, ES_OFFSETOF(ES_Object, property_count), reg_object);
    cg.SW(REG_T8, ES_OFFSETOF(ES_Object, klass), reg_object);
    cg.SW(reg_property_values, ES_OFFSETOF(ES_Object, properties), reg_object);
    cg.SW(reg_indexed_properties, ES_OFFSETOF(ES_Object, indexed_properties), reg_object);

    ANNOTATE("AllocateObject, initialize ES_Properties");

    cg.Load(REG_T8, GCTAG_ES_Box | (data.named_bytes << 16));
    cg.SW(REG_T8, data.main_bytes, reg_object);

    if (nindexed)
    {
        ANNOTATE("AllocateObject, initialize 'length' property");

        cg.Load(REG_T1, data.length);
        cg.SW(REG_T1, 0, reg_property_values);

        if (!representation)
        {
            ANNOTATE("AllocateObject, initialize ES_Compact_Indexed_Properties");

            cg.Load(REG_T8, GCTAG_ES_Compact_Indexed_Properties | (data.indexed_bytes << 16));
            cg.Load(REG_AT, *nindexed);

            cg.SW(REG_T8, ES_OFFSETOF(ES_Compact_Indexed_Properties, hdr.header), reg_indexed_properties);
            cg.SW(REG_AT, ES_OFFSETOF(ES_Compact_Indexed_Properties, capacity), reg_indexed_properties);
            cg.SW(REG_T1, ES_OFFSETOF(ES_Compact_Indexed_Properties, top), reg_indexed_properties);
        }
    }
}

void
ES_Native::EmitNewObject(VirtualRegister *target_vr, ES_Class *klass, ES_CodeWord *first_property, unsigned nproperties)
{
    ANNOTATE("EmitNewObject");

    OP_ASSERT(nproperties == klass->Count());

    EmitSlowCaseCall();
    AllocateObject(*this, klass, klass, 0, NULL, NULL, current_slow_case->GetJumpTarget());

    const Register reg_object = ALLOCATED_OBJECT_REGISTER;
    const Register reg_properties = ALLOCATED_OBJECT_NAMED_PROPERTIES_REGISTER;

    ANNOTATE("EmitNewObject, update target virtual register");

    cg.Load(REG_T8, GetConst(CONST_ESTYPE_OBJECT));
    StoreValue(cg, target_vr, reg_object, REG_T8);

#ifdef NATIVE_DISASSEMBLER_ANNOTATION_SUPPORT
    cg.Annotate(UNI_L("       EmitNewObject(): copy property values\n"));
#endif // NATIVE_DISASSEMBLER_ANNOTATION_SUPPORT

    for (unsigned index = 0; index < nproperties; index ++)
    {
        ES_CodeGenerator::JumpTarget *null_target = NULL, *int_to_double_target = NULL;

        ES_Layout_Info layout = klass->GetLayoutInfoAtIndex(ES_LayoutIndex(index));
        ES_StorageType type = layout.GetStorageType();
        ES_ValueType value_type;
        VirtualRegister *source_vr = VR(first_property[index].index);
        BOOL known_type = GetType(source_vr, value_type);
        ES_StorageType source_type = ES_STORAGE_WHATEVER;
        BOOL skip_write = FALSE;

        if (known_type)
            source_type = ES_Value_Internal::ConvertToStorageType(value_type);

        if (type != ES_STORAGE_WHATEVER && source_type != type)
            skip_write = EmitTypeConversionCheck(type, source_type, source_vr, current_slow_case->GetJumpTarget(), null_target, int_to_double_target);

        if (!skip_write)
            CopyValueToTypedData(*this, source_vr, reg_properties, layout.GetOffset(), layout.GetStorageType(), NULL);

        EmitTypeConversionHandlers(source_vr, reg_properties, layout.GetOffset(), null_target, int_to_double_target);
    }

    EmitNewObjectSetPropertyCount(nproperties);
}

static void
EmitInitProperty(ES_Native &self, ES_CodeWord *word, const Register reg_properties, unsigned index)
{
    ES_CodeGenerator &cg = self.cg;
    ES_Code *code = self.code;

    const Register reg_type = REG_S0;
    const Register reg_value = REG_S1;

    switch (word ? word->instruction : ESI_LOAD_UNDEFINED)
    {
    case ESI_LOAD_STRING:
        {
            JString *value = code->GetString(word[2].index);

            cg.Load(reg_type, self.GetConst(CONST_ESTYPE_STRING));
            cg.Load(reg_value, value);
        }
        break;

    case ESI_LOAD_DOUBLE:
        {
            double &value = code->data->doubles[word[2].index];

            if (op_isnan(value))
            {
                cg.Load(reg_type, self.GetConst(CONST_ESTYPE_DOUBLE));
                cg.SW(reg_type, TYPE_OFFSET(index), reg_properties);
                return;
            }
            else
            {
                Constant *constant = cg.NewConstant(value);
                cg.Load(REG_F28, constant);
                cg.SDC1(REG_F28, DVALUE_OFFSET(index), reg_properties);
                return;
            }
        }
        break;

    case ESI_LOAD_INT32:
        cg.Load(reg_type, self.GetConst(CONST_ESTYPE_INT32));
        cg.Load(reg_value, word[2].immediate);
        break;

    case ESI_LOAD_NULL:
        cg.Load(reg_type, self.GetConst(CONST_ESTYPE_NULL));
        cg.Load(reg_value, 0);
        break;

    case ESI_LOAD_UNDEFINED:
        cg.Load(reg_type, self.GetConst(CONST_ESTYPE_UNDEFINED));
        cg.Load(reg_value, 1); // "hidden boolean"
        break;

    case ESI_LOAD_TRUE:
        cg.Load(reg_type, self.GetConst(CONST_ESTYPE_BOOLEAN));
        cg.Load(reg_value, 1);
        break;

    case ESI_LOAD_FALSE:
        cg.Load(reg_type, self.GetConst(CONST_ESTYPE_BOOLEAN));
        cg.Load(reg_value, 0);
        break;

    case ESI_PUTI_IMM:
    case ESI_PUTN_IMM:
    case ESI_INIT_PROPERTY:
        {
            ES_Native::VirtualRegister *source_vr = self.VR(word[3].index);

            ES_ValueType value_type;
            BOOL type_known = self.GetType(source_vr, value_type);

            if (!type_known || (value_type != ESTYPE_UNDEFINED && value_type != ESTYPE_NULL))
                LoadValue(cg, source_vr, reg_value, reg_type);
            else
                cg.Load(reg_type, GetTypeConst(self, value_type));

            if (word->instruction == ESI_PUTI_IMM)
            {
                if (!type_known)
                {
                    cg.Load(REG_AT, self.GetConst(CONST_ESTYPE_UNDEFINED));
                    cg.XOR(REG_T8, reg_type, REG_AT);
                    cg.Load(REG_T9, 1);
                    cg.MOVZ(reg_value, REG_T9, REG_T8);
                }
                else if (value_type == ESTYPE_UNDEFINED)
                    cg.Load(reg_value, 1);
            }
        }
        break;

    default:
        OP_ASSERT(FALSE);
    }

    cg.SW(reg_type, TYPE_OFFSET(index), reg_properties);
    cg.SW(reg_value, IVALUE_OFFSET(index), reg_properties);
}

void
ES_Native::EmitNewObjectSetPropertyCount(unsigned nproperties)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitNewObjectSetPropertyCount");

    cg.Load(REG_T8, nproperties);
    cg.SW(REG_T8, ES_OFFSETOF(ES_Object, property_count), ALLOCATED_OBJECT_REGISTER);
}

void
ES_Native::EmitNewArray(VirtualRegister *target_vr, unsigned length, unsigned &capacity, ES_Compact_Indexed_Properties *representation)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitNewArray");

    ES_Class *klass = code->global_object->GetArrayClass();

    const Register reg_object = ALLOCATED_OBJECT_REGISTER;
    const Register reg_indexed_properties = ALLOCATED_OBJECT_INDEXED_PROPERTIES_REGISTER;

    if (CanAllocateObject(klass, length))
    {
        ES_CodeGenerator::OutOfOrderBlock *slow_case = cg.StartOutOfOrderBlock();

        EmitInstructionHandlerCall();

        if (!representation)
        {
            cg.LW(reg_object, IVALUE_OFFSET(target_vr), BASE_REGISTER(target_vr));
            cg.LW(reg_indexed_properties, ES_OFFSETOF(ES_Object, indexed_properties), reg_object);
        }

        cg.EndOutOfOrderBlock();

        AllocateObject(*this, klass, klass, 1, &length, representation, slow_case->GetJumpTarget());

        capacity = length;

        cg.Load(REG_T8, GetConst(CONST_ESTYPE_OBJECT));
        cg.SW(reg_object, IVALUE_OFFSET(target_vr), BASE_REGISTER(target_vr));
        cg.SW(REG_T8, TYPE_OFFSET(target_vr), BASE_REGISTER(target_vr));

        cg.SetOutOfOrderContinuationPoint(slow_case);
    }
    else
    {
        capacity = length;

        EmitInstructionHandlerCall();

        if (!representation)
        {
            cg.LW(reg_object, IVALUE_OFFSET(target_vr), BASE_REGISTER(target_vr));
            cg.LW(reg_indexed_properties, ES_OFFSETOF(ES_Object, indexed_properties), reg_object);
        }
    }

    cg.Add(reg_indexed_properties, ES_OFFSETOF(ES_Compact_Indexed_Properties, values));
}

void
ES_Native::EmitNewArrayValue(VirtualRegister *target_vr, ES_CodeWord *word, unsigned index)
{
    const Register reg_indexed_properties = ALLOCATED_OBJECT_INDEXED_PROPERTIES_REGISTER;

#ifdef NATIVE_DISASSEMBLER_ANNOTATION_SUPPORT
    OpString annotation;
    annotation.AppendFormat(UNI_L("EmitNewArrayValue, for index %u\n"), index);
    cg.Annotate(annotation.CStr());
#endif // NATIVE_DISASSEMBLER_ANNOTATION_SUPPORT

    EmitInitProperty(*this, word, reg_indexed_properties, index);
}

void
ES_Native::EmitNewArrayReset(VirtualRegister *target_vr, unsigned start_index, unsigned end_index)
{
    ANNOTATE("EmitNewArrayReset");

    const Register reg_indexed_properties = ALLOCATED_OBJECT_INDEXED_PROPERTIES_REGISTER;

    cg.Load(REG_T8, GetConst(CONST_ESTYPE_UNDEFINED));

    for (unsigned index = start_index; index < end_index; index++)
    {
        cg.SW(REG_T8, TYPE_OFFSET(index), reg_indexed_properties);
        cg.SW(REG_Z, IVALUE_OFFSET(index), reg_indexed_properties);
    }
}

static void
ES_CheckIsBuiltin(ES_CodeGenerator &cg, ES_Native::VirtualRegister *function_vr, ES_BuiltInFunction builtin, ES_CodeGenerator::JumpTarget *slow_case)
{
    DECLARE_NOTHING();

    cg.LW(REG_T8, IVALUE_OFFSET(function_vr), BASE_REGISTER(function_vr));
    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, object_bits), REG_T8);
    cg.SRL(REG_T8, REG_T8, ES_Object::FUNCTION_ID_SHIFT);
    cg.Load(REG_T9, builtin);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_NE(REG_T8, REG_T9));
}

/* Emit a call to a bytecode function or a builtin/host function. */
void
ES_Native::EmitCall(VirtualRegister *this_vr, VirtualRegister *function_vr, unsigned argc, ES_BuiltInFunction builtin)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitCall");

    ES_CodeGenerator::JumpTarget *slow_case;

    if (!current_slow_case)
        EmitSlowCaseCall();

    slow_case = current_slow_case->GetJumpTarget();

    BOOL is_call = CurrentCodeWord()->instruction != ESI_CONSTRUCT;

    if (builtin != ES_BUILTIN_FN_NONE && builtin != ES_BUILTIN_FN_DISABLE)
    {
        if (builtin == ES_BUILTIN_FN_sin ||
            builtin == ES_BUILTIN_FN_cos ||
            builtin == ES_BUILTIN_FN_tan ||
            (!ARCHITECTURE_HAS_FPU() && (
                builtin == ES_BUILTIN_FN_sqrt ||
                builtin == ES_BUILTIN_FN_floor ||
                builtin == ES_BUILTIN_FN_ceil ||
                builtin == ES_BUILTIN_FN_abs)))
        {
            if (argc != 1)
                goto normal_builtin_call;

            VirtualRegister *arg_vr = this_vr + 2;
            ES_ValueType argtype;

            if (!GetType(arg_vr, argtype))
                EmitRegisterTypeCheck(arg_vr, ESTYPE_INT32_OR_DOUBLE, slow_case);
            else if (argtype > ESTYPE_INT32)
                goto normal_builtin_call;

            ES_CheckIsBuiltin(cg, function_vr, builtin, slow_case);

            switch(builtin)
            {
            case ES_BUILTIN_FN_sin:
                cg.Load(REG_T9, GetConst(CONST_ESN_MATHSIN));
                break;
            case ES_BUILTIN_FN_cos:
                cg.Load(REG_T9, GetConst(CONST_ESN_MATHCOS));
                break;
            case ES_BUILTIN_FN_tan:
                cg.Load(REG_T9, GetConst(CONST_ESN_MATHTAN));
                break;
            case ES_BUILTIN_FN_sqrt:
                cg.Load(REG_T9, GetConst(CONST_ESN_MATHSQRT));
                break;
            case ES_BUILTIN_FN_floor:
                cg.Load(REG_T9, GetConst(CONST_ESN_MATHFLOOR));
                break;
            case ES_BUILTIN_FN_ceil:
                cg.Load(REG_T9, GetConst(CONST_ESN_MATHCEIL));
                break;
            case ES_BUILTIN_FN_abs:
                cg.Load(REG_T9, GetConst(CONST_ESN_MATHABS));
                break;
            }

            cg.JALR(REG_T9);
            cg.Add(REG_A0, REG_FP, REGISTER_OFFSET(this_vr));
        }
        else if (builtin == ES_BUILTIN_FN_sqrt ||
            builtin == ES_BUILTIN_FN_floor ||
            builtin == ES_BUILTIN_FN_ceil ||
            builtin == ES_BUILTIN_FN_abs)
        {
            if (argc != 1)
                goto normal_builtin_call;

            VirtualRegister *arg_vr = this_vr + 2;
            ES_ValueType argtype;

            if (!GetType(arg_vr, argtype) || argtype == ESTYPE_INT32_OR_DOUBLE)
            {
                unsigned argbits = code->data->profile_data[cw_index + 2];

                if (argbits == ESTYPE_BITS_INT32)
                    argtype = ESTYPE_INT32;
                else if ((argbits & ESTYPE_BITS_DOUBLE) || argtype == ESTYPE_INT32_OR_DOUBLE)
                    argtype = ESTYPE_DOUBLE;
                else
                    goto normal_builtin_call;

                EmitRegisterTypeCheck(arg_vr, argtype, slow_case);
            }
            else if (argtype > ESTYPE_INT32)
                goto normal_builtin_call;

            ES_CheckIsBuiltin(cg, function_vr, builtin, slow_case);

            if (builtin == ES_BUILTIN_FN_sqrt)
            {
                if (argtype == ESTYPE_INT32)
                {
                    cg.LWC1(REG_F28, IVALUE_OFFSET(arg_vr), BASE_REGISTER(arg_vr));
                    cg.CVT(D, W, REG_F30, REG_F28);
                }
                else
                    cg.LDC1(REG_F30, DVALUE_OFFSET(arg_vr), BASE_REGISTER(arg_vr));

                cg.SQRT(D, REG_F30, REG_F30);

                cg.SDC1(REG_F30, DVALUE_OFFSET(this_vr), BASE_REGISTER(this_vr));
            }
            else if (builtin == ES_BUILTIN_FN_floor || builtin == ES_BUILTIN_FN_ceil)
            {
                if (argtype == ESTYPE_DOUBLE)
                {
                    cg.LDC1(REG_F30, DVALUE_OFFSET(arg_vr), BASE_REGISTER(arg_vr));

                    if (builtin == ES_BUILTIN_FN_floor)
                        cg.FLOOR(W, D, REG_F28, REG_F30);
                    else
                        cg.CEIL(W, D, REG_F28, REG_F30);

                    CheckFPUException(cg, slow_case);

                    cg.MFC1(REG_T9, REG_F28);

                    ES_CodeGenerator::JumpTarget *store_int_value = cg.ForwardJump();
                    ES_CodeGenerator::OutOfOrderBlock *negative_zero_check = cg.StartOutOfOrderBlock();

                    ANNOTATE("EmitCall, floor/ceil negative zero check");

                    if(fpu_type == ES_CodeGenerator::FPU_DOUBLE)
                        cg.MFHC1(REG_AT, REG_F30);
                    else
                        cg.MFC1(REG_AT, REG_F31);

                    // -0 as a double is 0x8000000000000000
                    cg.LUI(REG_T8, 0x8000);
                    cg.AND(REG_AT, REG_AT, REG_T8);
                    cg.JumpEQ(store_int_value, REG_AT, REG_Z);

                    cg.EndOutOfOrderBlock();

                    cg.Jump(negative_zero_check->GetJumpTarget(), ES_NATIVE_CONDITION_EQZ(REG_T9));

                    cg.SetJumpTarget(store_int_value);
                    cg.Load(REG_T8, GetConst(CONST_ESTYPE_INT32));

                    cg.SetOutOfOrderContinuationPoint(negative_zero_check);
                    cg.SW(REG_T9, IVALUE_OFFSET(this_vr), BASE_REGISTER(this_vr));
                    cg.SW(REG_T8, TYPE_OFFSET(this_vr), BASE_REGISTER(this_vr));
                }
                else
                {
                    cg.LW(REG_T8, IVALUE_OFFSET(arg_vr), BASE_REGISTER(arg_vr));
                    cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));
                    cg.SW(REG_T8, IVALUE_OFFSET(this_vr), BASE_REGISTER(this_vr));
                    cg.SW(REG_T9, TYPE_OFFSET(this_vr), BASE_REGISTER(this_vr));
                }
            }
            else if (builtin == ES_BUILTIN_FN_abs)
            {
                if (argtype == ESTYPE_INT32)
                {
                    ES_CodeGenerator::JumpTarget *skip = cg.ForwardJump();

                    cg.LW(REG_T8, IVALUE_OFFSET(arg_vr), BASE_REGISTER(arg_vr));
                    cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));

                    cg.Jump(skip, ES_NATIVE_CONDITION_GEZ(REG_T8), TRUE);

                    ES_CodeGenerator::OutOfOrderBlock *int_min = cg.StartOutOfOrderBlock();
                    ANNOTATE("EmitCall, abs, store -INT_MIN as double");

                    // -INT_MIN as a double is 0x41e0000000000000
                    cg.Move(REG_T8, REG_Z);
                    cg.LUI(REG_T9, 0x41e0);
                    cg.EndOutOfOrderBlock();

                    cg.Load(REG_AT, INT_MIN);
                    cg.Jump(int_min->GetJumpTarget(), ES_NATIVE_CONDITION_EQ(REG_T8, REG_AT));

                    cg.SUBU(REG_T8, REG_Z, REG_T8);

                    cg.SetJumpTarget(skip);
                    cg.SetOutOfOrderContinuationPoint(int_min);

                    cg.SW(REG_T8, IVALUE_OFFSET(this_vr), BASE_REGISTER(this_vr));
                    cg.SW(REG_T9, TYPE_OFFSET(this_vr), BASE_REGISTER(this_vr));
                }
                else
                {
                    cg.LDC1(REG_F30, DVALUE_OFFSET(arg_vr), BASE_REGISTER(arg_vr));
                    cg.ABS(D, REG_F30, REG_F30);
                    cg.SDC1(REG_F30, DVALUE_OFFSET(this_vr), BASE_REGISTER(this_vr));
                }
            }
        }
        else if (builtin == ES_BUILTIN_FN_pow)
        {
            if (argc != 2)
                goto normal_builtin_call;

            if ((code->data->profile_data[cw_index + 2] & ~(ESTYPE_BITS_INT32 | ESTYPE_BITS_DOUBLE)) != 0)
                goto normal_builtin_call;

            ES_CheckIsBuiltin(cg, function_vr, builtin, slow_case);

            cg.Load(REG_T9, GetConst(CONST_ESN_MATHPOW));

            cg.JALR(REG_T9);
            cg.Add(REG_A0, REG_FP, REGISTER_OFFSET(this_vr));

            cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(REG_V0));
        }
        else if (builtin == ES_BUILTIN_FN_push)
        {
            if (argc != 1)
                goto normal_builtin_call;

            EmitRegisterTypeCheck(this_vr, ESTYPE_OBJECT, slow_case);

            ES_Value_Internal value;
            EmitInt32IndexedPut(this_vr, NULL, 0, this_vr + 2, FALSE, FALSE, value, TRUE);
        }
        else if (builtin == ES_BUILTIN_FN_charCodeAt || builtin == ES_BUILTIN_FN_charAt)
        {
            ES_CheckIsBuiltin(cg, function_vr, builtin, slow_case);

            EmitRegisterTypeCheck(this_vr, ESTYPE_STRING, slow_case);
            if (argc > 0)
                EmitRegisterTypeCheck(this_vr + 2, ESTYPE_INT32, slow_case);

            EmitInt32StringIndexedGet(this_vr, this_vr, argc > 0 ? this_vr + 2 : NULL, 0, builtin == ES_BUILTIN_FN_charCodeAt);
        }
        else if (builtin == ES_BUILTIN_FN_fromCharCode && argc == 1)
        {
            VirtualRegister *arg_vr = this_vr + 2;
            ES_Value_Internal arg_value;

            if (IsImmediate(arg_vr, arg_value, TRUE))
            {
                BOOL handled = FALSE;
                if (arg_value.IsInt32())
                {
                    int arg = arg_value.GetNumAsInt32();
                    if (arg >= 0 && arg < STRING_NUMCHARS)
                    {
                        ES_CheckIsBuiltin(cg, function_vr, builtin, slow_case);

                        const Register reg_type = REG_S0;
                        const Register reg_value = REG_S1;

                        cg.Load(reg_type, GetConst(CONST_ESTYPE_STRING));
                        cg.Load(reg_value, context->rt_data->strings[STRING_nul + arg]);
                        StoreValue(cg, this_vr, reg_value, reg_type);
                        handled = TRUE;
                    }
                }
                if (!handled)
                    goto normal_builtin_call;
            }
            else
            {
                ES_CheckIsBuiltin(cg, function_vr, builtin, slow_case);

                EmitRegisterTypeCheck(arg_vr, ESTYPE_INT32, slow_case);
                cg.LW(REG_T8, IVALUE_OFFSET(arg_vr), BASE_REGISTER(arg_vr));
                cg.JumpGEU(slow_case, REG_T8, STRING_NUMCHARS);

                const Register reg_type = REG_S0;
                const Register reg_value = REG_S1;
                cg.LW(reg_value, ES_OFFSETOF(ES_Execution_Context, rt_data), REG_S7);
                cg.Add(reg_value, ES_OFFSETOF(ESRT_Data, strings) + STRING_nul * sizeof(void *));
                cg.SLL(REG_T8, REG_T8, 2);
                cg.Add(reg_value, REG_T8);
                cg.LW(reg_value, 0, reg_value);

                cg.Load(reg_type, GetConst(CONST_ESTYPE_STRING));
                StoreValue(cg, this_vr, reg_value, reg_type);
            }
        }
        else if (builtin == ES_BUILTIN_FN_OTHER)
        {
normal_builtin_call:
            const Register reg_function = REG_S0;
            const Register reg_header = REG_S1;

            cg.LW(reg_function, IVALUE_OFFSET(function_vr), BASE_REGISTER(function_vr));
            cg.LW(reg_header, ES_OFFSETOF(ES_Object, hdr.header), reg_function);
            cg.And(reg_header, ES_Header::MASK_GCTAG);
            cg.JumpNE(slow_case, reg_header, GCTAG_ES_Object_Function);

            const Register reg_fncode = REG_S1;

            cg.LW(reg_fncode, ES_OFFSETOF(ES_Function, function_code), reg_function);
            cg.Jump(slow_case, ES_NATIVE_CONDITION_NEZ(reg_fncode));
            cg.Load(REG_A1, argc);

            cg.LW(REG_T9, ES_OFFSETOF(ES_Function, data.builtin.call), reg_function);
            cg.Add(REG_A2, REG_FP, REGISTER_OFFSET(this_vr->index + 2));
            cg.Add(REG_A3, REG_FP, REGISTER_OFFSET(this_vr));

            cg.JALR(REG_T9);
            cg.Move(REG_A0, REG_S7);

            ES_CodeGenerator::OutOfOrderBlock *exception_thrown = cg.StartOutOfOrderBlock();
            {
                /* This call doesn't return. */
                CallContextFunc(cg, GetConst(CONST_EXC_THROWFROMMACHINECODE));
            }
            cg.EndOutOfOrderBlock(FALSE);

            cg.Jump(exception_thrown->GetJumpTarget(), ES_NATIVE_CONDITION_EQZ(REG_V0), FALSE);
        }
        else
            goto normal_call;
    }
    else
    {
normal_call:

        const Register reg_function = REG_T9;
        const Register reg_function_bits = REG_T8;

        cg.LW(reg_function, IVALUE_OFFSET(function_vr), BASE_REGISTER(function_vr));
        cg.LW(reg_function_bits, ES_OFFSETOF(ES_Object, object_bits), reg_function);

        int mask = is_call ? ES_Object::MASK_IS_NATIVE_FN : ES_Object::MASK_IS_DISPATCHED_CTOR;

        cg.And(reg_function_bits, mask);
        cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(reg_function_bits));

        if (is_call)
        {
            cg.LW(reg_function, ES_OFFSETOF(ES_Function, function_code), reg_function);
            cg.LW(reg_function, ES_OFFSETOF(ES_FunctionCode, native_dispatcher), reg_function);
            cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(reg_function));
        }
        else
            cg.LW(reg_function, ES_OFFSETOF(ES_Function, data.native.native_ctor_dispatcher), reg_function);

        cg.Load(REG_A1, argc);
        cg.JALR(reg_function);
        cg.Add(REG_FP, this_vr->index * sizeof(ES_Value_Internal));

        SetFunctionCallReturnTarget(cg.Here());

        cg.Sub(REG_FP, this_vr->index * sizeof(ES_Value_Internal));
        cg.LoadDataSectionPointer();
    }
}

/* Emit a "redirected" call to a function. */
void
ES_Native::EmitRedirectedCall(VirtualRegister *function_vr, VirtualRegister *apply_vr, BOOL check_constructor_object_result)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitRedirectedCall");

    if (!current_slow_case)
        EmitSlowCaseCall();

    ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();
    const Register reg_apply = REG_A0;

    cg.LW(reg_apply, IVALUE_OFFSET(apply_vr), BASE_REGISTER(apply_vr));

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, object_bits), reg_apply);
    cg.And(REG_T8, ES_Object::MASK_IS_APPLY_FN);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(REG_T8));

    frame_size = ES_NativeStackFrame::GetFrameSize(code, TRUE);

    cg.LW(REG_A1, SP_OFFSET_ARGUMENT_COUNT(frame_size), REG_SP);
    cg.JumpGT(slow_case, REG_A1, ES_REDIRECTED_CALL_MAXIMUM_ARGUMENTS);

    const Register reg_function = REG_S0;
    const Register reg_function_code = REG_S1;
    const Register reg_dispatcher = REG_S2;

    cg.LW(reg_function, IVALUE_OFFSET(function_vr), BASE_REGISTER(function_vr));

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, object_bits), reg_function);
    cg.And(REG_T8, ES_Object::MASK_IS_NATIVE_FN);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(REG_T8));

    cg.LW(reg_function_code, ES_OFFSETOF(ES_Function, function_code), reg_function);
    cg.LW(reg_dispatcher, ES_OFFSETOF(ES_FunctionCode, native_dispatcher), reg_function_code);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(reg_dispatcher));

    if (constructor)
    {
        cg.Sub(REG_SP, 8);
        LoadValue(cg, VR(0), REG_T8, REG_T9);
        cg.SW(REG_T8, MIPS_STACK_ARG_AREA + 4, REG_SP);
        cg.SW(REG_T9, MIPS_STACK_ARG_AREA, REG_SP);
    }

    cg.JALR(reg_dispatcher);
    cg.SW(reg_function, IVALUE_OFFSET(VR(1)), BASE_REGISTER(VR(1)));

    SetFunctionCallReturnTarget(cg.Here());

    cg.LoadDataSectionPointer();

    if (constructor)
    {
        cg.LW(REG_T8, MIPS_STACK_ARG_AREA + 4, REG_SP);
        cg.LW(REG_T9, MIPS_STACK_ARG_AREA, REG_SP);
        cg.Add(REG_SP, 8);

        ES_CodeGenerator::JumpTarget *keep_object_result = NULL;
        if (check_constructor_object_result)
        {
            /* Restore reg[0] if return value isn't an object. */
            cg.Load(REG_AT, GetConst(CONST_ESTYPE_OBJECT));
            cg.LW(REG_A0, TYPE_OFFSET(VR(0)), BASE_REGISTER(VR(0)));

            keep_object_result = cg.ForwardJump();
            cg.Jump(keep_object_result, ES_NATIVE_CONDITION_EQ(REG_A0, REG_AT));
        }

        cg.SW(REG_T8, IVALUE_OFFSET(VR(0)), BASE_REGISTER(VR(0)));
        cg.SW(REG_T9, TYPE_OFFSET(VR(0)), BASE_REGISTER(VR(0)));
        if (keep_object_result)
            cg.SetJumpTarget(keep_object_result);
    }
}

/* Emit function call via ESI_APPLY. */
void
ES_Native::EmitApply(VirtualRegister *apply_vr, VirtualRegister *this_vr, VirtualRegister *function_vr, unsigned argc)
{
    DECLARE_NOTHING();

    if (!current_slow_case)
        EmitSlowCaseCall();

    ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();
    const Register reg_apply = REG_A0;

    cg.LW(reg_apply, IVALUE_OFFSET(apply_vr), BASE_REGISTER(apply_vr));

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, object_bits), reg_apply);
    cg.And(REG_T8, ES_Object::MASK_IS_APPLY_FN);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(REG_T8));

    const Register reg_function = REG_S0;
    const Register reg_function_code = REG_S1;
    const Register reg_dispatcher = REG_S2;

    cg.LW(reg_function, IVALUE_OFFSET(function_vr), BASE_REGISTER(function_vr));

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, object_bits), reg_function);
    cg.And(REG_T8, ES_Object::MASK_IS_NATIVE_FN);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(REG_T8));

    cg.LW(reg_function_code, ES_OFFSETOF(ES_Function, function_code), reg_function);
    cg.LW(reg_dispatcher, ES_OFFSETOF(ES_FunctionCode, native_dispatcher), reg_function_code);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(reg_dispatcher));

    cg.Add(REG_FP, REGISTER_OFFSET(this_vr));
    cg.Load(REG_A1, argc);

    cg.JALR(reg_dispatcher);
    cg.NOP();

    SetFunctionCallReturnTarget(cg.Here());

    cg.LoadDataSectionPointer();
    cg.Sub(REG_FP, REGISTER_OFFSET(this_vr));
}

/* Check that the called function really is builtin eval, and if so, call
   ES_Execution_Context::EvalFromMachineCode, otherwise do a regular function call. */
void
ES_Native::EmitEval(VirtualRegister *this_vr, VirtualRegister *function_vr, unsigned argc)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitEval");

    if (!current_slow_case)
        EmitSlowCaseCall();

    cg.LW(REG_T8, IVALUE_OFFSET(function_vr), BASE_REGISTER(function_vr));
    cg.LW(REG_T9, ES_OFFSETOF(ES_Object, object_bits), REG_T8);

    cg.And(REG_T9, ES_Object::MASK_IS_EVAL_FN);

    cg.Jump(current_slow_case->GetJumpTarget(), ES_NATIVE_CONDITION_EQZ(REG_T9));

    cg.Load(REG_A1, cw_index);
    CallContextFunc(cg, GetConst(CONST_EXC_EVALFROMMACHINECODE));
}

ES_CodeGenerator::OutOfOrderBlock*
ES_Native::EmitInlinedCallPrologue(VirtualRegister *this_vr, VirtualRegister *function_vr, ES_Function *function, unsigned char *mark_failure, unsigned argc, BOOL function_fetched)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitInlinedCallPrologue");

    ES_CodeGenerator::OutOfOrderBlock *failure = cg.StartOutOfOrderBlock();

    ANNOTATE("EmitInlinedCallPrologue, failure path");

    cg.Sub(REG_FP, REGISTER_OFFSET(this_vr->index));

    if (!function_fetched)
    {
        cg.Load(REG_T8, GetConst(CONST_ESTYPE_OBJECT));
        cg.Load(REG_T9, function);
        cg.SW(REG_T8, TYPE_OFFSET(function_vr), BASE_REGISTER(function_vr));
        cg.SW(REG_T9, IVALUE_OFFSET(function_vr), BASE_REGISTER(function_vr));
    }

    cg.Load(REG_T8, GetConst(CONST_CODE));
    cg.Load(REG_T9, 1);

    cg.SW(REG_T9, ES_OFFSETOF(ES_Code, has_failed_inlined_function), REG_T8);

    Constant *profile_data = cg.NewConstant((void*)mark_failure);
    cg.Load(REG_T8, profile_data);
    cg.LBU(REG_T9, 0, REG_T8);
    cg.ORI(REG_T9, REG_T9, ES_CodeStatic::GET_FAILED_INLINE);
    cg.SB(REG_T9, 0, REG_T8);

    cg.Load(REG_A1, cw_index + (constructor ? code->data->codewords_count : 0));

    CallContextFunc(cg, GetConst(CONST_EXC_FORCEUPDATENATIVEDISPATCHER));

    cg.JR(REG_V0);
    cg.NOP();

    cg.EndOutOfOrderBlock(FALSE);

    cg.Add(REG_FP, REGISTER_OFFSET(this_vr->index));

    return failure;
}

void
ES_Native::EmitInlinedCallEpilogue(VirtualRegister *this_vr, VirtualRegister *function_vr, unsigned argc)
{
    ANNOTATE("EmitInlinedCallEpilogue");

    cg.Sub(REG_FP, REGISTER_OFFSET(this_vr->index));
}

/* Emit a "slow" return. */
void
ES_Native::EmitReturn()
{
    if (cw_index != code->data->instruction_offsets[code->data->instruction_count - 1])
        cg.Jump(epilogue_jump_target);
}

void
ES_Native::EmitNormalizeValue(VirtualRegister *vr, ES_CodeGenerator::JumpTarget *slow_case)
{
    ANNOTATE("EmitNormalizeValue");

    ES_CodeGenerator::JumpTarget *is_int = NULL;

    if (!ARCHITECTURE_HAS_FPU())
    {
        EmitRegisterTypeCheck(vr, ESTYPE_INT32, slow_case);
        return;
    }
    else
    {
        is_int = cg.ForwardJump();
        EmitRegisterTypeCheck(vr, ESTYPE_INT32, is_int, TRUE);
    }

    EmitRegisterTypeCheck(vr, ESTYPE_DOUBLE, slow_case);

    if (!slow_case)
    {
        OP_ASSERT(current_slow_case != NULL);
        slow_case = current_slow_case->GetJumpTarget();
    }

    cg.LDC1(REG_F28, DVALUE_OFFSET(vr), BASE_REGISTER(vr));
    cg.FLOOR(W, D, REG_F30, REG_F28);

    CheckFPUException(cg, slow_case);

    cg.CVT(D, W, ES_CodeGenerator::REG_F26, REG_F30);

    cg.C(EQ, D, ES_CodeGenerator::REG_F26, REG_F28);

    cg.Jump(slow_case, ES_NATIVE_CONDITION_C1F(0));

    cg.MFC1(REG_T8, REG_F30);
    cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));

    StoreValue(cg, vr, REG_T8, REG_T9);

    cg.SetJumpTarget(is_int);
}

static void
LoadObjectOperand(ES_Native &n, Register reg_target, VirtualRegister *source)
{
    if (n.property_value_read_vr && n.property_value_nr)
    {
        OP_ASSERT(n.property_value_read_vr == source);
        n.cg.Load(reg_target, n.property_value_offset + (n.property_value_needs_type_check ? VALUE_IVALUE_OFFSET : 0), INTEGER_REGISTER(n.property_value_nr));
    }
    else
        n.cg.LW(reg_target, IVALUE_OFFSET(source), BASE_REGISTER(source));
}

void
ES_Native::EmitInt32IndexedGet(VirtualRegister *target, VirtualRegister *object, VirtualRegister *index, unsigned constant_index)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitInt32IndexedGet");

    if (!current_slow_case)
        EmitSlowCaseCall();

    ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();

    const Register reg_object = REG_V0;
    const Register reg_properties = REG_V1;

    LoadObjectOperand(*this, reg_object, object);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, object_bits), reg_object);
    cg.And(REG_T8, ES_Object::MASK_SIMPLE_COMPACT_INDEXED);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(REG_T8));

    cg.LW(reg_properties, ES_OFFSETOF(ES_Object, indexed_properties), reg_object);

    const Register reg_index = REG_A0;
    const Register reg_value = REG_A1;

    if (index)
    {
        cg.LW(reg_index, IVALUE_OFFSET(index), BASE_REGISTER(index));
        cg.Jump(slow_case, ES_NATIVE_CONDITION_LTZ(reg_index));
    }
    else
        cg.Load(reg_index, constant_index);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Compact_Indexed_Properties, capacity), reg_properties);
    cg.JumpGE(slow_case, reg_index, REG_T8);

    cg.SLL(REG_T8, reg_index, 3);
    cg.ADDU(reg_value, reg_properties, REG_T8);
    cg.Add(reg_value, ES_OFFSETOF(ES_Compact_Indexed_Properties, values));

    if (property_value_write_vr)
    {
        OP_ASSERT(property_value_write_vr == target);

        cg.LW(REG_T8, ES_OFFSETOF(ES_Value_Internal, private_type), reg_value);
        cg.Load(REG_T9, GetConst(CONST_ESTYPE_UNDEFINED));
        cg.Jump(slow_case, ES_NATIVE_CONDITION_EQ(REG_T8, REG_T9));

        cg.Move(VALUE_TRANSFER_REGISTER, reg_value);

        SetPropertyValueTransferRegister(VALUE_TRANSFER_NATIVE_REGISTER, 0, TRUE);

        if (current_slow_case)
        {
            ES_CodeGenerator::OutOfOrderBlock *recover = cg.StartOutOfOrderBlock();

            ANNOTATE("EmitInt32IndexedGet, recover slow case");

            cg.SetOutOfOrderContinuationPoint(current_slow_case);
            current_slow_case = NULL;

            cg.Add(VALUE_TRANSFER_REGISTER, REG_FP, REGISTER_OFFSET(target));

            cg.EndOutOfOrderBlock();
            cg.SetOutOfOrderContinuationPoint(recover);
        }
    }
    else
    {
        ES_CodeGenerator::JumpTarget *safe = cg.ForwardJump();

        cg.LW(REG_T8, ES_OFFSETOF(ES_Value_Internal, private_type), reg_value);
        cg.LW(REG_T9, ES_OFFSETOF(ES_Value_Internal, private_value), reg_value);

        cg.Load(REG_AT, GetConst(CONST_ESTYPE_UNDEFINED));
        cg.Jump(safe, ES_NATIVE_CONDITION_NE(REG_T8, REG_AT));

        cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(REG_T9));

        cg.SetJumpTarget(safe);

        cg.SW(REG_T8, TYPE_OFFSET(target), BASE_REGISTER(target));
        cg.SW(REG_T9, IVALUE_OFFSET(target), BASE_REGISTER(target));
    }
}

void
ES_Native::EmitInt32IndexedPut(VirtualRegister *object, VirtualRegister *index, unsigned constant_index, VirtualRegister *source, BOOL known_type, BOOL known_value, const ES_Value_Internal &value, BOOL is_push)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitInt32IndexedPut");

    unsigned length_offset = code->global_object->GetArrayClass()->GetLayoutInfoAtIndex(ES_LayoutIndex(0)).GetOffset();

    if (!current_slow_case)
        EmitSlowCaseCall();

    ES_CodeGenerator::JumpTarget *slow_case;

    const Register reg_object = REG_V0;

    if (!is_light_weight && property_value_read_vr && property_value_nr)
    {
        ES_CodeGenerator::OutOfOrderBlock *flush_object_vr = cg.StartOutOfOrderBlock();

        cg.Load(REG_T8, GetConst(CONST_ESTYPE_OBJECT));
        StoreValue(cg, object, reg_object, REG_T8);

        cg.Jump(current_slow_case_main);
        cg.EndOutOfOrderBlock(FALSE);

        slow_case = flush_object_vr->GetJumpTarget();
    }
    else
        slow_case = current_slow_case->GetJumpTarget();

    LoadObjectOperand(*this, reg_object, object);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, object_bits), reg_object);
    cg.And(REG_T8, ES_Object::MASK_MUTABLE_COMPACT_INDEXED);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(REG_T8));

    const Register reg_index = REG_A0;
    const Register reg_value = REG_A1;
    const Register reg_length = REG_A2;
    const Register reg_klass = REG_A3;

    const Register reg_properties = REG_T1;
    const Register reg_indexed_properties = REG_T0;

    cg.LW(reg_indexed_properties, ES_OFFSETOF(ES_Object, indexed_properties), reg_object);

    if (is_push)
    {
        /* Theory: When inlining Array.prototype.push, do the class check
           early since we'll need to get the 'length' property from the
           array object. If the class id check holds, it is not just
           guaranteed that the object is an array, but also that the type
           of length is UINT32. */
        cg.LW(reg_klass, ES_OFFSETOF(ES_Object, klass), reg_object);
        cg.LW(REG_T8, ES_OFFSETOF(ES_Class, class_id), reg_klass);
        cg.JumpNE(slow_case, REG_T8, code->global_object->GetArrayClass()->GetId(context));

        // When pushing index = length.
        cg.LW(reg_properties, ES_OFFSETOF(ES_Object, properties), reg_object);
        cg.Load(reg_index, length_offset, reg_properties);
    }
    else if (index)
    {
        cg.LW(reg_index, IVALUE_OFFSET(index), BASE_REGISTER(index));
        cg.Jump(slow_case, ES_NATIVE_CONDITION_LTZ(reg_index));
    }
    else
        cg.Load(reg_index, constant_index);

    ES_CodeGenerator::OutOfOrderBlock *check_capacity_and_length = is_push ? NULL : cg.StartOutOfOrderBlock();
    ES_CodeGenerator::JumpTarget *no_length_update = is_push ? NULL : check_capacity_and_length->GetContinuationTarget();

    ANNOTATE("EmitInt32IndexedPut, check capacity and length");

    cg.LW(REG_T8, ES_OFFSETOF(ES_Compact_Indexed_Properties, capacity), reg_indexed_properties);
    cg.JumpGE(slow_case, reg_index, REG_T8);

    /* Check if object requires a length update + if it is an instance of Array */
    if (!is_push)
    {
        cg.LW(REG_T8, ES_OFFSETOF(ES_Object, hdr.header), reg_object);
        cg.Load(REG_T9, GCTAG_ES_Object_Array);
        cg.And(REG_T8, ES_Header::MASK_GCTAG);
        cg.JumpNE(no_length_update, REG_T8, REG_T9);

        cg.LW(reg_klass, ES_OFFSETOF(ES_Object, klass), reg_object);
        cg.LW(REG_T8, ES_OFFSETOF(ES_Class, class_id), reg_klass);
        cg.Load(REG_T9, code->global_object->GetArrayClass()->GetId(context));
        cg.JumpNE(slow_case, REG_T8, REG_T9);

        cg.LW(reg_properties, ES_OFFSETOF(ES_Object, properties), reg_object);
        cg.Load(reg_length, length_offset, reg_properties);
        cg.JumpLT(no_length_update, reg_index, reg_length);
    }

    /* Update 'length' to index+1.  We're ignoring overflow here, see comment in
    EmitInt32IndexedPut() in es_native_ia32.cpp for an explanation.  Also
    update 'top' which must have been equal to 'length', since 'length' is
    obviously less 'capacity'. */
    cg.ADDIU(REG_T8, reg_index, 1);
    cg.Store(REG_T8, length_offset, reg_properties);
    cg.SW(REG_T8, ES_OFFSETOF(ES_Compact_Indexed_Properties, top), reg_indexed_properties);

    if (check_capacity_and_length)
    {
        cg.EndOutOfOrderBlock();

        cg.LW(REG_T8, ES_OFFSETOF(ES_Compact_Indexed_Properties, top), reg_indexed_properties);
        cg.JumpGE(check_capacity_and_length->GetJumpTarget(), reg_index, REG_T8);

        cg.SetOutOfOrderContinuationPoint(check_capacity_and_length);
    }

    cg.SLL(REG_T8, reg_index, 3);
    cg.ADDU(reg_value, reg_indexed_properties, REG_T8);
    cg.Add(reg_value, ES_OFFSETOF(ES_Compact_Indexed_Properties, values));

    const Register value_type = REG_S0;
    const Register value_value = REG_S1;

    if (known_type && value.Type() != ESTYPE_DOUBLE && value.Type() != ESTYPE_STRING && value.Type() != ESTYPE_OBJECT)
    {
        cg.Load(value_type, GetTypeConst(*this, value.Type()));

        if (known_value || value.IsNullOrUndefined())
        {
            switch (value.Type())
            {
            case ESTYPE_NULL:
                break;

            case ESTYPE_UNDEFINED:
                cg.Load(value_value, 1);
                break;

            case ESTYPE_BOOLEAN:
                cg.Load(value_value, value.GetBoolean() ? 1 : 0);
                break;

            case ESTYPE_INT32:
                cg.Load(value_value, value.GetInt32());
            }
        }
        else
            cg.LW(value_value, IVALUE_OFFSET(source), BASE_REGISTER(source));
    }
    else
    {
        cg.LW(value_type, TYPE_OFFSET(source), BASE_REGISTER(source));
        cg.LW(value_value, IVALUE_OFFSET(source), BASE_REGISTER(source));

        if (!known_type)
        {
            cg.Load(REG_T8, GetConst(CONST_ESTYPE_UNDEFINED));
            cg.BNE(value_type, REG_T8, 2);
            cg.NOP();
            cg.ORI(value_value, REG_Z, 1);
        }
    }

    cg.SW(value_type, ES_OFFSETOF(ES_Value_Internal, private_type), reg_value);
    cg.SW(value_value, ES_OFFSETOF(ES_Value_Internal, private_value), reg_value);

    if (is_push)
    {
        /* The return value of Array.prototype.push is the new value of the
           'length' property. We know that 'object_vr' is the receiver of
           the call to Array.prototype.push and that that register doubles as
           return register, so write the new length here instead of doing a
           EmitLengthGet which will check types that we've already checked. */
        cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));
        cg.Add(REG_T8, reg_index, 1);
        StoreValue(cg, object, REG_T8, REG_T9);
    }
}

void
ES_Native::EmitInt32ByteArrayGet(VirtualRegister *target, VirtualRegister *object, VirtualRegister *index, unsigned constant_index)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitInt32ByteArrayGet");

    if (!current_slow_case)
        EmitSlowCaseCall();

    ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();

    const Register reg_object = REG_A0;
    const Register reg_bytearray = REG_A1;
    const Register reg_index = REG_A2;

    LoadObjectOperand(*this, reg_object, object);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, object_bits), reg_object);
    cg.And(REG_T8, ES_Object::MASK_BYTE_ARRAY_INDEXED);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(REG_T8));

    cg.LW(reg_bytearray, ES_OFFSETOF(ES_Object, indexed_properties), reg_object);

    if (index)
    {
        cg.LW(reg_index, IVALUE_OFFSET(index), BASE_REGISTER(index));
        cg.Jump(slow_case, ES_NATIVE_CONDITION_LTZ(reg_index));
    }
    else
        cg.Load(reg_index, constant_index);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Byte_Array_Indexed, capacity), reg_bytearray);
    cg.JumpGE(slow_case, reg_index, REG_T8);

    cg.LW(reg_bytearray, ES_OFFSETOF(ES_Byte_Array_Indexed, byte_array), reg_bytearray);

    cg.Add(reg_bytearray, reg_index);
    cg.LBU(REG_T8, 0, reg_bytearray);
    cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));

    cg.SW(REG_T8, IVALUE_OFFSET(target), BASE_REGISTER(target));
    cg.SW(REG_T9, TYPE_OFFSET(target), BASE_REGISTER(target));
}

void
ES_Native::EmitInt32ByteArrayPut(VirtualRegister *object, VirtualRegister *index, unsigned constant_index, VirtualRegister *source, int *known_value)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitInt32ByteArrayPut");

    if (!current_slow_case)
        EmitSlowCaseCall();

    ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();

    const Register reg_object = REG_A0;
    const Register reg_bytearray = REG_A1;
    const Register reg_index = REG_A2;

    LoadObjectOperand(*this, reg_object, object);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, object_bits), reg_object);
    cg.And(REG_T8, ES_Object::MASK_BYTE_ARRAY_INDEXED);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(REG_T8));

    cg.LW(reg_bytearray, ES_OFFSETOF(ES_Object, indexed_properties), reg_object);

    if (index)
    {
        cg.LW(reg_index, IVALUE_OFFSET(index), BASE_REGISTER(index));
        cg.Jump(slow_case, ES_NATIVE_CONDITION_LTZ(reg_index));
    }
    else
        cg.Load(reg_index, constant_index);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Byte_Array_Indexed, capacity), reg_bytearray);
    cg.JumpGE(slow_case, reg_index, REG_T8);

    if (known_value)
        cg.Load (REG_T8, static_cast<signed char>(*known_value));
    else
        cg.LW(REG_T8, IVALUE_OFFSET(source), BASE_REGISTER(source));

    cg.LW(reg_bytearray, ES_OFFSETOF(ES_Byte_Array_Indexed, byte_array), reg_bytearray);
    cg.Add(reg_bytearray, reg_index);
    cg.SB(REG_T8, 0, reg_bytearray);
}

static void
EmitTypedArrayPrecondition(ES_Native &n, VirtualRegister *object, VirtualRegister *index, unsigned constant_index)
{
    DECLARE_NOTHING();

    ES_CodeGenerator &cg = n.cg;

    const Register reg_bytearray = REG_S0;
    const Register reg_index = REG_S1;

    if (!n.current_slow_case)
        n.EmitSlowCaseCall();

    ES_CodeGenerator::JumpTarget *slow_case = n.current_slow_case->GetJumpTarget();

    LoadObjectOperand(n, REG_T9, object);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, object_bits), REG_T9);
    cg.And(REG_T8, ES_Object::MASK_TYPE_ARRAY_INDEXED);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(REG_T8));

    cg.LW(reg_bytearray, ES_OFFSETOF(ES_Object, indexed_properties), REG_T9);

    if (index)
    {
        cg.LW(reg_index, IVALUE_OFFSET(index), BASE_REGISTER(index));
        cg.Jump(slow_case, ES_NATIVE_CONDITION_LTZ(reg_index));
    }
    else
        cg.Load(reg_index, constant_index);
}

void
ES_Native::EmitInt32TypedArrayGet(VirtualRegister *target, VirtualRegister *object, VirtualRegister *index, unsigned constant_index, unsigned type_array_bits)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitInt32TypedArrayGet");

    const Register reg_bytearray = REG_S0;
    const Register reg_index = REG_S1;
    const Register reg_kind = REG_S2;

    EmitTypedArrayPrecondition(*this, object, index, constant_index);

    ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();

    cg.LW(REG_T8, ES_OFFSETOF(ES_Type_Array_Indexed, capacity), reg_bytearray);
    cg.JumpGE(slow_case, reg_index, REG_T8);

    cg.LW(reg_kind, ES_OFFSETOF(ES_Type_Array_Indexed, kind), reg_bytearray);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Type_Array_Indexed, byte_offset), reg_bytearray);
    cg.LW(reg_bytearray, ES_OFFSETOF(ES_Type_Array_Indexed, byte_array), reg_bytearray);
    cg.LW(reg_bytearray, ES_OFFSETOF(ES_Byte_Array_Indexed, byte_array), reg_bytearray);
    cg.Add(reg_bytearray, REG_T8);

    ES_CodeGenerator::JumpTarget *done_int_target = NULL;
    ES_CodeGenerator::JumpTarget *done_target = NULL;

    unsigned kind = ES_Type_Array_Indexed::Int8Array;
    unsigned handled = type_array_bits;

    if (type_array_bits & ES_Type_Array_Indexed::AnyIntArray)
        done_int_target = cg.ForwardJump();

    for(; handled; kind++)
    {
        ES_CodeGenerator::JumpTarget *next;

        if (!(handled & 1))
        {
            handled = handled >> 1;
            continue;
        }

        handled = handled >> 1;

        if (handled)
            next = cg.ForwardJump();
        else
            next = slow_case;

        cg.JumpNE(next, reg_kind, kind);

        switch (kind)
        {
        case ES_Type_Array_Indexed::Int8Array:
            ANNOTATE("EmitInt32TypedArrayGet, Int8Array");

            cg.Add(reg_bytearray, reg_index);
            cg.LB(REG_T8, 0, reg_bytearray);

            cg.Jump(done_int_target, ES_NATIVE_UNCONDITIONAL, TRUE);
            break;

        case ES_Type_Array_Indexed::Uint8Array:
        case ES_Type_Array_Indexed::Uint8ClampedArray:
            if (kind == ES_Type_Array_Indexed::Uint8Array)
                ANNOTATE("EmitInt32TypedArrayGet, Uint8Array");
            else
                ANNOTATE("EmitInt32TypedArrayGet, Uint8ClampedArray");

            cg.Add(reg_bytearray, reg_index);
            cg.LBU(REG_T8, 0, reg_bytearray);

            cg.Jump(done_int_target, ES_NATIVE_UNCONDITIONAL, TRUE);
            break;

        case ES_Type_Array_Indexed::Int16Array:
            ANNOTATE("EmitInt32TypedArrayGet, Int16Array");

            cg.SLL(reg_index, reg_index, 1);
            cg.Add(reg_bytearray, reg_index);
            cg.LH(REG_T8, 0, reg_bytearray);

            cg.Jump(done_int_target, ES_NATIVE_UNCONDITIONAL, TRUE);
            break;

        case ES_Type_Array_Indexed::Uint16Array:
            ANNOTATE("EmitInt32TypedArrayGet, Uint16Array");

            cg.SLL(reg_index, reg_index, 1);
            cg.Add(reg_bytearray, reg_index);
            cg.LHU(REG_T8, 0, reg_bytearray);

            cg.Jump(done_int_target, ES_NATIVE_UNCONDITIONAL, TRUE);
            break;

        case ES_Type_Array_Indexed::Int32Array:
            ANNOTATE("EmitInt32TypedArrayGet, Int32Array");

            cg.SLL(reg_index, reg_index, 2);
            cg.Add(reg_bytearray, reg_index);
            cg.LW(REG_T8, 0, reg_bytearray);

            cg.Jump(done_int_target, ES_NATIVE_UNCONDITIONAL, TRUE);
            break;

        case ES_Type_Array_Indexed::Uint32Array:
            ANNOTATE("EmitInt32TypedArrayGet, Uint32Array");

            cg.SLL(reg_index, reg_index, 2);
            cg.Add(reg_bytearray, reg_index);
            cg.LW(REG_T8, 0, reg_bytearray);

            done_target = cg.ForwardJump();

            cg.Jump(done_int_target, ES_NATIVE_CONDITION_GEZ(REG_T8));

            if (ARCHITECTURE_HAS_FPU())
            {
                cg.MTC1(REG_T8, REG_F30);
                cg.CVT(D, W, REG_F28, REG_F30);

                // 4294967296 as double is 0x41f0000000000000
                cg.LUI(REG_T9, 0x41f0);
                cg.MTC1(REG_Z, REG_F30);
                if (fpu_type == ES_CodeGenerator::FPU_DOUBLE)
                    cg.MTHC1(REG_T9, REG_F30);
                else
                    cg.MTC1(REG_T9, REG_F31);

                cg.ADD(D, REG_F30, REG_F30, REG_F28);
                cg.SDC1(REG_F30, REGISTER_OFFSET(target), BASE_REGISTER(target));
                cg.Jump(done_target, ES_NATIVE_UNCONDITIONAL, TRUE);
            }
            else
            {
                cg.Load(REG_T9, GetConst(CONST_ESN_STOREUINTASDOUBLE));
                cg.Move(REG_A0, REG_T8);
                cg.JALR(REG_T9);
                cg.Add(REG_A1, BASE_REGISTER(target), REGISTER_OFFSET(target));
                cg.Jump(done_target);
            }
            break;

        case ES_Type_Array_Indexed::Float32Array:
            ANNOTATE("EmitInt32TypedArrayGet, Float32Array");

            cg.SLL(reg_index, reg_index, 2);
            cg.Add(reg_bytearray, reg_index);

            if (ARCHITECTURE_HAS_FPU())
            {
                cg.LWC1(REG_F28, 0, reg_bytearray);
                cg.CVT(D, S, REG_F30, REG_F28);
                cg.SDC1(REG_F30, DVALUE_OFFSET(target), BASE_REGISTER(target));
            }
            else
            {
                cg.Load(REG_T9, GetConst(CONST_ESN_STOREFLOATASDOUBLE));
                cg.Move(REG_A0, reg_bytearray);
                cg.JALR(REG_T9);
                cg.Add(REG_A1, BASE_REGISTER(target), DVALUE_OFFSET(target));
            }

            if (handled || done_int_target)
            {
                if (!done_target)
                    done_target = cg.ForwardJump();
                cg.Jump(done_target, ES_NATIVE_UNCONDITIONAL, ARCHITECTURE_HAS_FPU());
            }
            else
                return;

            break;

        case ES_Type_Array_Indexed::Float64Array:
            ANNOTATE("EmitInt32TypedArrayGet, Float64Array");

            cg.SLL(reg_index, reg_index, 3);
            cg.Add(reg_bytearray, reg_index);
            CopyValue(cg, reg_bytearray, 0, target);

            if (done_int_target)
            {
                if (!done_target)
                    done_target = cg.ForwardJump();
                cg.Jump(done_target, ES_NATIVE_UNCONDITIONAL, TRUE);
            }
            else if (!done_target)
                return;
        }

        if (next != slow_case)
            cg.SetJumpTarget(next);
    }

    if (done_int_target)
    {
        cg.SetJumpTarget(done_int_target);

        cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));
        StoreValue(cg, target, REG_T8, REG_T9);
    }

    if (done_target)
        cg.SetJumpTarget(done_target);
}

static void
LoadDoubleAsInt(ES_Native &n, VirtualRegister *source, Register reg_dest, BOOL do_truncate, ES_CodeGenerator::JumpTarget *slow_case)
{
    ES_CodeGenerator &cg = n.cg;

    if (n.fpu_type == ES_CodeGenerator::FPU_NONE)
    {
        if (do_truncate)
            cg.Load(REG_T9, n.GetConst(CONST_ESN_GETDOUBLEASINT));
        else
            cg.Load(REG_T9, n.GetConst(CONST_ESN_GETDOUBLEASINTROUND));
        cg.JALR(REG_T9);
        cg.Add(REG_A0, BASE_REGISTER(source), DVALUE_OFFSET(source));
        if (reg_dest != REG_V0)
            cg.Move(reg_dest, REG_V0);
    }
    else
    {
        cg.LDC1(REG_F28, DVALUE_OFFSET(source), BASE_REGISTER(source));
        if (do_truncate)
            cg.TRUNC(W, D, REG_F30, REG_F28);
        else
            cg.ROUND(W, D, REG_F30, REG_F28);
        CheckFPUException(cg, slow_case);
        cg.MFC1(reg_dest, REG_F30);
    }
}

static void
StoreDoubleAsFloat2(ES_Native &n, VirtualRegister *source, Register reg_dest, unsigned offset)
{
    ES_CodeGenerator &cg = n.cg;

    if (n.fpu_type == ES_CodeGenerator::FPU_NONE)
    {
        cg.Add(REG_A0, BASE_REGISTER(source), DVALUE_OFFSET(source));
        cg.Load(REG_T9, n.GetConst(CONST_ESN_STOREDOUBLEASFLOAT));
        cg.JALR(REG_T9);
        cg.Add(REG_A1, reg_dest, offset);
    }
    else
    {
        cg.LDC1(REG_F28, DVALUE_OFFSET(source), BASE_REGISTER(source));
        cg.CVT(S, D, REG_F30, REG_F28);
        cg.SWC1(REG_F30, offset, reg_dest);
    }
}

static void
StoreIntAsFloat2(ES_Native &n, VirtualRegister *source, Register reg_dest, unsigned offset)
{
    ES_CodeGenerator &cg = n.cg;

    if (n.fpu_type == ES_CodeGenerator::FPU_NONE)
    {
        cg.LW(REG_A0, IVALUE_OFFSET(source), BASE_REGISTER(source));
        cg.Load(REG_T9, n.GetConst(CONST_ESN_STOREINTASFLOAT));
        cg.JALR(REG_T9);
        cg.Add(REG_A1, reg_dest, offset);
    }
    else
    {
        cg.LWC1(REG_F28, IVALUE_OFFSET(source), BASE_REGISTER(source));
        cg.CVT(S, W, REG_F30, REG_F28);
        cg.SWC1(REG_F30, offset, reg_dest);
    }
}

static void
StoreIntAsDouble2(ES_Native &n, VirtualRegister *source, Register reg_dest, unsigned offset)
{
    ES_CodeGenerator &cg = n.cg;

    if (n.fpu_type == ES_CodeGenerator::FPU_NONE)
    {
        cg.LW(REG_A0, IVALUE_OFFSET(source), BASE_REGISTER(source));
        cg.Load(REG_T9, n.GetConst(CONST_ESN_STOREINTASDOUBLE));
        cg.JALR(REG_T9);
        cg.Add(REG_A1, reg_dest, offset);
    }
    else
    {
        cg.LWC1(REG_F28, IVALUE_OFFSET(source), BASE_REGISTER(source));
        cg.CVT(D, W, REG_F30, REG_F28);
        cg.SDC1(REG_F30, offset, reg_dest);
    }
}

static BOOL
EmitInt32TypedArrayPutLoadSource(ES_Native &native, ES_Native::VirtualRegister *source_vr, unsigned char source_type_bits, ES_ValueType *known_type, BOOL do_truncate, ES_CodeGenerator::JumpTarget *slow_case)
{
    ES_CodeGenerator &cg = native.cg;
    if (known_type)
    {
        if (*known_type == ESTYPE_DOUBLE)
        {
            LoadDoubleAsInt(native, source_vr, REG_T8, do_truncate, slow_case);
            return TRUE;
        }
        else if (*known_type == ESTYPE_INT32)
            cg.LW(REG_T8, TYPE_OFFSET(source_vr), BASE_REGISTER(source_vr));
    }
    else
    {
        if ((source_type_bits & (ESTYPE_BITS_INT32 | ESTYPE_BITS_DOUBLE)) == (ESTYPE_BITS_INT32 | ESTYPE_BITS_DOUBLE))
        {
            ES_CodeGenerator::OutOfOrderBlock *double_to_int = cg.StartOutOfOrderBlock();
            ANNOTATE("EmitInt32TypedArrayPut, convert double to int");
            LoadDoubleAsInt(native, source_vr, REG_T8, do_truncate, slow_case);
            cg.EndOutOfOrderBlock();

            cg.LW(REG_T8, TYPE_OFFSET(source_vr), BASE_REGISTER(source_vr));
            cg.Load(REG_T9, native.GetConst(CONST_ESTYPE_INT32));

            cg.JumpLT(double_to_int->GetJumpTarget(), REG_T8, REG_T9);
            cg.JumpGT(slow_case, REG_T8, REG_T9);

            cg.LW(REG_T8, IVALUE_OFFSET(source_vr), BASE_REGISTER(source_vr));
            cg.SetOutOfOrderContinuationPoint(double_to_int);
            return TRUE;
        }
        else if (source_type_bits & ESTYPE_BITS_INT32)
        {
            native.EmitRegisterTypeCheck(source_vr, ESTYPE_INT32, slow_case);
            cg.LW(REG_T8, IVALUE_OFFSET(source_vr), BASE_REGISTER(source_vr));
        }
        else if (source_type_bits & ESTYPE_BITS_DOUBLE)
        {
            native.EmitRegisterTypeCheck(source_vr, ESTYPE_DOUBLE, slow_case);
            LoadDoubleAsInt(native, source_vr, REG_T8, do_truncate, slow_case);
            return TRUE;
        }
    }
    return FALSE;
}

void
ES_Native::EmitInt32TypedArrayPut(VirtualRegister *object, VirtualRegister *index, unsigned constant_index, unsigned type_array_bits, VirtualRegister *source, unsigned char source_type_bits, ES_Value_Internal *known_value, ES_ValueType *known_type)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitInt32TypedArrayPut");

    OP_ASSERT(!known_type || *known_type == ESTYPE_DOUBLE || *known_type == ESTYPE_INT32);
    OP_ASSERT((source_type_bits & ~(ESTYPE_BITS_INT32 | ESTYPE_BITS_DOUBLE)) == 0);

    const Register reg_bytearray = REG_S0;
    const Register reg_index = REG_S1;
    const Register reg_kind = REG_S2;

    EmitTypedArrayPrecondition(*this, object, index, constant_index);

    ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();

    cg.LW(REG_T8, ES_OFFSETOF(ES_Type_Array_Indexed, capacity), reg_bytearray);
    cg.JumpGE(slow_case, reg_index, REG_T8);

    cg.LW(reg_kind, ES_OFFSETOF(ES_Type_Array_Indexed, kind), reg_bytearray);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Type_Array_Indexed, byte_offset), reg_bytearray);
    cg.LW(reg_bytearray, ES_OFFSETOF(ES_Type_Array_Indexed, byte_array), reg_bytearray);
    cg.LW(reg_bytearray, ES_OFFSETOF(ES_Byte_Array_Indexed, byte_array), reg_bytearray);
    cg.Add(reg_bytearray, REG_T8);

    ES_CodeGenerator::JumpTarget *next = NULL;
    ES_CodeGenerator::JumpTarget *done_target = cg.ForwardJump();

    if (type_array_bits & ES_Type_Array_Indexed::AnyIntArray)
    {
        if (type_array_bits & ES_Type_Array_Indexed::AnyFloatArray)
            next = cg.ForwardJump();
        else
            next = slow_case;

        cg.JumpGT(next, reg_kind, ES_Type_Array_Indexed::Uint32Array);

        BOOL needed_double_conversion = FALSE;
        if (known_value)
        {
            ES_CodeGenerator::JumpTarget *done = NULL;
            BOOL loaded_known_value = FALSE;
            if (type_array_bits & (0x1 << ES_Type_Array_Indexed::Uint8ClampedArray))
            {
                ES_CodeGenerator::JumpTarget *not_clamped = NULL;
                if ((type_array_bits & ES_Type_Array_Indexed::AnyIntArray) != (0x1 << ES_Type_Array_Indexed::Uint8ClampedArray))
                {
                    not_clamped = cg.ForwardJump();
                    cg.JumpNE(not_clamped, reg_kind, ES_Type_Array_Indexed::Uint8ClampedArray);
                }
                cg.Load(REG_T8, known_value->AsNumber(context).GetNumAsUint8Clamped());
                if (not_clamped)
                {
                    done = cg.ForwardJump();
                    cg.Jump(done);
                    cg.SetJumpTarget(not_clamped);
                }
                else
                    loaded_known_value = TRUE;
            }
            if (!loaded_known_value)
                cg.Load(REG_T8, known_value->AsNumber(context).GetNumAsUInt32());
            if (done)
                cg.SetJumpTarget(done);
        }
        else
        {
            BOOL needs_truncation = (type_array_bits & ES_Type_Array_Indexed::AnyIntArray) != (0x1 << ES_Type_Array_Indexed::Uint8ClampedArray);
            needed_double_conversion = EmitInt32TypedArrayPutLoadSource(*this, source, source_type_bits, known_type, needs_truncation, slow_case);
        }

        ES_CodeGenerator::JumpTarget *next_int = NULL;

        if (type_array_bits & ES_Type_Array_Indexed::AnyInt8Array)
        {
            ANNOTATE("EmitInt32TypedArrayPut, (U)Int8Array");

            if (type_array_bits & (ES_Type_Array_Indexed::AnyInt16Array | ES_Type_Array_Indexed::AnyInt32Array))
                next_int = cg.ForwardJump();
            else
                next_int = next;

            if (type_array_bits & (0x1 << ES_Type_Array_Indexed::Uint8ClampedArray))
                cg.JumpGEU(next_int, reg_kind, ES_Type_Array_Indexed::Int16Array);
            else
                cg.JumpGEU(next_int, reg_kind, ES_Type_Array_Indexed::Uint8ClampedArray);

            if (!known_value && (type_array_bits & (0x1 << ES_Type_Array_Indexed::Uint8ClampedArray)))
            {
                ES_CodeGenerator::JumpTarget *write_value = cg.ForwardJump();
                cg.SLT(REG_T9, REG_T8, REG_Z);
                cg.JumpNE(write_value, reg_kind, ES_Type_Array_Indexed::Uint8ClampedArray);
                /* If source_vr has been converted from a double _and_ it wasn't only
                   at a clamped type, repeat the load, but with different rounding. */
                if (needed_double_conversion && ((type_array_bits & ES_Type_Array_Indexed::AnyIntArray) != (0x1 << ES_Type_Array_Indexed::Uint8ClampedArray)))
                    EmitInt32TypedArrayPutLoadSource(*this, source, source_type_bits, known_type, FALSE, slow_case);
                cg.MOVN(REG_T8, REG_Z, REG_T9);
                cg.Load(REG_AT, 0xff);
                cg.SLT(REG_T9, REG_AT, REG_T8);
                cg.MOVN(REG_T8, REG_AT, REG_T9);

                cg.SetJumpTarget(write_value);
            }
            cg.Add(reg_bytearray, reg_index);
            cg.SB(REG_T8, 0, reg_bytearray);
            cg.Jump(done_target, ES_NATIVE_UNCONDITIONAL, TRUE);
        }

        if (type_array_bits & ES_Type_Array_Indexed::AnyInt16Array)
        {
            ANNOTATE("EmitInt32TypedArrayPut, (U)Int16Array");

            if (next_int)
                cg.SetJumpTarget(next_int);

            if (type_array_bits & ES_Type_Array_Indexed::AnyInt32Array)
                next_int = cg.ForwardJump();
            else
                next_int = next;

            cg.Sub(REG_T9, reg_kind, ES_Type_Array_Indexed::Int16Array);
            cg.JumpGEU(next_int, REG_T9, 2);

            cg.SLL(reg_index, reg_index, 1);
            cg.Add(reg_bytearray, reg_index);
            cg.SH(REG_T8, 0, reg_bytearray);
            cg.Jump(done_target, ES_NATIVE_UNCONDITIONAL, TRUE);
        }

        if (type_array_bits & ES_Type_Array_Indexed::AnyInt32Array)
        {
            ANNOTATE("EmitInt32TypedArrayPut, (U)Int32Array");

            if (next_int)
                cg.SetJumpTarget(next_int);

            cg.Sub(REG_T9, reg_kind, ES_Type_Array_Indexed::Int32Array);
            cg.JumpGEU(next, REG_T9, 2);

            cg.SLL(reg_index, reg_index, 2);
            cg.Add(reg_bytearray, reg_index);
            cg.SW(REG_T8, 0, reg_bytearray);
            cg.Jump(done_target, ES_NATIVE_UNCONDITIONAL, TRUE);
        }
    }

    if (type_array_bits & ES_Type_Array_Indexed::Float32ArrayBit)
    {
        ANNOTATE("EmitInt32TypedArrayPut, Float32Array");

        if (next)
            cg.SetJumpTarget(next);

        if (type_array_bits & ES_Type_Array_Indexed::Float64ArrayBit)
            next = cg.ForwardJump();
        else
            next = slow_case;

        cg.JumpNE(next, reg_kind, ES_Type_Array_Indexed::Float32Array);

        cg.SLL(reg_index, reg_index, 2);
        cg.Add(reg_bytearray, reg_index);

        if (known_type)
        {
            if (*known_type == ESTYPE_DOUBLE)
                StoreDoubleAsFloat2(*this, source, reg_bytearray, 0);
            else if (*known_type == ESTYPE_INT32)
                StoreIntAsFloat2(*this, source, reg_bytearray, 0);
        }
        else
        {
            if ((source_type_bits & (ESTYPE_BITS_INT32 | ESTYPE_BITS_DOUBLE)) == (ESTYPE_BITS_INT32 | ESTYPE_BITS_DOUBLE))
            {
                ES_CodeGenerator::OutOfOrderBlock *double_to_float = cg.StartOutOfOrderBlock();
                ANNOTATE("EmitInt32TypedArrayPut, convert double to float");
                StoreDoubleAsFloat2(*this, source, reg_bytearray, 0);
                cg.EndOutOfOrderBlock();

                cg.LW(REG_T8, TYPE_OFFSET(source), BASE_REGISTER(source));
                cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));

                cg.JumpLT(double_to_float->GetJumpTarget(), REG_T8, REG_T9);
                cg.JumpGT(slow_case, REG_T8, REG_T9);

                StoreIntAsFloat2(*this, source, reg_bytearray, 0);
                cg.SetOutOfOrderContinuationPoint(double_to_float);
            }
            else if (source_type_bits & ESTYPE_BITS_INT32)
            {
                EmitRegisterTypeCheck(source, ESTYPE_INT32, slow_case);
                StoreIntAsFloat2(*this, source, reg_bytearray, 0);
            }
            else if (source_type_bits & ESTYPE_BITS_DOUBLE)
            {
                EmitRegisterTypeCheck(source, ESTYPE_DOUBLE, slow_case);
                StoreDoubleAsFloat2(*this, source, reg_bytearray, 0);
            }
        }

        cg.Jump(done_target, ES_NATIVE_UNCONDITIONAL);
    }

    if (type_array_bits & ES_Type_Array_Indexed::Float64ArrayBit)
    {
        ANNOTATE("EmitInt32TypedArrayPut, Float64Array");

        if (next)
            cg.SetJumpTarget(next);

        cg.JumpNE(slow_case, reg_kind, ES_Type_Array_Indexed::Float64Array);

        cg.SLL(reg_index, reg_index, 3);
        cg.Add(reg_bytearray, reg_index);

        if (known_type)
        {
            if (*known_type == ESTYPE_DOUBLE)
                CopyValue(cg, source, reg_bytearray, 0);
            else if (*known_type == ESTYPE_INT32)
                StoreIntAsDouble2(*this, source, reg_bytearray, 0);
        }
        else
        {
            if ((source_type_bits & (ESTYPE_BITS_INT32 | ESTYPE_BITS_DOUBLE)) == (ESTYPE_BITS_INT32 | ESTYPE_BITS_DOUBLE))
            {
                ES_CodeGenerator::OutOfOrderBlock *int_to_double = cg.StartOutOfOrderBlock();
                ANNOTATE("EmitInt32TypedArrayPut, convert int to double");
                StoreIntAsDouble2(*this, source, reg_bytearray, 0);
                cg.EndOutOfOrderBlock();

                cg.LW(REG_T8, TYPE_OFFSET(source), BASE_REGISTER(source));
                cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));

                cg.JumpEQ(int_to_double->GetJumpTarget(), REG_T8, REG_T9);
                cg.JumpGT(slow_case, REG_T8, REG_T9);

                CopyValue(cg, source, reg_bytearray, 0);
                cg.SetOutOfOrderContinuationPoint(int_to_double);
            }
            else if (source_type_bits & ESTYPE_BITS_INT32)
            {
                EmitRegisterTypeCheck(source, ESTYPE_INT32, slow_case);
                StoreIntAsDouble2(*this, source, reg_bytearray, 0);
            }
            else if (source_type_bits & ESTYPE_BITS_DOUBLE)
            {
                EmitRegisterTypeCheck(source, ESTYPE_DOUBLE, slow_case);
                CopyValue(cg, source, reg_bytearray, 0);
            }
        }
    }

    cg.SetJumpTarget(done_target);
}

void
ES_Native::EmitGetEnumerated(VirtualRegister *target, VirtualRegister *object, VirtualRegister *name)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitGetEnumerated");

    if (!current_slow_case)
        EmitSlowCaseCall();

    ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();

    const Register reg_name = REG_A0;
    const Register reg_object = REG_A1;
    const Register reg_enumerated_class_id = REG_A2;
    const Register reg_object_class_id = REG_A3;
    const Register reg_property_value = REG_S0;

    cg.LW(reg_name, IVALUE_OFFSET(name), BASE_REGISTER(name));

    LoadObjectOperand(*this, reg_object, object);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Execution_Context, enumerated_string), REG_S7);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_NE(REG_T8, reg_name));

    cg.LW(reg_enumerated_class_id, ES_OFFSETOF(ES_Execution_Context, enumerated_class_id), REG_S7);

    ES_CodeGenerator::OutOfOrderBlock *indexed = cg.StartOutOfOrderBlock();
    ANNOTATE("EmitGetEnumerated, indexed");

    const Register reg_index = REG_S1;
    const Register reg_indexed_properties = REG_S2;

    cg.Load(REG_T8, ES_Class::NOT_CACHED_CLASS_ID);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_NE(REG_T8, reg_enumerated_class_id));

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, object_bits), reg_object);
    cg.And(REG_T8, ES_Object::MASK_SIMPLE_COMPACT_INDEXED);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(REG_T8));

    cg.LW(reg_index, ES_OFFSETOF(ES_Execution_Context, enumerated_index), REG_S7);
    cg.LW(reg_indexed_properties, ES_OFFSETOF(ES_Object, indexed_properties), reg_object);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Compact_Indexed_Properties, capacity), reg_indexed_properties);
    cg.JumpGE(slow_case, reg_index, REG_T8);

    cg.SLL(REG_T8, reg_index, 3);
    cg.ADDU(reg_property_value, reg_indexed_properties, REG_T8);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Compact_Indexed_Properties, values) + ES_OFFSETOF(ES_Value_Internal, private_type), reg_property_value);
    cg.LW(REG_T9, ES_OFFSETOF(ES_Compact_Indexed_Properties, values) + ES_OFFSETOF(ES_Value_Internal, private_value), reg_property_value);

    ES_CodeGenerator::JumpTarget *store_jt = cg.ForwardJump();

    cg.Load(REG_AT, GetConst(CONST_ESTYPE_UNDEFINED));
    cg.Jump(store_jt, ES_NATIVE_CONDITION_NE(REG_T8, REG_AT));
    cg.Jump(slow_case, ES_NATIVE_CONDITION_EQZ(REG_T9));

    cg.SetJumpTarget(store_jt);

    cg.SW(REG_T8, TYPE_OFFSET(target), BASE_REGISTER(target));
    cg.SW(REG_T9, IVALUE_OFFSET(target), BASE_REGISTER(target));

    cg.EndOutOfOrderBlock();

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, klass), reg_object);
    cg.LW(reg_object_class_id, ES_OFFSETOF(ES_Class, class_id), REG_T8);
    cg.Jump(indexed->GetJumpTarget(), ES_NATIVE_CONDITION_NE(reg_enumerated_class_id, reg_object_class_id));

    const Register reg_enumerated_limit = REG_S1;
    const Register reg_enumerated_cached_type = REG_S2;
    const Register reg_enumerated_cached_offset = REG_S3;

    cg.LW(reg_enumerated_limit, ES_OFFSETOF(ES_Execution_Context, enumerated_limit), REG_S7);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, property_count), reg_object);
    cg.JumpLE(current_slow_case->GetJumpTarget(), REG_T8, reg_enumerated_limit);

    cg.LW(reg_enumerated_cached_type, ES_OFFSETOF(ES_Execution_Context, enumerated_cached_type), REG_S7);
    cg.LW(reg_enumerated_cached_offset, ES_OFFSETOF(ES_Execution_Context, enumerated_cached_offset), REG_S7);

    cg.LW(reg_property_value, ES_OFFSETOF(ES_Object, properties), reg_object);

    cg.Add(reg_property_value, reg_enumerated_cached_offset);

    CopyDataToValue(*this, reg_property_value, reg_enumerated_cached_type, target);

    cg.SetOutOfOrderContinuationPoint(indexed);
}

void
ES_Native::EmitPutEnumerated(VirtualRegister *object, VirtualRegister *name, VirtualRegister *source)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitPutEnumerated");

    if (!current_slow_case)
        EmitSlowCaseCall();

    ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();

    const Register reg_name = REG_A0;
    const Register reg_object = REG_A1;
    const Register reg_enumerated_class_id = REG_A2;
    const Register reg_object_class_id = REG_A3;
    const Register reg_property_value = REG_S0;
    const Register reg_enumerated_limit = REG_S1;
    const Register reg_enumerated_cached_type = REG_S2;
    const Register reg_enumerated_cached_offset = REG_S3;

    cg.LW(reg_name, IVALUE_OFFSET(name), BASE_REGISTER(name));

    LoadObjectOperand(*this, reg_object, object);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Execution_Context, enumerated_string), REG_S7);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_NE(REG_T8, reg_name));

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, klass), reg_object);
    cg.LW(reg_enumerated_class_id, ES_OFFSETOF(ES_Execution_Context, enumerated_class_id), REG_S7);
    cg.LW(reg_object_class_id, ES_OFFSETOF(ES_Class, class_id), REG_T8);

    cg.Jump(slow_case, ES_NATIVE_CONDITION_NE(reg_enumerated_class_id, reg_object_class_id));

    cg.LW(reg_enumerated_limit, ES_OFFSETOF(ES_Execution_Context, enumerated_limit), REG_S7);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, property_count), reg_object);
    cg.JumpLE(current_slow_case->GetJumpTarget(), REG_T8, reg_enumerated_limit);

    cg.LW(reg_enumerated_cached_type, ES_OFFSETOF(ES_Execution_Context, enumerated_cached_type), REG_S7);
    cg.LW(reg_enumerated_cached_offset, ES_OFFSETOF(ES_Execution_Context, enumerated_cached_offset), REG_S7);

    cg.LW(reg_property_value, ES_OFFSETOF(ES_Object, properties), reg_object);
    cg.Add(reg_property_value, reg_enumerated_cached_offset);

    CopyValueToData(*this, source, reg_property_value, 0, reg_enumerated_cached_type, current_slow_case->GetJumpTarget());
}

void
ES_Native::EmitInt32StringIndexedGet(VirtualRegister *target, VirtualRegister *object, VirtualRegister *index, unsigned constant_index, BOOL returnCharCode)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitInt32StringIndexedGet");

    if (!current_slow_case)
        EmitSlowCaseCall();

    ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();

    const Register reg_string = REG_A0;
    const Register reg_value = REG_A1;
    const Register reg_offset = REG_A2;
    const Register reg_length = REG_A3;
    const Register reg_character = REG_S0;

    LoadObjectOperand(*this, reg_string, object);

    cg.LW(reg_value, ES_OFFSETOF(JString, value), reg_string);

    cg.ANDI(REG_T8, reg_value, 1);
    cg.Jump(slow_case, ES_NATIVE_CONDITION_NEZ(REG_T8));

    cg.Add(reg_value, ES_OFFSETOF(JStringStorage, storage));

    cg.LW(reg_offset, ES_OFFSETOF(JString, offset), reg_string);
    cg.SLL(reg_offset, reg_offset, 1);

    cg.Add(reg_value, reg_offset);

    cg.LW(reg_length, ES_OFFSETOF(JString, length), reg_string);

    if (index || constant_index * 2 > SHRT_MAX)
    {
        if (index)
        {
            cg.LW(reg_offset, IVALUE_OFFSET(index), BASE_REGISTER(index));
            cg.Jump(slow_case, ES_NATIVE_CONDITION_LTZ(reg_offset));
        }
        else
            cg.Load(reg_offset, constant_index);

        cg.JumpGE(slow_case, reg_offset, reg_length);

        cg.SLL(reg_offset, reg_offset, 1);
        cg.Add(reg_value, reg_offset);

        cg.LHU(reg_character, 0, reg_value);
    }
    else
    {
        cg.JumpLE(slow_case, reg_length, constant_index);
        cg.LHU(reg_character, constant_index * 2, reg_value);
    }

    if (returnCharCode)
        cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));
    else
    {
        cg.JumpGE(slow_case, reg_character, STRING_NUMCHARS);

        cg.Load(REG_T9, GetConst(CONST_ESTYPE_STRING));

        cg.LW(reg_string, ES_OFFSETOF(ES_Execution_Context, rt_data), REG_S7);

        cg.SLL(REG_T8, reg_character, 2);
        cg.Add(reg_string, REG_T8);

        OP_ASSERT(ES_OFFSETOF(ESRT_Data, strings) + STRING_nul * sizeof(void *) < SHRT_MAX);
        cg.LW(reg_character, ES_OFFSETOF(ESRT_Data, strings) + STRING_nul * sizeof(void *), reg_string);
    }

    cg.SW(reg_character, IVALUE_OFFSET(target), BASE_REGISTER(target));
    cg.SW(REG_T9, TYPE_OFFSET(target), BASE_REGISTER(target));
}

void
ES_Native::EmitPrimitiveNamedGet(VirtualRegister *target, ES_Object *check_object, ES_Object *property_object, unsigned class_id, unsigned property_offset, ES_StorageType storage_type, ES_Object *function)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitPrimitiveNamedGet");

    const Register reg_object = REG_S0;
    const Register reg_object_class = REG_S1;
    const Register reg_class_id = REG_S2;
    const Register reg_value = VALUE_TRANSFER_REGISTER;

    cg.Load(reg_object, check_object);
    cg.LW(reg_object_class, ES_OFFSETOF(ES_Object, klass), reg_object);
    cg.LW(reg_class_id, ES_OFFSETOF(ES_Class, class_id), reg_object_class);
    cg.JumpNE(current_slow_case->GetJumpTarget(), reg_class_id, class_id);

    if (function)
    {
        cg.Load(REG_T8, function);
        cg.Load(REG_T9, GetConst(CONST_ESTYPE_OBJECT));
        StoreValue(cg, target, REG_T8, REG_T9);
    }
    else if (property_object)
    {
        if (check_object != property_object)
            cg.Load(reg_object, property_object);

        cg.LW(reg_value, ES_OFFSETOF(ES_Object, properties), reg_object);

        if (property_value_write_vr == target && (storage_type == ES_STORAGE_WHATEVER || storage_type == ES_STORAGE_OBJECT))
        {
            SetPropertyValueTransferRegister(VALUE_TRANSFER_NATIVE_REGISTER, property_offset, storage_type != ES_STORAGE_OBJECT);
            if (current_slow_case)
            {
                ES_CodeGenerator::OutOfOrderBlock *recover_from_failure = RecoverFromFailedPropertyValueTransfer(*this, target, VALUE_TRANSFER_REGISTER, storage_type);
                if (recover_from_failure)
                    cg.SetOutOfOrderContinuationPoint(recover_from_failure);
            }
        }
        else
            CopyTypedDataToValue(*this, reg_value, property_offset, storage_type, target);
    }
    else
        EmitSetRegisterType(target, ESTYPE_UNDEFINED);
}

void
ES_Native::EmitFetchFunction(VirtualRegister *target_vr, ES_Object *function, VirtualRegister *object_vr, unsigned class_id, BOOL fetch_value)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitFetchFunction");

    const Register reg_object = REG_S0;
    const Register reg_object_class = REG_S1;
    const Register reg_class_id = REG_S2;

    LoadObjectOperand(*this, reg_object, object_vr);

    cg.LW(reg_object_class, ES_OFFSETOF(ES_Object, klass), reg_object);
    cg.LW(reg_class_id, ES_OFFSETOF(ES_Class, class_id), reg_object_class);
    cg.JumpNE(current_slow_case->GetJumpTarget(), reg_class_id, class_id);

    if (fetch_value)
    {
        cg.Load(REG_T8, function);
        cg.Load(REG_T9, GetConst(CONST_ESTYPE_OBJECT));
        StoreValue(cg, target_vr, REG_T8, REG_T9);
    }
}


static void
EmitGetCachePositiveBranch(ES_CodeGenerator &cg, const ES_Native::GetCacheGroup &group, const ES_Native::GetCacheGroup::Entry &entry, ES_CodeGenerator::JumpTarget **copy_value, Register reg_object, Register reg_value, BOOL properties_loaded, BOOL jump_when_finished)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitGetCachePositiveBranch");

    OP_ASSERT(entry.positive);

    if (!properties_loaded)
        cg.LW(reg_value, ES_OFFSETOF(ES_Object, properties), reg_object);

    if (!group.single_offset && entry.positive_offset)
        cg.Add(reg_value, entry.positive_offset);

    if (jump_when_finished)
        cg.Jump(copy_value[group.storage_type]);
}

static const ES_Native::GetCacheGroup *
FindGroupByStorageType(const ES_Native::GetCacheGroup *groups, ES_StorageType storage_type)
{
    while (groups->storage_type != storage_type)
        ++groups;
    return groups;
}

static void
EmitGetCacheNegativeBranch(ES_CodeGenerator &cg, const ES_Native::GetCacheGroup *groups, const ES_Native::GetCacheGroup &group, const ES_Native::GetCacheGroup::Entry &entry, ES_CodeGenerator::JumpTarget **copy_value, Register reg_object, Register reg_value, BOOL jump_when_finished)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitGetCacheNegativeBranch");

    OP_ASSERT(entry.prototype);

    const ES_Native::GetCacheGroup *actual_group;
    if (entry.prototype_storage_type == group.storage_type)
        actual_group = &group;
    else
    {
        actual_group = FindGroupByStorageType(groups, entry.prototype_storage_type);
        jump_when_finished = TRUE;
    }

    cg.Load(reg_object, entry.prototype);
    cg.LW(reg_value, ES_OFFSETOF(ES_Object, properties), reg_object);

    if (!actual_group->single_offset && entry.prototype_offset)
        cg.Add(reg_value, entry.prototype_offset);

    if (jump_when_finished)
        cg.Jump(copy_value[actual_group->storage_type]);
}

void
ES_Native::EmitNamedGet(VirtualRegister *target, VirtualRegister *object, const GetCacheAnalysis &analysis, BOOL for_inlined_function_call, BOOL fetch_value)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitNamedGet");

    const Register reg_object = REG_S0;
    const Register reg_object_class = REG_S1;
    const Register reg_class_id = REG_S2;
    const Register reg_property_count = REG_S3;
    Register reg_value;

    const GetCacheGroup *groups = analysis.groups;
    unsigned groups_count = analysis.groups_count;
    const GetCacheNegative *negatives = analysis.negatives;
    unsigned negatives_count = analysis.negatives_count;

    /* Storage type indexed array of jump targets for jumping to the code that
        actually copies the value from a certain storage type.  Only set for
        those storage types we actually handle.  */
    ES_CodeGenerator::JumpTarget *copy_value[FIRST_SPECIAL_TYPE] = { NULL };

    BOOL load_properties = FALSE;
    unsigned positive_count = 0;

    for (unsigned group_index = 0; group_index < groups_count; ++group_index)
    {
        const GetCacheGroup &group = groups[group_index];

        if (!load_properties)
            for (unsigned entry_index = 0; !load_properties && entry_index < group.entries_count; ++entry_index)
                if (group.entries[entry_index].positive && ++positive_count > 1)
                    load_properties = TRUE;

        if (!copy_value[group.storage_type])
            copy_value[group.storage_type] = cg.ForwardJump();
    }

    if (property_value_read_vr)
        reg_value = REG_T6;
    else
        reg_value = VALUE_TRANSFER_REGISTER;

    ES_CodeGenerator::JumpTarget *jt_not_found = negatives_count ? cg.ForwardJump() : NULL;
    ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();
    ES_CodeGenerator::JumpTarget *jt_finished = cg.ForwardJump();
    ES_CodeGenerator::JumpTarget *jt_next_group = NULL;

    LoadObjectOperand(*this, reg_object, object);

    property_value_nr = NULL;

    cg.LW(reg_object_class, ES_OFFSETOF(ES_Object, klass), reg_object);
    cg.LW(reg_class_id, ES_OFFSETOF(ES_Class, class_id), reg_object_class);

    if (load_properties)
        cg.LW(reg_value, ES_OFFSETOF(ES_Object, properties), reg_object);

    ES_CodeGenerator::OutOfOrderBlock *recover_from_failure = NULL;
    BOOL property_value_transfer = property_value_write_vr == target;

    if (property_value_transfer)
    {
        SetPropertyValueTransferRegister(VALUE_TRANSFER_NATIVE_REGISTER, property_value_write_offset, FALSE);
        recover_from_failure = RecoverFromFailedPropertyValueTransfer(*this, target, VALUE_TRANSFER_REGISTER, ES_STORAGE_OBJECT);
    }

    /* Somewhat silly, but makes it easier to read the code below.  :-) */
    BOOL properties_loaded = load_properties;

    for (unsigned group_index = 0; group_index < groups_count; ++group_index)
    {
        const GetCacheGroup &group = groups[group_index];
        BOOL last_group = group_index == groups_count - 1;
        BOOL last_group_with_entries = last_group || groups[group_index + 1].only_secondary_entries;

        if (!jt_next_group)
            jt_next_group = last_group_with_entries && !negatives_count ? slow_case : cg.ForwardJump();

        unsigned entry_index = 0;

        while (entry_index < group.entries_count)
        {
            const GetCacheGroup::Entry &entry = group.entries[entry_index];
            if (entry.prototype && entry.prototype_secondary_entry)
            {
                OP_ASSERT(!entry.positive && !entry.negative);
                ++entry_index;
            }
            else
                break;
        }

        for (unsigned index = entry_index; index < group.entries_count; ++index)
            if (group.entries[index].limit != UINT_MAX)
            {
                cg.LW(reg_property_count, ES_OFFSETOF(ES_Object, property_count), reg_object);
                break;
            }

        for (; entry_index < group.entries_count; ++entry_index)
        {
            const GetCacheGroup::Entry &entry = group.entries[entry_index];

            BOOL last_entry = entry_index == group.entries_count - 1;
            ES_CodeGenerator::JumpTarget *jt_next_entry = NULL;

            BOOL need_positive_cache_hit_code = FALSE;
            BOOL need_negative_cache_hit_code = FALSE;
            BOOL need_limit_check = FALSE;

            if (entry.positive)
            {
                if (!properties_loaded)
                    /* Need to load properties pointer. */
                    need_positive_cache_hit_code = TRUE;

                if (!group.single_offset && entry.positive_offset)
                    /* Need to add offset to 'properties'. */
                    need_positive_cache_hit_code = TRUE;
            }

            if (entry.limit != UINT_MAX)
                /* Need limit check. */
                need_limit_check = TRUE;

            if (entry.prototype)
            {
                /* All secondary entries should precede any regular entries, so
                    the loop above should have skipped them all. */
                OP_ASSERT(!entry.prototype_secondary_entry);

                /* Need to fetch properties pointer from prototype object. */
                need_negative_cache_hit_code = TRUE;
            }

            if (!need_positive_cache_hit_code && !need_negative_cache_hit_code && !need_limit_check)
                /* The class ID check is all we need. */
                if (last_entry)
                    /* This is the last entry; jump to next group, or slow-case,
                        if class ID check failed, and fall through to the value
                        copying code if it succeeded. */
                    cg.JumpNE(jt_next_group, reg_class_id, entry.class_id);
                else
                    /* Otherwise jump to the value copying code if the class ID
                        check succeeded, and fall through to the next entry if it
                        failed. */
                    cg.JumpEQ(copy_value[group.storage_type], reg_class_id, entry.class_id);
            else
            {
                /* We need to execute one or more instructions after a
                    successful class ID check, so first jump to the next entry,
                    or next group (or slow-case,) if it failed. */
                if (last_entry)
                    cg.JumpNE(jt_next_group, reg_class_id, entry.class_id);
                else
                    cg.JumpNE(jt_next_entry = cg.ForwardJump(), reg_class_id, entry.class_id);

                if (need_limit_check)
                {
                    /* Perform limit check. */

                    if (!need_negative_cache_hit_code)
                    {
                        /* This cache either has no negative side at all, or
                            doesn't need to execute any instructions if the limit
                            check fails. */
                        ES_CodeGenerator::JumpTarget *jt_negative;
                        if (entry.negative)
                        {
                            /* Failed limit check => property doesn't exist. */
                            if (!jt_not_found)
                                jt_not_found = cg.ForwardJump();
                            jt_negative = jt_not_found;
                        }
                        else
                            /* Failed limit check => cache entry not valid, and
                                since we've done a class ID check, we know no
                                other cache entry is valid either. */
                            jt_negative = slow_case;

                        if (!need_positive_cache_hit_code && !last_entry)
                        {
                            cg.JumpGT(copy_value[group.storage_type], reg_property_count, entry.limit);
                            cg.Jump(jt_negative);
                        }
                        else
                        {
                            cg.JumpLE(jt_negative, reg_property_count, entry.limit);
                            EmitGetCachePositiveBranch(cg, group, entry, copy_value, reg_object, reg_value, properties_loaded, !last_entry);
                        }
                    }
                    else if (!need_positive_cache_hit_code)
                    {
                        /* This cache either has no positive side at all, or
                            doesn't need to execute any additional instructions
                            if the limit check succeeded, but do if the limit
                            check failed. */
                        ES_CodeGenerator::JumpTarget *jt_positive;
                        if (entry.positive)
                            /* Successful limit check => copy value (at single
                                offset, or zero offset.) */
                            jt_positive = copy_value[group.storage_type];
                        else
                            /* "Successful" limit check => cache entry not
                                valid, and since we've done a class ID check, we
                                know no other cache entry is valid either. */
                            jt_positive = slow_case;

                        cg.JumpGT(jt_positive, reg_property_count, entry.limit);

                        EmitGetCacheNegativeBranch(cg, groups, group, entry, copy_value, reg_object, reg_value, !last_entry);
                    }
                    else
                    {
                        /* We need to execute some additional instruction both
                            on successful and failed limit check.  Generate jump
                            that skips the "negative branch" if the limit check
                            fails. */
                        ES_CodeGenerator::JumpTarget *jt_positive = cg.ForwardJump();
                        cg.JumpGT(jt_positive, reg_property_count, entry.limit);

                        EmitGetCacheNegativeBranch(cg, groups, group, entry, copy_value, reg_object, reg_value, TRUE);

                        cg.SetJumpTarget(jt_positive);

                        EmitGetCachePositiveBranch(cg, group, entry, copy_value, reg_object, reg_value, properties_loaded, !last_entry);
                    }
                }
                else
                {
                    /* No limit check needed, but some additional instructions
                        need to be executed. */

                    /* Since we'll have no limit check, we must have exactly one
                        branch of additional instructions. */
                    OP_ASSERT(need_positive_cache_hit_code != need_negative_cache_hit_code);

                    if (need_positive_cache_hit_code)
                        EmitGetCachePositiveBranch(cg, group, entry, copy_value, reg_object, reg_value, properties_loaded, !last_entry);
                    else
                        EmitGetCacheNegativeBranch(cg, groups, group, entry, copy_value, reg_object, reg_value, !last_entry);
                }

                if (jt_next_entry)
                    cg.SetJumpTarget(jt_next_entry);
            }
        }

        unsigned offset_immediate = group.single_offset ? group.common_offset : 0;

        cg.SetJumpTarget(copy_value[group.storage_type]);

        if (property_value_transfer)
        {
            if (group.storage_type == ES_STORAGE_WHATEVER)
            {
                cg.Load(REG_T8, offset_immediate + VALUE_TYPE_OFFSET, reg_value);
                cg.Load(REG_T9, GetConst(CONST_ESTYPE_OBJECT));
                cg.JumpNE(slow_case, REG_T8, REG_T9);
                offset_immediate += VALUE_IVALUE_OFFSET;
            }
            else if (group.storage_type == ES_STORAGE_OBJECT_OR_NULL)
            {
                cg.Load(REG_T8, offset_immediate, reg_value);
                cg.JumpEQ(slow_case, REG_T8, REG_Z);
            }

            if (offset_immediate != property_value_offset)
                cg.Add(VALUE_TRANSFER_REGISTER, reg_value, offset_immediate);
            else if (reg_value != VALUE_TRANSFER_REGISTER)
                cg.Move(VALUE_TRANSFER_REGISTER, reg_value);
        }
        else if (fetch_value)
            CopyTypedDataToValue(*this, reg_value, offset_immediate, group.storage_type, target);

        if (!last_group || jt_not_found)
            cg.Jump(jt_finished);

        if (!last_group_with_entries)
        {
            cg.SetJumpTarget(jt_next_group);
            jt_next_group = NULL;
        }
    }

    if (negatives_count)
    {
        if (jt_next_group)
            cg.SetJumpTarget(jt_next_group);

        for (unsigned index = 0; index < negatives_count; ++index)
            if (negatives[index].limit != UINT_MAX)
            {
                cg.LW(reg_property_count, ES_OFFSETOF(ES_Object, property_count), reg_object);
                break;
            }

        for (unsigned index = 0; index < negatives_count; ++index)
        {
            BOOL last_missing = index == negatives_count - 1;

            if (negatives[index].limit == UINT_MAX)
                if (last_missing)
                    cg.JumpNE(slow_case, reg_class_id, negatives[index].class_id);
                else
                    cg.JumpEQ(jt_not_found, reg_class_id, negatives[index].class_id);
            else
            {
                ES_CodeGenerator::JumpTarget *jt_next = NULL;

                if (last_missing)
                    jt_next = slow_case;
                else
                    jt_next = cg.ForwardJump();

                cg.JumpNE(jt_next, reg_class_id, negatives[index].class_id);

                if (last_missing)
                    cg.JumpGT(slow_case, reg_property_count, negatives[index].limit);
                else
                {
                    cg.JumpLE(jt_not_found, reg_property_count, negatives[index].limit);
                    cg.Jump(slow_case);

                    cg.SetJumpTarget(jt_next);
                }
            }
        }
    }

    if (jt_not_found)
    {
        cg.SetJumpTarget(jt_not_found);
        EmitSetRegisterType(target, ESTYPE_UNDEFINED);
    }

    cg.SetJumpTarget(jt_finished);

    if (!property_value_nr)
        property_value_write_vr = NULL;

    if (recover_from_failure)
        cg.SetOutOfOrderContinuationPoint(recover_from_failure);
}

static void
ES_EmitPropertyArraySizeCheck(ES_CodeGenerator &cg, ES_CodeGenerator::Register properties, unsigned size, ES_CodeGenerator::Register scratch, ES_CodeGenerator::JumpTarget *failure)
{
    ES_DECLARE_NOTHING();

    /* Offset from 'properties' to the size part of ES_Header::header. */
    int objectheader_offset = -static_cast<int>(sizeof(ES_Header));

    cg.LW(scratch, objectheader_offset, properties);
    cg.SRL(scratch, scratch, ES_Header::SIZE_SHIFT);

    if (size >= LARGE_OBJECT_LIMIT)
    {
        /* If the object's size is not 0xffff, then it's not allocated as a
           large object, meaning its size is smaller than LARGE_OBJECT_LIMIT and
           thus smaller than 'size'. */
        cg.JumpNE(failure, scratch, 0xffff);

        /* Offset from 'properties' to ES_PageHeader::limit. */
        int pagelimit_offset = -static_cast<int>(sizeof(ES_Header)) - ES_PageHeader::HeaderSize() + ES_OFFSETOF(ES_PageHeader, limit);

        /* The object must be allocated as a large object, so calculate the
           page's actual size and use that as the size of the object. */
        cg.LW(scratch, pagelimit_offset, properties);
        cg.Sub(scratch, properties);
        cg.Add(scratch, static_cast<int>(sizeof(ES_Header)));
    }

    /* Check that the object's size is at least 'size'. */
    cg.JumpLT(failure, scratch, size);
}

void
ES_Native::EmitNamedPut(VirtualRegister *object, VirtualRegister *source, JString *name, BOOL has_complex_cache, ES_StorageType source_type)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitNamedPut");

    const Register reg_object = REG_S0;
    const Register reg_object_class = REG_S1;
    const Register reg_class_id = REG_S2;
    const Register reg_source = REG_S3;
    const Register reg_properties = REG_S4;

    unsigned constant_cached_offset = 0;
    ES_StorageType constant_cached_type = ES_STORAGE_UNDEFINED;

    LoadObjectOperand(*this, reg_object, object);

    ES_Code::PropertyCache *cache = &code->property_put_caches[CurrentCodeWord()[4].index];

    OP_ASSERT(cache->class_id != ES_Class::NOT_CACHED_CLASS_ID);

#ifdef SUPPORT_INLINE_ALLOCATION
    if (constructor_final_class && object_vr->index == 0)
    {
        CopyValue(cg, source, reg_properties, cache->cached_index, reg_object, reg_object_class);
        return;
    }
#endif // SUPPORT_INLINE_ALLOCATION

    if (!current_slow_case)
        EmitSlowCaseCall();

    cg.LW(reg_properties, ES_OFFSETOF(ES_Object, properties), reg_object);

    ES_CodeGenerator::JumpTarget *slow_case;

    if (!is_light_weight && property_value_read_vr && property_value_nr)
    {
        ES_CodeGenerator::OutOfOrderBlock *flush_object_vr = cg.StartOutOfOrderBlock();

        cg.Load(REG_T9, GetConst(CONST_ESTYPE_OBJECT));
        StoreValue(cg, object, reg_object, REG_T9);

        cg.Jump(current_slow_case_main);
        cg.EndOutOfOrderBlock(FALSE);

        slow_case = flush_object_vr->GetJumpTarget();
    }
    else
        slow_case = current_slow_case->GetJumpTarget();

    cg.LW(reg_object_class, ES_OFFSETOF(ES_Object, klass), reg_object);
    cg.LW(reg_class_id, ES_OFFSETOF(ES_Class, class_id), reg_object_class);

    while (!ES_Code::IsSimplePropertyCache(cache, FALSE))
        cache = cache->next;

    ES_CodeGenerator::JumpTarget *size_4_target = NULL, *size_8_target = NULL, *null_target = NULL, *int_to_double_target = NULL;

    cg.Add(reg_source, BASE_REGISTER(source), REGISTER_OFFSET(source));

    if (has_complex_cache)
    {
        unsigned cache_size = 0;
        ES_CodeGenerator::JumpTarget *new_class_slow_case = NULL;

        while (TRUE)
        {
            ES_Code::PropertyCache *next_cache = cache_size++ < MAX_PROPERTY_CACHE_SIZE ? cache->next : NULL;

            while (next_cache && (!ES_Code::IsSimplePropertyCache(next_cache, FALSE) || cache->class_id == next_cache->class_id))
                next_cache = next_cache->next;

            unsigned current_id = cache->class_id;

            ES_CodeGenerator::JumpTarget *jt_next_unpaired = next_cache ? cg.ForwardJump() : slow_case;

            cg.JumpNE(jt_next_unpaired, reg_class_id, current_id);

            ES_CodeGenerator::JumpTarget *jt_next = NULL;

            while (cache->class_id == current_id)
            {
                ES_StorageType type = cache->GetStorageType();
                ES_CodeGenerator::JumpTarget *current_slow_case = slow_case;
                jt_next = cache->next != next_cache && next_cache ? cg.ForwardJump() : NULL;

                BOOL needs_limit_check = cache->object_class->NeedLimitCheck(cache->GetLimit(), cache->data.new_class != NULL);
                unsigned size = ES_Layout_Info::SizeOf(type);

                if (needs_limit_check)
                {
                    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, property_count), reg_object);
                    cg.Load(REG_T9, cache->GetLimit());

                    /* The cache limit checks are as follows:

                    In both cases the stored cache limit is the index that we wrote the property to originally.

                    For the case where cache->data.new_class is not NULL, we are appending a new property
                    which means that the stored index must at this point be equal to the property count, and
                    if it isn't the cache is invalid.

                    If cache->data.new_class is NULL we are writing into an object, and thus for the cache to
                    be valid, the cache limit must be less than the property count, i.e. if the property count
                    is less than or equal to the cache limit the cache is invalid.
                    */

                    if (cache->data.new_class)
                        cg.JumpNE(jt_next ? jt_next : slow_case, REG_T8, REG_T9);
                    else
                        cg.JumpLE(jt_next ? jt_next : slow_case, REG_T8, REG_T9);
                }

                if (cache->data.new_class)
                {
                    OP_ASSERT(!cache->data.new_class->HasHashTableProperties());

                    ES_EmitPropertyArraySizeCheck(cg, reg_properties, cache->GetOffset() + size + sizeof(ES_Header), REG_T8, slow_case);

                    if (cache->data.new_class != cache->object_class)
                    {
                        cg.Load(REG_T8, cache->data.new_class);
                        cg.SW(REG_T8, ES_OFFSETOF(ES_Object, klass), reg_object);
                    }

                    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, property_count), reg_object);
                    cg.Add(REG_T8, 1);
                    cg.SW(REG_T8, ES_OFFSETOF(ES_Object, property_count), reg_object);

                    if (!new_class_slow_case)
                    {
                        ES_CodeGenerator::OutOfOrderBlock *revert_put_cached_new = cg.StartOutOfOrderBlock();
                        cg.SW(reg_object_class, ES_OFFSETOF(ES_Object, klass), reg_object);
                        cg.LW(REG_T8, ES_OFFSETOF(ES_Object, property_count), reg_object);
                        cg.Sub(REG_T8, 1);
                        cg.SW(REG_T8, ES_OFFSETOF(ES_Object, property_count), reg_object);
                        cg.Jump(slow_case);
                        cg.EndOutOfOrderBlock(FALSE);
                        new_class_slow_case = revert_put_cached_new->GetJumpTarget();
                    }

                    current_slow_case = new_class_slow_case;

                }
                if (cache->GetOffset() != 0)
                    cg.Add(reg_properties, cache->GetOffset());

                BOOL skip_write = FALSE;
                if (type != ES_STORAGE_WHATEVER && source_type != type)
                    skip_write = EmitTypeConversionCheck(type, source_type, source, current_slow_case, null_target, int_to_double_target);

                if (type == ES_STORAGE_NULL)
                {
                    if (!null_target)
                        null_target = cg.ForwardJump();

                    cg.Jump(null_target);
                }
                else if (!skip_write)
                    JumpToSize(cg, size, size_4_target, size_8_target);

                if (jt_next)
                    cg.SetJumpTarget(jt_next);

                if (cache->next)
                    cache = cache->next;
                else
                {
                    OP_ASSERT(next_cache == NULL);
                    break;
                }
            }

            if (next_cache)
            {
                cg.SetJumpTarget(jt_next_unpaired);
                cache = next_cache;
            }
            else
                break;
        }

        if (size_4_target || size_8_target)
            CopyValueToData2(cg, reg_source, reg_properties, constant_cached_offset, size_4_target, size_8_target);
    }
    else
    {
        cg.Load(REG_T8, cache->class_id);
        cg.JumpNE(slow_case, reg_class_id, REG_T8);

        constant_cached_offset = cache->GetOffset();
        constant_cached_type = cache->GetStorageType();

        if (cache->data.new_class)
        {
            OP_ASSERT(!cache->data.new_class->HasHashTableProperties());
            if (cache->object_class->NeedLimitCheck(cache->GetLimit(), TRUE))
            {
                cg.LW(REG_T8, ES_OFFSETOF(ES_Object, property_count), reg_object);
                cg.Load(REG_T9, cache->GetLimit());
                cg.JumpNE(slow_case, REG_T8, REG_T9);
            }

            ES_EmitPropertyArraySizeCheck(cg, reg_properties, constant_cached_offset + ES_Layout_Info::SizeOf(constant_cached_type) + sizeof(ES_Header), REG_T8, slow_case);

            if (cache->data.new_class != cache->object_class)
            {
                cg.Load(REG_T8, cache->data.new_class);
                cg.SW(REG_T8, ES_OFFSETOF(ES_Object, klass), reg_object);
            }
            cg.LW(REG_T8, ES_OFFSETOF(ES_Object, property_count), reg_object);
            cg.Add(REG_T8, 1);
            cg.SW(REG_T8, ES_OFFSETOF(ES_Object, property_count), reg_object);

            ES_CodeGenerator::OutOfOrderBlock *revert_put_cached_new = cg.StartOutOfOrderBlock();
            if (cache->data.new_class != cache->object_class)
                cg.SW(reg_object_class, ES_OFFSETOF(ES_Object, klass), reg_object);
            cg.LW(REG_T8, ES_OFFSETOF(ES_Object, property_count), reg_object);
            cg.Sub(REG_T8, 1);
            cg.SW(REG_T8, ES_OFFSETOF(ES_Object, property_count), reg_object);
            cg.Jump(slow_case);
            cg.EndOutOfOrderBlock(FALSE);
            slow_case = revert_put_cached_new->GetJumpTarget();
        }
        else if (cache->object_class->NeedLimitCheck(cache->GetLimit(), FALSE))
        {
            cg.LW(REG_T8, ES_OFFSETOF(ES_Object, property_count), reg_object);
            cg.JumpLE(slow_case, REG_T8, cache->GetLimit());
        }

        BOOL skip_write = FALSE;

        if (constant_cached_type != ES_STORAGE_WHATEVER && source_type != constant_cached_type)
            skip_write = EmitTypeConversionCheck(constant_cached_type, source_type, source, slow_case, null_target, int_to_double_target);

        if (!skip_write)
            CopyValueToTypedData(*this, source, reg_properties, constant_cached_offset, constant_cached_type, NULL);
    }

    EmitTypeConversionHandlers(source, reg_properties, constant_cached_offset, null_target, int_to_double_target);
}

void
ES_Native::EmitLengthGet(VirtualRegister *target, VirtualRegister *object, unsigned handled, unsigned possible, ES_CodeGenerator::JumpTarget *slow_case)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitLengthGet");

    ES_CodeGenerator::OutOfOrderBlock *handle_object = NULL;

    if (handled & ESTYPE_BITS_OBJECT)
    {
        if (!slow_case)
        {
            if (!current_slow_case)
                EmitSlowCaseCall();
            slow_case = current_slow_case->GetJumpTarget();
        }

        if (handled & ESTYPE_BITS_STRING)
            handle_object = cg.StartOutOfOrderBlock();

        if (handled != possible)
            EmitRegisterTypeCheck(object, ESTYPE_OBJECT, slow_case);

        const Register reg_array = REG_S0;
        const Register reg_class = REG_S1;
        const Register reg_properties = REG_S2;
        const Register reg_class_id = REG_S3;

        LoadObjectOperand(*this, reg_array, object);

        cg.LW(reg_class, ES_OFFSETOF(ES_Object, klass), reg_array);
        cg.LW(reg_class_id, ES_OFFSETOF(ES_Class, class_id), reg_class);
        cg.Load(REG_T8, code->global_object->GetArrayClass()->GetId(context));
        cg.JumpNE(slow_case, reg_class_id, REG_T8);
        cg.LW(reg_properties, ES_OFFSETOF(ES_Object, properties), reg_array);

        CopyTypedDataToValue(*this, reg_properties, 0, ES_STORAGE_INT32, target);

        if (!handle_object)
            return;

        cg.EndOutOfOrderBlock();
        slow_case = handle_object->GetJumpTarget();
    }

    if (possible != ESTYPE_BITS_STRING)
    {
        if (!slow_case)
        {
            if (!current_slow_case)
                EmitSlowCaseCall();
            slow_case = current_slow_case->GetJumpTarget();
        }

        EmitRegisterTypeCheck(object, ESTYPE_STRING, slow_case);
    }

    LoadObjectOperand(*this, REG_T8, object);

    cg.Load(REG_T9, GetConst(CONST_ESTYPE_INT32));
    cg.LW(REG_T8, ES_OFFSETOF(JString, length), REG_T8);

    StoreValue(cg, target, REG_T8, REG_T9);

    if (handle_object)
        cg.SetOutOfOrderContinuationPoint(handle_object);
}

void
ES_Native::EmitFetchPropertyValue(VirtualRegister *target_vr, VirtualRegister *object_vr, ES_Object *object, ES_Class *klass, unsigned property_index)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitFetchPropertyValue");

    const Register reg_object = REG_A0;
    const Register reg_properties = VALUE_TRANSFER_REGISTER;

    if (object_vr)
        cg.LW(reg_object, IVALUE_OFFSET(object_vr), BASE_REGISTER(object_vr));
    else
        cg.Load(reg_object, object);

    cg.LW(reg_properties, ES_OFFSETOF(ES_Object, properties), reg_object);

    ES_Layout_Info layout = klass->GetLayoutInfoAtInfoIndex(ES_PropertyIndex(property_index));

    if (property_value_write_vr && (layout.GetStorageType() == ES_STORAGE_OBJECT || layout.GetStorageType() == ES_STORAGE_WHATEVER))
    {
        OP_ASSERT(property_value_write_vr == target_vr);

        SetPropertyValueTransferRegister(VALUE_TRANSFER_NATIVE_REGISTER, layout.GetOffset(), layout.GetStorageType() != ES_STORAGE_OBJECT);
    }
    else
        CopyTypedDataToValue(*this, reg_properties, layout.GetOffset(), layout.GetStorageType(), target_vr);
}

void
ES_Native::EmitGlobalGet(VirtualRegister *target, unsigned class_id, unsigned cached_offset, unsigned cached_type)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitGlobalGet");

    if (class_id != ES_Class::NOT_CACHED_CLASS_ID)
    {
        const Register reg_global_object = REG_A0;
        const Register reg_class = REG_A1;
        const Register reg_class_id = REG_A2;

        if (!current_slow_case)
            EmitSlowCaseCall();

        ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();
        ES_StorageType type = ES_Value_Internal::StorageTypeFromCachedTypeBits(cached_type);

        cg.Load(reg_global_object, GetConst(CONST_GLOBAL_OBJECT));

        cg.LW(reg_class, ES_OFFSETOF(ES_Object, klass), reg_global_object);
        cg.LW(reg_class_id, ES_OFFSETOF(ES_Class, class_id), reg_class);
        cg.JumpNE(slow_case, reg_class_id, class_id);

        cg.LW(VALUE_TRANSFER_REGISTER, ES_OFFSETOF(ES_Object, properties), reg_global_object);

        if (property_value_write_vr && (type == ES_STORAGE_OBJECT || type == ES_STORAGE_WHATEVER))
        {
            OP_ASSERT(property_value_write_vr == target);

            SetPropertyValueTransferRegister(VALUE_TRANSFER_NATIVE_REGISTER, cached_offset, type != ES_STORAGE_OBJECT);

            if (current_slow_case)
            {
                ES_CodeGenerator::OutOfOrderBlock *recover_from_failure = RecoverFromFailedPropertyValueTransfer(*this, target, VALUE_TRANSFER_REGISTER, type);
                if (recover_from_failure)
                    cg.SetOutOfOrderContinuationPoint(recover_from_failure);
            }
        }
        else
            CopyTypedDataToValue(*this, VALUE_TRANSFER_REGISTER, cached_offset, type, target);
    }
    else
        EmitInstructionHandlerCall();

    if (!property_value_nr)
        property_value_write_vr = NULL;
}

void
ES_Native::EmitGlobalPut(VirtualRegister *source, unsigned class_id, unsigned cached_offset, unsigned cached_type, ES_StorageType source_type)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitGlobalPut");

    if (class_id != ES_Class::NOT_CACHED_CLASS_ID)
    {
        const Register reg_global_object = REG_A0;
        const Register reg_class = REG_A1;
        const Register reg_class_id = REG_A2;
        const Register reg_properties = REG_A3;

        if (!current_slow_case)
            EmitSlowCaseCall();

        ES_CodeGenerator::JumpTarget *slow_case = current_slow_case->GetJumpTarget();

        cg.Load(reg_global_object, GetConst(CONST_GLOBAL_OBJECT));

        cg.LW(reg_class, ES_OFFSETOF(ES_Object, klass), reg_global_object);
        cg.LW(reg_class_id, ES_OFFSETOF(ES_Class, class_id), reg_class);
        cg.JumpNE(slow_case, reg_class_id, class_id);

        cg.LW(reg_properties, ES_OFFSETOF(ES_Object, properties), reg_global_object);

        ES_CodeGenerator::JumpTarget *null_target = NULL, *int_to_double_target = NULL;
        ES_StorageType type = ES_Value_Internal::StorageTypeFromCachedTypeBits(cached_type);
        BOOL skip_write = FALSE;

        if (type != ES_STORAGE_WHATEVER && source_type != type)
            skip_write = EmitTypeConversionCheck(type, source_type, source, slow_case, null_target, int_to_double_target);

        if (!skip_write)
            CopyValueToTypedData(*this, source, reg_properties, cached_offset, type, slow_case);

        EmitTypeConversionHandlers(source, reg_properties, cached_offset, null_target, int_to_double_target);
    }
    else
        EmitInstructionHandlerCall();
}

void
ES_Native::EmitGlobalImmediateGet(VirtualRegister *target, unsigned index, BOOL for_inlined_function_call, BOOL fetch_value)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitGlobalImmediateGet");

    if (for_inlined_function_call)
    {
        const Register reg_code = REG_A0;
        const Register reg_global_object = REG_A1;

        cg.Load(reg_code, GetConst(CONST_CODE));

        ES_CodeGenerator::OutOfOrderBlock *failed_inlined_function = cg.StartOutOfOrderBlock();

        cg.Load(REG_T8, 1);
        cg.SW(REG_T8, ES_OFFSETOF(ES_Code, has_failed_inlined_function), reg_code);

        EmitInstructionHandlerCall();

        cg.EndOutOfOrderBlock(FALSE);

        cg.LW(reg_global_object, ES_OFFSETOF(ES_Code, global_object), reg_code);

        cg.Load(REG_T9, code->global_object->immediate_function_serial_nr);

        cg.LW(REG_T8, ES_OFFSETOF(ES_Global_Object, immediate_function_serial_nr), reg_global_object);

        cg.Jump(failed_inlined_function->GetJumpTarget(), ES_NATIVE_CONDITION_NE(REG_T8, REG_T9));
    }

    if (fetch_value)
    {
        const Register reg_values = REG_A0;

        cg.Load(REG_T8, GetConst(CONST_GLOBAL_OBJECT));
        cg.LW(reg_values, ES_OFFSETOF(ES_Global_Object, variable_values), REG_T8);
        CopyValue(cg, reg_values, index * sizeof(ES_Value_Internal), target);
    }
}

void
ES_Native::EmitGlobalImmediatePut(unsigned index, VirtualRegister *source)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitGlobalImmediatePut");

    const Register reg_values = REG_A0;

    cg.Load(REG_T8, GetConst(CONST_GLOBAL_OBJECT));
    cg.LW(reg_values, ES_OFFSETOF(ES_Global_Object, variable_values), REG_T8);

    CopyValue(cg, source, reg_values, index * sizeof(ES_Value_Internal));
}

void
ES_Native::EmitLexicalGet(VirtualRegister *target, VirtualRegister *function, unsigned scope_index, unsigned variable_index)
{
    DECLARE_NOTHING();

    ANNOTATE("EmitLexicalGet");

    const Register reg_function = REG_A0;
    const Register reg_variables = REG_A1;
    const Register reg_properties = REG_A2;

    if (!current_slow_case)
        EmitSlowCaseCall();

    cg.LW(reg_function, IVALUE_OFFSET(function), BASE_REGISTER(function));

    unsigned offset = ES_OFFSETOF(ES_Function, scope_chain) + scope_index * sizeof(ES_Object *);

    cg.Load(reg_variables, offset, reg_function);

    cg.LW(reg_properties, ES_OFFSETOF(ES_Object, properties), reg_variables);
    cg.Add(reg_properties, 4);

    cg.LW(REG_T8, TYPE_OFFSET(variable_index), reg_properties);

    cg.Load(REG_T9, GetConst(CONST_ESTYPE_BOXED));
    cg.Jump(current_slow_case->GetJumpTarget(), ES_NATIVE_CONDITION_EQ(REG_T8, REG_T9));

    cg.LW(REG_T9, IVALUE_OFFSET(variable_index), reg_properties);

    cg.SW(REG_T8, TYPE_OFFSET(target), BASE_REGISTER(target));
    cg.SW(REG_T9, IVALUE_OFFSET(target), BASE_REGISTER(target));
}

void
ES_Native::EmitFormatString(VirtualRegister *target, VirtualRegister *source, unsigned cache_index)
{
    ANNOTATE("EmitFormatString");

    OP_ASSERT(target->stack_frame_offset == INT_MAX);

    if (!current_slow_case)
        EmitSlowCaseCall();

    cg.Load(REG_T9, &code->format_string_caches[cache_index].from);
    cg.LW(REG_T8, IVALUE_OFFSET(source), BASE_REGISTER(source));
    cg.LW(REG_T9, 0, REG_T9);

    cg.Jump(current_slow_case->GetJumpTarget(), ES_NATIVE_CONDITION_NE(REG_T8, REG_T9));

    cg.Load(REG_T8, &code->format_string_caches[cache_index].to);
    cg.LW(REG_T8, 0, REG_T8);

    if (target != source)
        cg.Load(REG_T9, GetConst(CONST_ESTYPE_STRING));

    cg.SW(REG_T8, IVALUE_OFFSET(target), BASE_REGISTER(target));

    if (target != source)
        cg.SW(REG_T9, TYPE_OFFSET(target), BASE_REGISTER(target));
}

void
ES_Native::EmitTableSwitch(VirtualRegister *value, int minimum, int maximum, Constant *table, ES_CodeGenerator::JumpTarget *default_target)
{
    DECLARE_NOTHING();
    ANNOTATE("EmitTableSwitch");

    ES_ValueType known_type;

    if (!GetType(value, known_type))
        known_type = ESTYPE_UNDEFINED;
    else if (known_type != ESTYPE_INT32 && known_type != ESTYPE_DOUBLE && known_type != ESTYPE_INT32_OR_DOUBLE)
    {
        cg.Jump(default_target);
        return;
    }

    if (known_type != ESTYPE_INT32)
    {
        ES_CodeGenerator::OutOfOrderBlock *convert_double = NULL;
        if (known_type != ESTYPE_DOUBLE)
            convert_double = cg.StartOutOfOrderBlock();

        ANNOTATE("EmitTableSwitch, convert double");

        cg.Add(REG_A1, REG_FP, REGISTER_OFFSET(value));
        CallContextFunc(cg, GetConst(CONST_ESN_GETDOUBLEASSWITCHVALUE));

        cg.LW(REG_T8, ES_OFFSETOF(ES_Execution_Context, implicit_bool), REG_S7);
        cg.Jump(default_target, ES_NATIVE_CONDITION_NEZ(REG_T8));

        if (convert_double)
        {
            cg.EndOutOfOrderBlock();

            cg.LW(REG_T8, TYPE_OFFSET(value), BASE_REGISTER(value));
            cg.Load(REG_T9, GetConst(CONST_ESTYPE_DOUBLE));
            cg.JumpLE(convert_double->GetJumpTarget(), REG_T8, REG_T9);

            if (known_type != ESTYPE_INT32_OR_DOUBLE)
            {
                cg.Sub(REG_T8, REG_T9);
                cg.JumpGT(default_target, REG_T8, ESTYPE_INT32 - ESTYPE_DOUBLE);
            }
            cg.LW(REG_V0, IVALUE_OFFSET(value), BASE_REGISTER(value));

            cg.SetOutOfOrderContinuationPoint(convert_double);
        }
    }
    else
        cg.LW(REG_V0, IVALUE_OFFSET(value), BASE_REGISTER(value));

    if (minimum != 0)
        cg.Sub(REG_V0, minimum);

    cg.Jump(default_target, ES_NATIVE_CONDITION_LTZ(REG_V0));
    cg.JumpGT(default_target, REG_V0, maximum - minimum);

    cg.Load(REG_T9, cg.AddressOf(table));
    cg.SLL(REG_V0, REG_V0, 2);
    cg.ADDU(REG_T8, REG_V0, REG_T9);
    cg.LW(REG_T9, 0, REG_T8);
    cg.JR(REG_T9);
    cg.NOP();
}

ES_CodeGenerator::JumpTarget*
ES_Native::EmitPreConditionsStart(unsigned cw_index)
{
    DECLARE_NOTHING();

    ES_CodeGenerator::OutOfOrderBlock *slow_case = cg.StartOutOfOrderBlock();

    ANNOTATE("EmitPreConditionsStart");

    cg.Load(REG_T8, GetConst(CONST_CODE));
    cg.Load(REG_T9, 1);
    cg.SW(REG_T9, ES_OFFSETOF(ES_Code, has_failed_preconditions), REG_T8);

    cg.Load(REG_A1, cw_index + (constructor ? code->data->codewords_count : 0));

    CallContextFunc(cg, GetConst(CONST_EXC_FORCEUPDATENATIVEDISPATCHER));

    cg.JR(REG_V0);
    cg.NOP();

    cg.EndOutOfOrderBlock(FALSE);

    return slow_case->GetJumpTarget();
}

void
ES_Native::EmitArithmeticBlockSlowPath(ArithmeticBlock *arithmetic_block)
{
    DECLARE_NOTHING();

    unsigned end_instruction_index = arithmetic_block->end_instruction_index;
    unsigned last_instruction_index = end_instruction_index - 1;
    unsigned last_cw_index = code->data->instruction_offsets[last_instruction_index];

    switch (code->data->codewords[last_cw_index].instruction)
    {
    case ESI_RETURN_VALUE:
    case ESI_JUMP_IF_TRUE:
    case ESI_JUMP_IF_FALSE:
        end_instruction_index = last_instruction_index;
    }

    unsigned end_cw_index = code->data->instruction_offsets[end_instruction_index];

    arithmetic_block->slow_case = current_slow_case = cg.StartOutOfOrderBlock();
    ANNOTATE("EmitArithmeticBlockSlowPath");

    if (!is_light_weight)
        EmitExecuteBytecode(arithmetic_block->start_instruction_index, end_instruction_index, end_cw_index != last_cw_index);

    if (end_cw_index == last_cw_index)
    {
        if (end_cw_index == entry_point_cw_index)
            entry_point_jump_target = EmitEntryPoint();

        switch (code->data->codewords[last_cw_index].instruction)
        {
        case ESI_RETURN_VALUE:
            cw_index = last_cw_index;

            EmitRegisterCopy(VR(code->data->codewords[last_cw_index + 1].index), VR(0));
            cg.Jump(epilogue_jump_target);

            cg.EndOutOfOrderBlock(FALSE);
            break;

        case ESI_JUMP_IF_TRUE:
        case ESI_JUMP_IF_FALSE:
            ES_CodeGenerator::JumpTarget *true_target = NULL, *false_target = NULL;
            Operand value_target;

            GetConditionalTargets(last_cw_index, value_target, true_target, false_target);

            cg.LW(REG_T8, ES_OFFSETOF(ES_Execution_Context, implicit_bool), REG_S7);

            if (true_target)
                cg.Jump(true_target, ES_NATIVE_CONDITION_NEZ(REG_T8));
            else
                cg.Jump(false_target, ES_NATIVE_CONDITION_EQZ(REG_T8));

            cg.EndOutOfOrderBlock();
        }
    }
    else
        cg.EndOutOfOrderBlock();
}

ES_CodeGenerator::JumpTarget*
ES_Native::EmitEntryPoint(BOOL custom_light_weight_entry)
{
    DECLARE_NOTHING();
    OP_ASSERT(!custom_light_weight_entry); // Parameter not in use any more.

    ES_CodeGenerator::OutOfOrderBlock *entry_point;

    if (is_light_weight)
    {
        if (entry_point_cw_index == 0 && prologue_entry_point)
            return NULL;

        entry_point = cg.StartOutOfOrderBlock();

        ANNOTATE("EmitEntryPoint");

        frame_size = ES_NativeStackFrame::GetFrameSize(code, TRUE);

        cg.LoadDataSectionPointer();

        cg.LW(REG_T8, SP_OFFSET_NEXT_STACK_FRAME(frame_size), REG_SP);
        cg.LW(REG_S5, SP_OFFSET_RETURN_ADDRESS(frame_size), REG_SP);
        cg.Add(REG_SP, frame_size);
        cg.SW(REG_T8, ES_OFFSETOF(ES_Execution_Context, native_stack_frame), REG_S7);

        cg.EndOutOfOrderBlock();
        cg.SetOutOfOrderContinuationPoint(entry_point);

        return entry_point->GetJumpTarget();
    }
    else
    {
        if (entry_point_cw_index == 0 && prologue_entry_point)
            return cg.Here();

        entry_point = cg.StartOutOfOrderBlock();

        ANNOTATE("EmitEntryPoint");

        cg.LoadDataSectionPointer();

        cg.EndOutOfOrderBlock();
        cg.SetOutOfOrderContinuationPoint(entry_point);

        return entry_point->GetJumpTarget();
    }
}

ES_CodeGenerator::JumpTarget *
ES_Native::EmitInlineResumption(unsigned class_id, VirtualRegister *object_vr, ES_CodeGenerator::JumpTarget *failure, unsigned char *mark_failure)
{
    DECLARE_NOTHING();

    entry_point_jump_target = EmitEntryPoint(FALSE);
    ES_CodeGenerator::OutOfOrderBlock *extra_check = cg.StartOutOfOrderBlock();

    const Register reg_object = REG_A0;
    const Register reg_object_class_id = REG_A1;

    LoadObjectOperand(*this, reg_object, object_vr);

    cg.LW(REG_T8, ES_OFFSETOF(ES_Object, klass), reg_object);
    cg.LW(reg_object_class_id, ES_OFFSETOF(ES_Class, class_id), REG_T8);
    cg.JumpEQ(entry_point_jump_target, reg_object_class_id, class_id);

    /* Update profile data to indicate that the instruction failed inlining. */
    Constant *profile_data = cg.NewConstant((void*)mark_failure);
    cg.Load(REG_T8, profile_data);
    cg.LBU(REG_T9, 0, REG_T8);
    cg.ORI(REG_T9, REG_T9, ES_CodeStatic::GET_FAILED_INLINE);
    cg.SB(REG_T9, 0, REG_T8);
    cg.Jump(failure);

    cg.EndOutOfOrderBlock(FALSE);
    return extra_check->GetJumpTarget();
}

void
ES_Native::EmitTick()
{
    DECLARE_NOTHING();

    ANNOTATE("EmitTick");

    ES_CodeGenerator::JumpTarget *eot = cg.ForwardJump();
    ES_CodeGenerator::JumpTarget *end = cg.ForwardJump();

    cg.LW(REG_T8, ES_OFFSETOF(ES_Execution_Context, time_until_check), REG_S7);

    cg.Sub(REG_T8, 1);

    cg.Jump(eot, ES_NATIVE_CONDITION_EQZ(REG_T8));

    cg.SW(REG_T8, ES_OFFSETOF(ES_Execution_Context, time_until_check), REG_S7);
    cg.Jump(end, ES_NATIVE_UNCONDITIONAL, TRUE);

    cg.SetJumpTarget(eot);
    CallContextFunc(cg, GetConst(CONST_EXC_CHECKOUTOFTIMEFROMMACHINECODE));

    cg.SetJumpTarget(end);
}

static ES_CodeGenerator::OutOfOrderBlock*
InitializeFormals(ES_Native &n, unsigned formals_count)
{
    ES_CodeGenerator &cg = n.cg;

    ES_CodeGenerator::OutOfOrderBlock *initialize_formals = cg.StartOutOfOrderBlock();

    ANNOTATE("GeneratePrologue, initializing formals to undefined");

    cg.Load(REG_T8, n.GetConst(CONST_ESTYPE_UNDEFINED));

    unsigned vr_index = 2 + formals_count - 1;

    if (formals_count > 1)
    {
        cg.Load(REG_T9, formals_count);
        cg.SUBU(REG_A1, REG_T9, REG_A1);

        cg.Add(REG_T9, REG_FP, TYPE_OFFSET(vr_index));

        // loop
        cg.SW(REG_T8, 0, REG_T9);
        cg.Sub(REG_A1, 1);
        cg.BNE(REG_Z, REG_A1, -3); // branch to loop
        cg.ADDIU(REG_T9, REG_T9, (short) -sizeof(ES_Value_Internal));
    }
    else
        cg.SW(REG_T8, TYPE_OFFSET(vr_index), REG_FP);

    cg.EndOutOfOrderBlock();

    return initialize_formals;
}

/* Generate dispatcher function prologue.  This function will be called
   after all regular code has been generated, since it will need to know
   what native registers have been used to know which it needs to store on
   the stack. */
void
ES_Native::GeneratePrologue()
{
    DECLARE_NOTHING();

    cg.StartPrologue();

    ANNOTATE("GeneratePrologue");

    if (entry_point_cw == code->data->codewords && prologue_entry_point && !entry_point_jump_target)
        entry_point_jump_target = cg.Here();

    cg.LoadDataSectionPointer();

    if ((is_trivial || is_light_weight) && !is_inlined_function_call)
        cg.Move(REG_S5, REG_RA);

    if (is_trivial)
    {
        if (code->type == ES_Code::TYPE_FUNCTION)
        {
            ES_FunctionCodeStatic *data = fncode->GetData();

            if (is_inlined_function_call)
            {
                for (unsigned index = inlined_call_argc; index < data->formals_count; index++)
                    EmitSetRegisterType(VR(2 + index), ESTYPE_UNDEFINED);
            }
            else
            {
                if (data->formals_count > 0)
                {
                    ES_CodeGenerator::OutOfOrderBlock *initialize_formals = InitializeFormals(*this, data->formals_count);

                    cg.JumpLT(initialize_formals->GetJumpTarget(), REG_A1, data->formals_count);

                    cg.SetOutOfOrderContinuationPoint(initialize_formals);
                }
            }
        }
    }
    else
    {
        if (is_light_weight)
        {
            if (!light_weight_failure)
                EmitLightWeightFailure();

            if (fncode)
            {
                if (is_inlined_function_call)
                {
                    for (unsigned index = inlined_call_argc; index < fncode->GetData()->formals_count; index++)
                        EmitSetRegisterType(VR(2 + index), ESTYPE_UNDEFINED);
                }
                else
                {
                    cg.Load(REG_T8, fncode->GetData()->formals_count);
                    cg.Jump(light_weight_wrong_argc, ES_NATIVE_CONDITION_NE(REG_T8, REG_A1));
                }
            }

            if (optimization_data->uses_this && !code->data->is_strict_mode)
            {
                ES_CodeGenerator::OutOfOrderBlock *prepare_this = cg.StartOutOfOrderBlock();

                ANNOTATE("GeneratePrologue, prepare this object (light weight)");

                cg.Move(REG_A1, REG_FP);
                CallContextFunc(cg, GetConst(CONST_ESN_PREPARETHISOBJECTLIGHTWEIGHT));

                cg.EndOutOfOrderBlock();

                EmitRegisterTypeCheck(VR(0), ESTYPE_OBJECT, prepare_this->GetJumpTarget());

                cg.SetOutOfOrderContinuationPoint(prepare_this);
            }
        }
        else
        {
            /* A loop dispatcher will have its stack frame created by
                ES_Execution_Context::ReconstructNativeStack(). */
            if (!loop_dispatcher)
                frame_size = CreateNativeStackFrame(*this);
            else
                frame_size = ES_NativeStackFrame::GetFrameSize(code, TRUE);

            ES_CodeGenerator::OutOfOrderBlock *stack_or_register_limit_exceeded = cg.StartOutOfOrderBlock();

            ANNOTATE("GeneratePrologue, stack or register limit exceeded");

            cg.Load(REG_A1, constructor ? 1 : 0);
            CallContextFunc(cg, GetConst(CONST_EXC_STACKORREGISTERLIMITEXCEEDED));

            cg.LW(REG_RA, SP_OFFSET_RETURN_ADDRESS(frame_size), REG_SP);
            cg.JR();
            cg.Add(REG_SP, frame_size);

            cg.EndOutOfOrderBlock(FALSE);

            if (entry_point_cw_index == 0 && !prologue_entry_point)
            {
                ES_CodeGenerator::OutOfOrderBlock *entry_point = cg.StartOutOfOrderBlock();

                entry_point_jump_target = cg.Here();

                cg.LW(REG_A1, SP_OFFSET_ARGUMENT_COUNT(frame_size), REG_SP);
                cg.LoadDataSectionPointer();
                cg.EndOutOfOrderBlock();
                cg.SetOutOfOrderContinuationPoint(entry_point);
            }

            cg.LW(REG_T8, ES_OFFSETOF(ES_Execution_Context, stack_limit), REG_S7);
            cg.JumpLT(stack_or_register_limit_exceeded->GetJumpTarget(), REG_SP, REG_T8);

            cg.LW(REG_T9, ES_OFFSETOF(ES_Execution_Context, reg_limit), REG_S7);
            cg.Sub(REG_T9, code->data->register_frame_size * sizeof(ES_Value_Internal));
            cg.JumpLT(stack_or_register_limit_exceeded->GetJumpTarget(), REG_T9, REG_FP);

            if (fncode)
            {
                if (!constructor && optimization_data->uses_this && !code->data->is_strict_mode)
                {
                    ES_CodeGenerator::OutOfOrderBlock *prepare_this = cg.StartOutOfOrderBlock();

                    ANNOTATE("GeneratePrologue, prepare this object");

                    cg.Move(REG_S1, REG_A1);

                    cg.Move(REG_A1, REG_FP);
                    CallContextFunc(cg, GetConst(CONST_ESN_PREPARETHISOBJECT));

                    cg.Move(REG_A1, REG_S1);

                    cg.EndOutOfOrderBlock();

                    EmitRegisterTypeCheck(VR(0), ESTYPE_OBJECT, prepare_this->GetJumpTarget());

                    cg.SetOutOfOrderContinuationPoint(prepare_this);
                }

                ES_FunctionCodeStatic *data = fncode->GetData();
                ES_CodeGenerator::OutOfOrderBlock *create_arguments_object;

                /* Create arguments array, if necessary. */
                if (data->arguments_index == ES_FunctionCodeStatic::ARGUMENTS_NOT_USED && !data->is_strict_mode)
                    create_arguments_object = cg.StartOutOfOrderBlock();
                else
                    create_arguments_object = NULL;

                if (data->arguments_index != ES_FunctionCodeStatic::ARGUMENTS_NOT_USED || !data->is_strict_mode)
                {
                    ANNOTATE("GeneratePrologue, create arguments object");

                    if (data->formals_count != 0)
                        cg.Move(REG_S1, REG_A1);

                    cg.Move(REG_A3, REG_A1);
                    cg.Move(REG_A2, REG_FP);
                    cg.LW(REG_A1, IVALUE_OFFSET(VR(1)), BASE_REGISTER(VR(1)));

                    CallContextFunc(cg, GetConst(CONST_EXC_CREATEARGUMENTSOBJECT));

                    if (data->formals_count != 0)
                        cg.Move(REG_A1, REG_S1);

                    if (create_arguments_object)
                        cg.EndOutOfOrderBlock();
                }

                ES_CodeGenerator::OutOfOrderBlock *initialize_formals;

                if (data->formals_count != 0)
                    initialize_formals = InitializeFormals(*this, data->formals_count);
                else
                    initialize_formals = NULL;

                if (create_arguments_object || initialize_formals)
                {
                    cg.Sub(REG_T8, REG_A1, data->formals_count);

                    if (initialize_formals)
                        cg.Jump(initialize_formals->GetJumpTarget(), ES_NATIVE_CONDITION_LTZ(REG_T8));
                    if (create_arguments_object)
                        cg.Jump(create_arguments_object->GetJumpTarget(), ES_NATIVE_CONDITION_GTZ(REG_T8));

                    if (initialize_formals)
                        cg.SetOutOfOrderContinuationPoint(initialize_formals);
                    if (create_arguments_object)
                        cg.SetOutOfOrderContinuationPoint(create_arguments_object);
                }

                EmitTick();

                if (constructor)
                {
                    const Register reg_object = ALLOCATED_OBJECT_REGISTER;

                    BOOL inline_allocation = constructor_final_class && CanAllocateObject(constructor_final_class, 0);
                    ES_CodeGenerator::OutOfOrderBlock *complex_case = cg.StartOutOfOrderBlock();

                    ANNOTATE("GeneratePrologue, calling ES_Execution_Context::AllocateObjectFromMachineCodeComplex()");

                    CallContextFunc(cg, GetConst(CONST_EXC_ALLOCATEOBJECTFROMMACHINECODECOMPLEX));

                    cg.Move(reg_object, REG_V0);
                    cg.EndOutOfOrderBlock();

                    ANNOTATE("GeneratePrologue, verifying that ctor.prototype hasn't changed");

                    const Register reg_function = REG_A1;
                    const Register reg_function_properties = REG_S2;

                    cg.LW(reg_function, IVALUE_OFFSET(VR(1)), BASE_REGISTER(VR(1)));
                    cg.LW(reg_function_properties, ES_OFFSETOF(ES_Object, properties), reg_function);

                    unsigned prototype_offset = ES_Function::GetPropertyOffset(ES_Function::PROTOTYPE);

                    if (constructor_prototype)
                    {
                        ES_CodeGenerator::JumpTarget *skip = cg.ForwardJump();

                        LoadValue(cg, reg_function_properties, prototype_offset, REG_S3, REG_S4);
                        cg.Load(REG_T9, GetConst(CONST_ESTYPE_OBJECT));
                        cg.Load(REG_T8, constructor_prototype);

                        cg.Jump(skip, ES_NATIVE_CONDITION_NE(REG_S4, REG_T9));
                        cg.Jump(complex_case->GetJumpTarget(), ES_NATIVE_CONDITION_NE(REG_S3, REG_T8));

                        cg.SetJumpTarget(skip);
                    }
                    else
                    {
                        cg.Load(REG_T8, prototype_offset + VALUE_TYPE_OFFSET, reg_function_properties);
                        cg.Load(REG_T9, GetConst(CONST_ESTYPE_OBJECT));
                        cg.JumpEQ(complex_case->GetJumpTarget(), REG_T8, REG_T9);
                    }

                    ES_CodeGenerator::OutOfOrderBlock *non_trivial;

                    if (inline_allocation)
                        non_trivial = cg.StartOutOfOrderBlock();
                    else
                        non_trivial = NULL;

                    ANNOTATE("GeneratePrologue, calling ES_Execution_Context::AllocateObjectFromMachineCodeSimple()");

                    cg.Load(REG_A1, constructor_class);
                    CallContextFunc(cg, GetConst(CONST_EXC_ALLOCATEOBJECTFROMMACHINECODESIMPLE));
                    cg.Move(reg_object, REG_V0);

                    if (inline_allocation)
                    {
                        cg.EndOutOfOrderBlock();

                        ANNOTATE("GeneratePrologue, inlined object allocation");

                        AllocateObject(*this, constructor_class, constructor_final_class, 0, 0, NULL, non_trivial->GetJumpTarget());

                        cg.SetOutOfOrderContinuationPoint(non_trivial);
                    }

                    cg.SetOutOfOrderContinuationPoint(complex_case);

                    ANNOTATE("GeneratePrologue, store object");

                    cg.Load(REG_T8, GetConst(CONST_ESTYPE_OBJECT));
                    cg.SW(reg_object, IVALUE_OFFSET(VR(0)), BASE_REGISTER(VR(0)));
                    cg.SW(REG_T8, TYPE_OFFSET(VR(0)), BASE_REGISTER(VR(0)));
                }
            }
        }

        if (loop_dispatcher && first_loop_io)
        {
            cg.Move(REG_A2, REG_FP);
            cg.Load(REG_A1, GetConst(CONST_CODE));
            CallContextFunc(cg, GetConst(CONST_EXC_READLOOPIO));
        }
    }

    cg.EndPrologue();
}

/* Generate dispatcher function epilogue. */
void
ES_Native::GenerateEpilogue()
{
    DECLARE_NOTHING();

    ANNOTATE("GenerateEpilogue");

    cg.SetJumpTarget(epilogue_jump_target);

    if (loop_dispatcher && first_loop_io)
    {
        cg.Move(REG_A2, REG_FP);
        cg.Load(REG_A1, GetConst(CONST_CODE));
        CallContextFunc(cg, GetConst(CONST_EXC_WRITELOOPIO));
    }

    if (!is_trivial && !is_light_weight)
    {
        OP_ASSERT(frame_size != UINT_MAX);

        if (code->type == ES_Code::TYPE_FUNCTION)
        {
            ES_FunctionCode *fncode = static_cast<ES_FunctionCode *>(code);
            ES_FunctionCodeStatic *data = fncode->GetData();

            ES_CodeGenerator::OutOfOrderBlock *detach_arguments;

            if (data->arguments_index == ES_FunctionCodeStatic::ARGUMENTS_NOT_USED)
                detach_arguments = cg.StartOutOfOrderBlock();
            else
                detach_arguments = NULL;

            ANNOTATE("GenerateEpilogue, detach arguments object");

            if (!detach_arguments)
                cg.LW(REG_A1, SP_OFFSET_ARGUMENTS_OBJECT(frame_size), REG_SP);

            CallContextFunc(cg, GetConst(CONST_EXC_DETACHARGUMENTSOBJECT));

            if (detach_arguments)
            {
                cg.EndOutOfOrderBlock();

                cg.LW(REG_A1, SP_OFFSET_ARGUMENTS_OBJECT(frame_size), REG_SP);
                cg.Jump(detach_arguments->GetJumpTarget(), ES_NATIVE_CONDITION_NEZ(REG_A1));

                cg.SetOutOfOrderContinuationPoint(detach_arguments);
            }

            if (data->CanHaveVariableObject())
            {
                ES_CodeGenerator::OutOfOrderBlock *detach_variables = cg.StartOutOfOrderBlock();

                ANNOTATE("GenerateEpilogue, detach variables object");

                CallContextFunc(cg, GetConst(CONST_EXC_DETACHVARIABLEOBJECT));
                cg.EndOutOfOrderBlock();

                cg.LW(REG_A1, SP_OFFSET_VARIABLE_OBJECT(frame_size), REG_SP);
                cg.Jump(detach_variables->GetJumpTarget(), ES_NATIVE_CONDITION_NEZ(REG_A1));

                cg.SetOutOfOrderContinuationPoint(detach_variables);
            }
        }

        cg.LW(REG_T8, SP_OFFSET_NEXT_STACK_FRAME(frame_size), REG_SP);
        cg.LW(REG_RA, SP_OFFSET_RETURN_ADDRESS(frame_size), REG_SP);
        cg.SW(REG_T8, ES_OFFSETOF(ES_Execution_Context, native_stack_frame), REG_S7);
        cg.JR();
        cg.Add(REG_SP, frame_size);
    }
    else if (!is_inlined_function_call)
    {
        cg.JR(REG_S5);
        cg.NOP();
    }
}

/* static */ unsigned
ES_NativeStackFrame::GetFrameSize(ES_Code *code, BOOL include_return_address)
{
    unsigned frame_size = 0;

    frame_size += 4 * sizeof(void *);

    if (code->type == ES_Code::TYPE_FUNCTION)
    {
        frame_size += 2 * sizeof(void *);

        if (code->CanHaveVariableObject())
            frame_size += sizeof(void *);
    }

    frame_size = (frame_size + MIPS_STACK_ALIGNMENT - 1) & ~(MIPS_STACK_ALIGNMENT - 1);

    if (!include_return_address)
        frame_size -= sizeof(void *);

    return frame_size;
}

Constant*
ES_ArchitecureMixin::GetConst(IntegerConstants c)
{
#ifdef NATIVE_DISASSEMBLER_ANNOTATION_SUPPORT
    OpString annotation;
    annotation.AppendFormat(UNI_L("GetConst integer 0x%x"), int_const_template[c]);
    static_cast<ES_Native*>(this)->cg.Annotate(annotation.CStr());
#endif // NATIVE_DISASSEMBLER_ANNOTATION_SUPPORT

    if (int_constants[c])
        return int_constants[c];

    return int_constants[c] = static_cast<ES_Native*>(this)->cg.NewConstant(int_const_template[c]);
}

Constant*
ES_ArchitecureMixin::GetConst(FunctionConstants c)
{
#ifdef NATIVE_DISASSEMBLER_ANNOTATION_SUPPORT
    OpString annotation;
    annotation.AppendFormat(UNI_L("GetConst function 0x%08x"), function_const_template[c]);
    static_cast<ES_Native*>(this)->cg.Annotate(annotation.CStr());
#endif // NATIVE_DISASSEMBLER_ANNOTATION_SUPPORT

    if (function_constants[c])
        return function_constants[c];

    return function_constants[c] = static_cast<ES_Native*>(this)->cg.NewConstant(reinterpret_cast<void*>(function_const_template[c]));
}

Constant*
ES_ArchitecureMixin::GetConst(PointerConstants c)
{
#ifdef NATIVE_DISASSEMBLER_ANNOTATION_SUPPORT
    OpString annotation;
    annotation.AppendFormat(UNI_L("GetConst pointer 0x%08x"), pointer_const_template[c]);
    static_cast<ES_Native*>(this)->cg.Annotate(annotation.CStr());
#endif // NATIVE_DISASSEMBLER_ANNOTATION_SUPPORT

    if (pointer_constants[c])
        return pointer_constants[c];

    return pointer_constants[c] = static_cast<ES_Native*>(this)->cg.NewConstant(pointer_const_template[c]);
}

/* static */
int ES_ArchitecureMixin::int_const_template[NUMBER_OF_INT_CONSTANTS] =
{
    ESTYPE_DOUBLE,
    ESTYPE_INT32_OR_DOUBLE,
    ESTYPE_INT32,
    ESTYPE_UNDEFINED,
    ESTYPE_NULL,
    ESTYPE_BOOLEAN,
    ESTYPE_BOXED,
    ESTYPE_STRING,
    ESTYPE_OBJECT,
    ES_VALUE_TYPE_INIT_MASK,
    INT_MAX,
    (~(ES_STORAGE_BITS_DOUBLE | ES_STORAGE_BITS_WHATEVER) << ES_STORAGE_TYPE_SHIFT) & ES_STORAGE_TYPE_MASK
};

/* static */
FuncPtr ES_ArchitecureMixin::function_const_template[NUMBER_OF_FUNCTION_CONSTANTS] =
{
    reinterpret_cast<FuncPtr>(&AllocateSimple),
    reinterpret_cast<FuncPtr>(&Equals),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::AllocateObjectFromMachineCodeComplex),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::AllocateObjectFromMachineCodeSimple),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::CallFromMachineCode),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::CheckOutOfTimeFromMachineCode),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::ConstructFromMachineCode),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::CreateArgumentsObject),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::DetachArgumentsObject),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::DetachVariableObject),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::EqFromMachineCode),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::EqStrict),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::EvalFromMachineCode),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::ExecuteBytecode),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::ForceUpdateNativeDispatcher),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::GetGlobal),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::GetGlobalImmediate),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::GetNamedImmediate),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::LightWeightDispatcherFailure),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::PutGlobal),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::PutNamedImmediate),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::ReadLoopIO),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::StackOrRegisterLimitExceeded),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::ThrowFromMachineCode),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::UpdateComparison),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::UpdateNativeDispatcher),
    reinterpret_cast<FuncPtr>(&ES_Execution_Context::WriteLoopIO),
    reinterpret_cast<FuncPtr>(&ES_Native::CompareShiftyObjects),
    reinterpret_cast<FuncPtr>(&ES_Native::GetDoubleAsSwitchValue),
    reinterpret_cast<FuncPtr>(&ES_Native::InstanceOf),
    reinterpret_cast<FuncPtr>(&ES_Native::MathAbs),
    reinterpret_cast<FuncPtr>(&ES_Native::MathCeil),
    reinterpret_cast<FuncPtr>(&ES_Native::MathCos),
    reinterpret_cast<FuncPtr>(&ES_Native::MathFloor),
    reinterpret_cast<FuncPtr>(&ES_Native::MathPow),
    reinterpret_cast<FuncPtr>(&ES_Native::MathSin),
    reinterpret_cast<FuncPtr>(&ES_Native::MathSqrt),
    reinterpret_cast<FuncPtr>(&ES_Native::MathTan),
    reinterpret_cast<FuncPtr>(&ES_Native::PrepareThisObject),
    reinterpret_cast<FuncPtr>(&ES_Native::PrepareThisObjectLightWeight),
    reinterpret_cast<FuncPtr>(&ES_Native::StoreDoubleAsFloat),
    reinterpret_cast<FuncPtr>(&ES_Native::StoreFloatAsDouble),
    reinterpret_cast<FuncPtr>(&ES_Native::StoreIntAsDouble),
    reinterpret_cast<FuncPtr>(&ES_Native::StoreIntAsFloat),
    reinterpret_cast<FuncPtr>(&ES_Native::StoreUIntAsDouble),
    reinterpret_cast<FuncPtr>(&ES_Native::GetDoubleAsInt),
    reinterpret_cast<FuncPtr>(&ES_Native::GetDoubleAsIntRound),
    reinterpret_cast<FuncPtr>(&ES_Value_Internal::ReturnAsBoolean)
};

#undef private_type
#undef private_value

#undef D
#undef W

#undef EQ

#undef REG_Z
#undef REG_AT
#undef REG_V0
#undef REG_V1
#undef REG_A0
#undef REG_A1
#undef REG_A2
#undef REG_A3
#undef REG_T0
#undef REG_T1
#undef REG_T2
#undef REG_T3
#undef REG_T4
#undef REG_T5
#undef REG_T6
#undef REG_T7
#undef REG_S0
#undef REG_S1
#undef REG_S2
#undef REG_S3
#undef REG_S4
#undef REG_S5
#undef REG_S6
#undef REG_S7
#undef REG_T8
#undef REG_T9
#undef REG_SP
#undef REG_FP
#undef REG_RA

#undef REG_F28
#undef REG_F30

#undef REG_FEXR

#undef ALLOCATED_OBJECT_REGISTER
#undef ALLOCATED_OBJECT_NAMED_PROPERTIES_REGISTER
#undef ALLOCATED_OBJECT_INDEXED_PROPERTIES_REGISTER

#undef MIPS_STACK_ALIGNMENT
#undef MIPS_STACK_ARG_AREA

#undef ANNOTATE

#undef DECLARE_NOTHING

#endif // ARCHITECTURE_MIPS
#endif // ES_NATIVE_SUPPORT
