# include "hwloc.h"

# include "hwloc.h"

#endif

#endif

#include <xmmintrin.h>

#include <xmmintrin.h>

#endif

#endif

};

};

#endif /* KMP_OS_LINUX */

#endif /* KMP_OS_LINUX */

enum mic_type {

enum mic_type {

    non_mic,

    non_mic,

    mic1,

    mic1,

#define KMP_FAST_REDUCTION_BARRIER 1

#define KMP_FAST_REDUCTION_BARRIER 1

#undef KMP_FAST_REDUCTION_CORE_DUO

#undef KMP_FAST_REDUCTION_CORE_DUO

    #define KMP_FAST_REDUCTION_CORE_DUO 1

    #define KMP_FAST_REDUCTION_CORE_DUO 1

#endif

#endif

enum affinity_top_method {

enum affinity_top_method {

    affinity_top_method_all = 0, // try all (supported) methods, in order

    affinity_top_method_all = 0, // try all (supported) methods, in order

    affinity_top_method_apicid,

    affinity_top_method_apicid,

    affinity_top_method_x2apicid,

    affinity_top_method_x2apicid,

    affinity_top_method_cpuinfo, // KMP_CPUINFO_FILE is usable on Windows* OS, too

    affinity_top_method_cpuinfo, // KMP_CPUINFO_FILE is usable on Windows* OS, too

#if KMP_GROUP_AFFINITY

#if KMP_GROUP_AFFINITY

    affinity_top_method_group,

    affinity_top_method_group,

#if KMP_ARCH_X86

#if KMP_ARCH_X86

# define KMP_DEFAULT_STKSIZE     ((size_t)(2 * 1024 * 1024))

# define KMP_DEFAULT_STKSIZE     ((size_t)(2 * 1024 * 1024))

# define KMP_DEFAULT_STKSIZE     ((size_t)(4 * 1024 * 1024))

# define KMP_DEFAULT_STKSIZE     ((size_t)(4 * 1024 * 1024))

# define KMP_BACKUP_STKSIZE      ((size_t)(2 * 1024 * 1024))

# define KMP_BACKUP_STKSIZE      ((size_t)(2 * 1024 * 1024))

#else

#else

/* Minimum number of threads before switch to TLS gtid (experimentally determined) */

/* Minimum number of threads before switch to TLS gtid (experimentally determined) */

/* josh TODO: what about OS X* tuning? */

/* josh TODO: what about OS X* tuning? */

# define KMP_TLS_GTID_MIN     5

# define KMP_TLS_GTID_MIN     5

#else

#else

# define KMP_TLS_GTID_MIN     INT_MAX

# define KMP_TLS_GTID_MIN     INT_MAX

#  define KMP_NEXT_WAIT   512U          /* susequent number of spin-tests */

#  define KMP_NEXT_WAIT   512U          /* susequent number of spin-tests */

#endif

#endif

typedef struct kmp_cpuid {

typedef struct kmp_cpuid {

    kmp_uint32  eax;

    kmp_uint32  eax;

    kmp_uint32  ebx;

    kmp_uint32  ebx;

    long nivcsw;          /* the number of times a context switch was forced           */

    long nivcsw;          /* the number of times a context switch was forced           */

} kmp_sys_info_t;

} kmp_sys_info_t;

typedef struct kmp_cpuinfo {

typedef struct kmp_cpuinfo {

    int        initialized;  // If 0, other fields are not initialized.

    int        initialized;  // If 0, other fields are not initialized.

    int        signature;    // CPUID(1).EAX

    int        signature;    // CPUID(1).EAX

// Set up how many argv pointers will fit in cache lines containing t_inline_argv. Historically, we

// Set up how many argv pointers will fit in cache lines containing t_inline_argv. Historically, we

// have supported at least 96 bytes. Using a larger value for more space between the master write/worker

// have supported at least 96 bytes. Using a larger value for more space between the master write/worker

// read section and read/write by all section seems to buy more performance on EPCC PARALLEL.

// read section and read/write by all section seems to buy more performance on EPCC PARALLEL.

# define KMP_INLINE_ARGV_BYTES         ( 4 * CACHE_LINE - ( ( 3 * KMP_PTR_SKIP + 2 * sizeof(int) + 2 * sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32) ) % CACHE_LINE ) )

# define KMP_INLINE_ARGV_BYTES         ( 4 * CACHE_LINE - ( ( 3 * KMP_PTR_SKIP + 2 * sizeof(int) + 2 * sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32) ) % CACHE_LINE ) )

#else

#else

# define KMP_INLINE_ARGV_BYTES         ( 2 * CACHE_LINE - ( ( 3 * KMP_PTR_SKIP + 2 * sizeof(int) ) % CACHE_LINE ) )

# define KMP_INLINE_ARGV_BYTES         ( 2 * CACHE_LINE - ( ( 3 * KMP_PTR_SKIP + 2 * sizeof(int) ) % CACHE_LINE ) )

    ompt_lw_taskteam_t      *ompt_serialized_team_info;

    ompt_lw_taskteam_t      *ompt_serialized_team_info;

#endif

#endif

    kmp_int8                 t_fp_control_saved;

    kmp_int8                 t_fp_control_saved;

    kmp_int8                 t_pad2b;

    kmp_int8                 t_pad2b;

    kmp_int16                t_x87_fpu_control_word; // FP control regs

    kmp_int16                t_x87_fpu_control_word; // FP control regs

    kmp_uint32               t_mxcsr;

    kmp_uint32               t_mxcsr;

    void                    *t_inline_argv[ KMP_INLINE_ARGV_ENTRIES ];

    void                    *t_inline_argv[ KMP_INLINE_ARGV_ENTRIES ];

    int t_size_changed; // team size was changed?: 0: no, 1: yes, -1: changed via omp_set_num_threads() call

    int t_size_changed; // team size was changed?: 0: no, 1: yes, -1: changed via omp_set_num_threads() call

    // Read/write by workers as well -----------------------------------------------------------------------

    // Read/write by workers as well -----------------------------------------------------------------------

    // Using CACHE_LINE=64 reduces memory footprint, but causes a big perf regression of epcc 'parallel'

    // Using CACHE_LINE=64 reduces memory footprint, but causes a big perf regression of epcc 'parallel'

    // and 'barrier' on fxe256lin01. This extra padding serves to fix the performance of epcc 'parallel'

    // and 'barrier' on fxe256lin01. This extra padding serves to fix the performance of epcc 'parallel'

    // and 'barrier' when CACHE_LINE=64. TODO: investigate more and get rid if this padding.

    // and 'barrier' when CACHE_LINE=64. TODO: investigate more and get rid if this padding.

extern int      __kmp_storage_map_verbose; /* True means storage map includes placement info */

extern int      __kmp_storage_map_verbose; /* True means storage map includes placement info */

extern int      __kmp_storage_map_verbose_specified;

extern int      __kmp_storage_map_verbose_specified;

extern kmp_cpuinfo_t    __kmp_cpuinfo;

extern kmp_cpuinfo_t    __kmp_cpuinfo;

#endif

#endif

#endif

#endif

extern int        __kmp_tls_gtid_min;   /* #threads below which use sp search for gtid */

extern int        __kmp_tls_gtid_min;   /* #threads below which use sp search for gtid */

extern int        __kmp_foreign_tp;     /* If true, separate TP var for each foreign thread */

extern int        __kmp_foreign_tp;     /* If true, separate TP var for each foreign thread */

extern int        __kmp_inherit_fp_control; /* copy fp creg(s) parent->workers at fork */

extern int        __kmp_inherit_fp_control; /* copy fp creg(s) parent->workers at fork */

extern kmp_int16  __kmp_init_x87_fpu_control_word; /* init thread's FP control reg */

extern kmp_int16  __kmp_init_x87_fpu_control_word; /* init thread's FP control reg */

extern kmp_uint32 __kmp_init_mxcsr;      /* init thread's mxscr */

extern kmp_uint32 __kmp_init_mxcsr;      /* init thread's mxscr */

extern int        __kmp_dflt_max_active_levels; /* max_active_levels for nested parallelism enabled by default a la OMP_MAX_ACTIVE_LEVELS */

extern int        __kmp_dflt_max_active_levels; /* max_active_levels for nested parallelism enabled by default a la OMP_MAX_ACTIVE_LEVELS */

extern int        __kmp_dispatch_num_buffers; /* max possible dynamic loops in concurrent execution per team */

extern int        __kmp_dispatch_num_buffers; /* max possible dynamic loops in concurrent execution per team */

extern int __kmp_clock_function_param;

extern int __kmp_clock_function_param;

# endif /* KMP_OS_LINUX */

# endif /* KMP_OS_LINUX */

extern enum mic_type __kmp_mic_type;

extern enum mic_type __kmp_mic_type;

#endif

#endif

#endif

#endif

  microtask_t microtask, launch_t invoker,

  microtask_t microtask, launch_t invoker,

/* TODO: revert workaround for Intel(R) 64 tracker #96 */

/* TODO: revert workaround for Intel(R) 64 tracker #96 */

                             va_list *ap

                             va_list *ap

#else

#else

                             va_list ap

                             va_list ap

// Assembly routines that have no compiler intrinsic replacement

// Assembly routines that have no compiler intrinsic replacement

//

//

extern void       __kmp_query_cpuid( kmp_cpuinfo_t *p );

extern void       __kmp_query_cpuid( kmp_cpuinfo_t *p );

extern void __kmp_clear_x87_fpu_status_word();

extern void __kmp_clear_x87_fpu_status_word();

# define KMP_X86_MXCSR_MASK      0xffffffc0   /* ignore status flags (6 lsb) */

# define KMP_X86_MXCSR_MASK      0xffffffc0   /* ignore status flags (6 lsb) */

extern int __kmp_invoke_microtask( microtask_t pkfn, int gtid, int npr, int argc, void *argv[]

extern int __kmp_invoke_microtask( microtask_t pkfn, int gtid, int npr, int argc, void *argv[]

#if OMPT_SUPPORT

#if OMPT_SUPPORT

# endif /* KMP_GROUP_AFFINITY */

# endif /* KMP_GROUP_AFFINITY */

static int

static int

__kmp_cpuid_mask_width(int count) {

__kmp_cpuid_mask_width(int count) {

}

}

#define osIdIndex       0

#define osIdIndex       0

        }

        }

# endif

# endif

        if (depth < 0) {

        if (depth < 0) {

            if (__kmp_affinity_verbose) {

            if (__kmp_affinity_verbose) {

            }

            }

        }

        }

# if KMP_OS_LINUX

# if KMP_OS_LINUX

    // group affinity, which might have been implicitly set.

    // group affinity, which might have been implicitly set.

    //

    //

    else if (__kmp_affinity_top_method == affinity_top_method_x2apicid) {

    else if (__kmp_affinity_top_method == affinity_top_method_x2apicid) {

        if (__kmp_affinity_verbose) {

        if (__kmp_affinity_verbose) {

        }

        }

    }

    }

    else if (__kmp_affinity_top_method == affinity_top_method_cpuinfo) {

    else if (__kmp_affinity_top_method == affinity_top_method_cpuinfo) {

        const char *filename;

        const char *filename;

// end of the first part of the workaround for C78287

// end of the first part of the workaround for C78287

#endif // USE_CMPXCHG_FIX

#endif // USE_CMPXCHG_FIX

// ------------------------------------------------------------------------

// ------------------------------------------------------------------------

// X86 or X86_64: no alignment problems ====================================

// X86 or X86_64: no alignment problems ====================================

}

}

// end of the second part of the workaround for C78287

// end of the second part of the workaround for C78287

#endif // USE_CMPXCHG_FIX

#endif // USE_CMPXCHG_FIX

// Routines for ATOMIC 4-byte operands addition and subtraction

// Routines for ATOMIC 4-byte operands addition and subtraction

ATOMIC_FIXED_ADD( fixed4, add, kmp_int32,  32, +, 4i, 3, 0            )  // __kmpc_atomic_fixed4_add

ATOMIC_FIXED_ADD( fixed4, add, kmp_int32,  32, +, 4i, 3, 0            )  // __kmpc_atomic_fixed4_add

    OP_CRITICAL( = *lhs OP, LCK_ID )                                      \

    OP_CRITICAL( = *lhs OP, LCK_ID )                                      \

}

}

// ------------------------------------------------------------------------

// ------------------------------------------------------------------------

// X86 or X86_64: no alignment problems ===================================

// X86 or X86_64: no alignment problems ===================================

        OP_CRITICAL(= *lhs OP,LCK_ID)  /* unaligned - use critical */     \

        OP_CRITICAL(= *lhs OP,LCK_ID)  /* unaligned - use critical */     \

    }                                                                     \

    }                                                                     \

}

}

ATOMIC_CMPX_L( fixed1, andl, char,       8, &&, 1i, 0, KMP_ARCH_X86 )  // __kmpc_atomic_fixed1_andl

ATOMIC_CMPX_L( fixed1, andl, char,       8, &&, 1i, 0, KMP_ARCH_X86 )  // __kmpc_atomic_fixed1_andl

ATOMIC_CMPX_L( fixed1,  orl, char,       8, ||, 1i, 0, KMP_ARCH_X86 )  // __kmpc_atomic_fixed1_orl

ATOMIC_CMPX_L( fixed1,  orl, char,       8, ||, 1i, 0, KMP_ARCH_X86 )  // __kmpc_atomic_fixed1_orl

    }                                                                      \

    }                                                                      \

}

}

// -------------------------------------------------------------------------

// -------------------------------------------------------------------------

// X86 or X86_64: no alignment problems ====================================

// X86 or X86_64: no alignment problems ====================================

        }                                                                  \

        }                                                                  \

    }                                                                      \

    }                                                                      \

}

}

MIN_MAX_COMPXCHG( fixed1,  max, char,        8, <, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_max

MIN_MAX_COMPXCHG( fixed1,  max, char,        8, <, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_max

MIN_MAX_COMPXCHG( fixed1,  min, char,        8, >, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_min

MIN_MAX_COMPXCHG( fixed1,  min, char,        8, >, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_min

}

}

// ------------------------------------------------------------------------

// ------------------------------------------------------------------------

// ------------------------------------------------------------------------

// ------------------------------------------------------------------------

// X86 or X86_64: no alignment problems ===================================

// X86 or X86_64: no alignment problems ===================================

#define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \

#define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \

        OP_CRITICAL(^=~,LCK_ID)    /* unaligned address - use critical */ \

        OP_CRITICAL(^=~,LCK_ID)    /* unaligned address - use critical */ \

    }                                                                     \

    }                                                                     \

}

}

ATOMIC_CMPXCHG(  fixed1, neqv, kmp_int8,   8,   ^, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_neqv

ATOMIC_CMPXCHG(  fixed1, neqv, kmp_int8,   8,   ^, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_neqv

ATOMIC_CMPXCHG(  fixed2, neqv, kmp_int16, 16,   ^, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_neqv

ATOMIC_CMPXCHG(  fixed2, neqv, kmp_int16, 16,   ^, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_neqv

// OpenMP 4.0: x = expr binop x for non-commutative operations.

// OpenMP 4.0: x = expr binop x for non-commutative operations.

// Supported only on IA-32 architecture and Intel(R) 64

// Supported only on IA-32 architecture and Intel(R) 64

// ------------------------------------------------------------------------

// ------------------------------------------------------------------------

// Operation on *lhs, rhs bound by critical section

// Operation on *lhs, rhs bound by critical section

#endif

#endif

// End of OpenMP 4.0: x = expr binop x for non-commutative operations.

// End of OpenMP 4.0: x = expr binop x for non-commutative operations.

#endif //OMP_40_ENABLED

#endif //OMP_40_ENABLED

}

}

// -------------------------------------------------------------------------

// -------------------------------------------------------------------------

// -------------------------------------------------------------------------

// -------------------------------------------------------------------------

// X86 or X86_64: no alignment problems ====================================

// X86 or X86_64: no alignment problems ====================================

#define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \

#define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \

        OP_CRITICAL(OP##=,LCK_ID)  /* unaligned address - use critical */                   \

        OP_CRITICAL(OP##=,LCK_ID)  /* unaligned address - use critical */                   \

    }                                                                                       \

    }                                                                                       \

}

}

// RHS=float8

// RHS=float8

ATOMIC_CMPXCHG_MIX( fixed1, char,       mul,  8, *, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_mul_float8

ATOMIC_CMPXCHG_MIX( fixed1, char,       mul,  8, *, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_mul_float8

ATOMIC_CRITICAL_FP( float10, long double,    div, /, fp, _Quad, 10r,   1 )            // __kmpc_atomic_float10_div_fp

ATOMIC_CRITICAL_FP( float10, long double,    div, /, fp, _Quad, 10r,   1 )            // __kmpc_atomic_float10_div_fp

#endif

#endif

// ------------------------------------------------------------------------

// ------------------------------------------------------------------------

// X86 or X86_64: no alignment problems ====================================

// X86 or X86_64: no alignment problems ====================================

#if USE_CMPXCHG_FIX

#if USE_CMPXCHG_FIX

        OP_CRITICAL(OP##=,LCK_ID)  /* unaligned address - use critical */                     \

        OP_CRITICAL(OP##=,LCK_ID)  /* unaligned address - use critical */                     \

    }                                                                                         \

    }                                                                                         \

}

}

ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, add, 64, +, cmplx8,  kmp_cmplx64,  8c, 7, KMP_ARCH_X86 ) // __kmpc_atomic_cmplx4_add_cmplx8

ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, add, 64, +, cmplx8,  kmp_cmplx64,  8c, 7, KMP_ARCH_X86 ) // __kmpc_atomic_cmplx4_add_cmplx8

ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, sub, 64, -, cmplx8,  kmp_cmplx64,  8c, 7, KMP_ARCH_X86 ) // __kmpc_atomic_cmplx4_sub_cmplx8

ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, sub, 64, -, cmplx8,  kmp_cmplx64,  8c, 7, KMP_ARCH_X86 ) // __kmpc_atomic_cmplx4_sub_cmplx8

ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, div, 64, /, cmplx8,  kmp_cmplx64,  8c, 7, KMP_ARCH_X86 ) // __kmpc_atomic_cmplx4_div_cmplx8

ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, div, 64, /, cmplx8,  kmp_cmplx64,  8c, 7, KMP_ARCH_X86 ) // __kmpc_atomic_cmplx4_div_cmplx8

// READ, WRITE, CAPTURE are supported only on IA-32 architecture and Intel(R) 64

// READ, WRITE, CAPTURE are supported only on IA-32 architecture and Intel(R) 64

//////////////////////////////////////////////////////////////////////////////////////////////////////

//////////////////////////////////////////////////////////////////////////////////////////////////////

// ------------------------------------------------------------------------

// ------------------------------------------------------------------------

#endif //OMP_40_ENABLED

#endif //OMP_40_ENABLED

#undef OP_CRITICAL

#undef OP_CRITICAL

    if (

    if (

#if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)

#if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)

        FALSE                                   /* must use lock */

        FALSE                                   /* must use lock */

	TRUE					/* no alignment problems */

	TRUE					/* no alignment problems */

#else

#else

	! ( (kmp_uintptr_t) lhs & 0x1)		/* make sure address is 2-byte aligned */

	! ( (kmp_uintptr_t) lhs & 0x1)		/* make sure address is 2-byte aligned */

        // FIXME: On IA-32 architecture, gcc uses cmpxchg only for 4-byte ints.

        // FIXME: On IA-32 architecture, gcc uses cmpxchg only for 4-byte ints.

        // Gomp compatibility is broken if this routine is called for floats.

        // Gomp compatibility is broken if this routine is called for floats.

        //

        //

	TRUE					/* no alignment problems */

	TRUE					/* no alignment problems */

#else

#else

	! ( (kmp_uintptr_t) lhs & 0x3)		/* make sure address is 4-byte aligned */

	! ( (kmp_uintptr_t) lhs & 0x3)		/* make sure address is 4-byte aligned */

#if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)

#if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)

        FALSE                                   /* must use lock */

        FALSE                                   /* must use lock */

	TRUE					/* no alignment problems */

	TRUE					/* no alignment problems */

#else

#else

	! ( (kmp_uintptr_t) lhs & 0x7)		/* make sure address is 8-byte aligned */

	! ( (kmp_uintptr_t) lhs & 0x7)		/* make sure address is 8-byte aligned */

// OpenMP 4.0: x = expr binop x for non-commutative operations.

// OpenMP 4.0: x = expr binop x for non-commutative operations.

// Supported only on IA-32 architecture and Intel(R) 64

// Supported only on IA-32 architecture and Intel(R) 64

void __kmpc_atomic_fixed1_sub_rev(  ident_t *id_ref, int gtid, char * lhs, char rhs );

void __kmpc_atomic_fixed1_sub_rev(  ident_t *id_ref, int gtid, char * lhs, char rhs );

void __kmpc_atomic_fixed1_div_rev(  ident_t *id_ref, int gtid, char * lhs, char rhs );

void __kmpc_atomic_fixed1_div_rev(  ident_t *id_ref, int gtid, char * lhs, char rhs );

#endif

#endif

#endif // KMP_HAVE_QUAD

#endif // KMP_HAVE_QUAD

#endif //OMP_40_ENABLED

#endif //OMP_40_ENABLED

void __kmpc_atomic_32( ident_t *id_ref, int gtid, void* lhs, void* rhs, void (*f)( void *, void *, void * ) );

void __kmpc_atomic_32( ident_t *id_ref, int gtid, void* lhs, void* rhs, void (*f)( void *, void *, void * ) );

// READ, WRITE, CAPTURE are supported only on IA-32 architecture and Intel(R) 64

// READ, WRITE, CAPTURE are supported only on IA-32 architecture and Intel(R) 64

//

//

//  Below routines for atomic READ are listed

//  Below routines for atomic READ are listed

#endif //OMP_40_ENABLED

#endif //OMP_40_ENABLED

/* ------------------------------------------------------------------------ */

/* ------------------------------------------------------------------------ */

/* ------------------------------------------------------------------------ */

/* ------------------------------------------------------------------------ */

            VOLATILE_CAST(microtask_t) microtask, // "wrapped" task

            VOLATILE_CAST(microtask_t) microtask, // "wrapped" task

            VOLATILE_CAST(launch_t)    __kmp_invoke_task_func,

            VOLATILE_CAST(launch_t)    __kmp_invoke_task_func,

/* TODO: revert workaround for Intel(R) 64 tracker #96 */

/* TODO: revert workaround for Intel(R) 64 tracker #96 */

            &ap

            &ap

#else

#else

            ap

            ap

#endif

#endif

            VOLATILE_CAST(microtask_t) __kmp_teams_master, // "wrapped" task

            VOLATILE_CAST(microtask_t) __kmp_teams_master, // "wrapped" task

            VOLATILE_CAST(launch_t)    __kmp_invoke_teams_master,

            VOLATILE_CAST(launch_t)    __kmp_invoke_teams_master,

            &ap

            &ap

#else

#else

            ap

            ap

        /* return to the parallel section */

        /* return to the parallel section */

        if ( __kmp_inherit_fp_control && serial_team->t.t_fp_control_saved ) {

        if ( __kmp_inherit_fp_control && serial_team->t.t_fp_control_saved ) {

            __kmp_clear_x87_fpu_status_word();

            __kmp_clear_x87_fpu_status_word();

            __kmp_load_x87_fpu_control_word( &serial_team->t.t_x87_fpu_control_word );

            __kmp_load_x87_fpu_control_word( &serial_team->t.t_x87_fpu_control_word );

            __kmp_load_mxcsr( &serial_team->t.t_mxcsr );

            __kmp_load_mxcsr( &serial_team->t.t_mxcsr );

        }

        }

        this_thr -> th.th_team           = serial_team -> t.t_parent;

        this_thr -> th.th_team           = serial_team -> t.t_parent;

        this_thr -> th.th_info.ds.ds_tid = serial_team -> t.t_master_tid;

        this_thr -> th.th_info.ds.ds_tid = serial_team -> t.t_master_tid;

    /* need explicit __mf() here since use volatile instead in library */

    /* need explicit __mf() here since use volatile instead in library */

    KMP_MB();       /* Flush all pending memory write invalidates.  */

    KMP_MB();       /* Flush all pending memory write invalidates.  */

        #if KMP_MIC

        #if KMP_MIC

            // fence-style instructions do not exist, but lock; xaddl $0,(%rsp) can be used.

            // fence-style instructions do not exist, but lock; xaddl $0,(%rsp) can be used.

            // We shouldn't need it, though, since the ABI rules require that

            // We shouldn't need it, though, since the ABI rules require that

# define KMP_TSX_LOCK(seq) __kmp_user_lock_seq

# define KMP_TSX_LOCK(seq) __kmp_user_lock_seq

#endif

#endif

# define KMP_CPUINFO_RTM (__kmp_cpuinfo.rtm)

# define KMP_CPUINFO_RTM (__kmp_cpuinfo.rtm)

#else

#else

# define KMP_CPUINFO_RTM 0

# define KMP_CPUINFO_RTM 0

    if ( ( __kmp_user_lock_kind == lk_tas )

    if ( ( __kmp_user_lock_kind == lk_tas )

      && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {

      && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {

        // "fast" path implemented to fix customer performance issue

        // "fast" path implemented to fix customer performance issue

#if USE_ITT_BUILD

#if USE_ITT_BUILD

        __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );

        __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );

    if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )

    if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )

      + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {

      + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {

        // "fast" path implemented to fix customer performance issue

        // "fast" path implemented to fix customer performance issue

        kmp_tas_lock_t *tl = (kmp_tas_lock_t*)user_lock;

        kmp_tas_lock_t *tl = (kmp_tas_lock_t*)user_lock;

#if USE_ITT_BUILD

#if USE_ITT_BUILD

kmp_key_t __kmp_gtid_threadprivate_key;

kmp_key_t __kmp_gtid_threadprivate_key;

kmp_cpuinfo_t   __kmp_cpuinfo = { 0 }; // Not initialized

kmp_cpuinfo_t   __kmp_cpuinfo = { 0 }; // Not initialized

#endif

#endif

/* Barrier method defaults, settings, and strings */

/* Barrier method defaults, settings, and strings */

/* branch factor = 2^branch_bits (only relevant for tree and hyper barrier types) */

/* branch factor = 2^branch_bits (only relevant for tree and hyper barrier types) */

kmp_uint32 __kmp_barrier_gather_bb_dflt      = 2;  /* branch_factor = 4 */ /* hyper2: C78980 */

kmp_uint32 __kmp_barrier_gather_bb_dflt      = 2;  /* branch_factor = 4 */ /* hyper2: C78980 */

kmp_uint32 __kmp_barrier_release_bb_dflt     = 2;  /* branch_factor = 4 */ /* hyper2: C78980 */

kmp_uint32 __kmp_barrier_release_bb_dflt     = 2;  /* branch_factor = 4 */ /* hyper2: C78980 */

#else

#else

kmp_uint32 __kmp_barrier_gather_bb_dflt      = 2;  /* branch_factor = 4 */ /* communication in core for MIC */

kmp_uint32 __kmp_barrier_gather_bb_dflt      = 2;  /* branch_factor = 4 */ /* communication in core for MIC */

kmp_uint32 __kmp_barrier_release_bb_dflt     = 2;  /* branch_factor = 4 */ /* communication in core for MIC */

kmp_uint32 __kmp_barrier_release_bb_dflt     = 2;  /* branch_factor = 4 */ /* communication in core for MIC */

kmp_bar_pat_e __kmp_barrier_gather_pat_dflt  = bp_hyper_bar;  /* hyper2: C78980 */

kmp_bar_pat_e __kmp_barrier_gather_pat_dflt  = bp_hyper_bar;  /* hyper2: C78980 */

kmp_bar_pat_e __kmp_barrier_release_pat_dflt = bp_hyper_bar;  /* hyper2: C78980 */

kmp_bar_pat_e __kmp_barrier_release_pat_dflt = bp_hyper_bar;  /* hyper2: C78980 */

#else

#else

#endif /* KMP_TDATA_GTID */

#endif /* KMP_TDATA_GTID */

int          __kmp_tls_gtid_min = INT_MAX;

int          __kmp_tls_gtid_min = INT_MAX;

int            __kmp_foreign_tp = TRUE;

int            __kmp_foreign_tp = TRUE;

int    __kmp_inherit_fp_control = TRUE;

int    __kmp_inherit_fp_control = TRUE;

kmp_int16  __kmp_init_x87_fpu_control_word = 0;

kmp_int16  __kmp_init_x87_fpu_control_word = 0;

kmp_uint32     __kmp_init_mxcsr = 0;

kmp_uint32     __kmp_init_mxcsr = 0;

#ifdef USE_LOAD_BALANCE

#ifdef USE_LOAD_BALANCE

double  __kmp_load_balance_interval   = 1.0;

double  __kmp_load_balance_interval   = 1.0;

int __kmp_clock_function_param;

int __kmp_clock_function_param;

#endif /* KMP_OS_LINUX */

#endif /* KMP_OS_LINUX */

enum mic_type __kmp_mic_type = non_mic;

enum mic_type __kmp_mic_type = non_mic;

#endif

#endif

      VOLATILE_CAST(void *) unwrapped_task,

      VOLATILE_CAST(void *) unwrapped_task,

#endif

#endif

      wrapper, __kmp_invoke_task_func,

      wrapper, __kmp_invoke_task_func,

      &ap

      &ap

#else

#else

      ap

      ap

     * therefore uninteresting when collecting traces for architecture simulation.

     * therefore uninteresting when collecting traces for architecture simulation.

     */

     */

    #ifndef INCLUDE_SSC_MARKS

    #ifndef INCLUDE_SSC_MARKS

    #endif

    #endif

    /* Linux 64 only for now */

    /* Linux 64 only for now */

    // Portable (at least for gcc and icc) code to insert the necessary instructions

    // Portable (at least for gcc and icc) code to insert the necessary instructions

    // to set %ebx and execute the unlikely no-op.

    // to set %ebx and execute the unlikely no-op.

      #if defined( __INTEL_COMPILER )

      #if defined( __INTEL_COMPILER )

    int res = -1;

    int res = -1;

#if KMP_OS_WINDOWS

#if KMP_OS_WINDOWS

    _asm {

    _asm {

        _emit 0xC7

        _emit 0xC7

        _emit 0xF8

        _emit 0xF8

        mov   res, eax

        mov   res, eax

    L2:

    L2:

    }

    }

#else

#else

    /* Note that %eax must be noted as killed (clobbered), because

    /* Note that %eax must be noted as killed (clobbered), because

     * the XSR is returned in %eax(%rax) on abort.  Other register

     * the XSR is returned in %eax(%rax) on abort.  Other register

}

}

// Time stamp counter

// Time stamp counter

# define __kmp_tsc() __kmp_hardware_timestamp()

# define __kmp_tsc() __kmp_hardware_timestamp()

// Runtime's default backoff parameters

// Runtime's default backoff parameters

kmp_backoff_t __kmp_spin_backoff_params = { 1, 4096, 100 };

kmp_backoff_t __kmp_spin_backoff_params = { 1, 4096, 100 };

#define KMP_LOCK_ACQUIRED_FIRST 1

#define KMP_LOCK_ACQUIRED_FIRST 1

#define KMP_LOCK_ACQUIRED_NEXT  0

#define KMP_LOCK_ACQUIRED_NEXT  0

#if KMP_USE_FUTEX

#if KMP_USE_FUTEX

extern int ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );

extern int ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );

#define __kmp_acquire_user_lock_with_checks(lck,gtid)                                           \

#define __kmp_acquire_user_lock_with_checks(lck,gtid)                                           \

    if (__kmp_user_lock_kind == lk_tas) {                                                       \

    if (__kmp_user_lock_kind == lk_tas) {                                                       \

extern int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );

extern int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );

#include "kmp_i18n.h"                       /* AC: KMP_FATAL definition */

#include "kmp_i18n.h"                       /* AC: KMP_FATAL definition */

extern int __kmp_env_consistency_check;     /* AC: copy from kmp.h here */

extern int __kmp_env_consistency_check;     /* AC: copy from kmp.h here */

extern int ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );

extern int ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );

#define __kmp_acquire_nested_user_lock_with_checks(lck,gtid,depth)                                  \

#define __kmp_acquire_nested_user_lock_with_checks(lck,gtid,depth)                                  \

    if (__kmp_user_lock_kind == lk_tas) {                                                           \

    if (__kmp_user_lock_kind == lk_tas) {                                                           \

extern int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );

extern int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );

static inline int

static inline int

__kmp_test_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )

__kmp_test_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )

{

{

#include <stdint.h> // for uintptr_t

#include <stdint.h> // for uintptr_t

// Shortcuts

// Shortcuts

#define KMP_USE_INLINED_FUTEX KMP_USE_FUTEX && 0

#define KMP_USE_INLINED_FUTEX KMP_USE_FUTEX && 0

// List of lock definitions; all nested locks are indirect locks.

// List of lock definitions; all nested locks are indirect locks.

#if (KMP_OS_LINUX || KMP_OS_WINDOWS) && !KMP_OS_CNK && !KMP_ARCH_PPC64

#if (KMP_OS_LINUX || KMP_OS_WINDOWS) && !KMP_OS_CNK && !KMP_ARCH_PPC64

# define KMP_AFFINITY_SUPPORTED 1

# define KMP_AFFINITY_SUPPORTED 1

#  define KMP_GROUP_AFFINITY    1

#  define KMP_GROUP_AFFINITY    1

# else

# else

#  define KMP_GROUP_AFFINITY    0

#  define KMP_GROUP_AFFINITY    0

/* Check for quad-precision extension. */

/* Check for quad-precision extension. */

#define KMP_HAVE_QUAD 0

#define KMP_HAVE_QUAD 0

# if KMP_COMPILER_ICC

# if KMP_COMPILER_ICC

   /* _Quad is already defined for icc */

   /* _Quad is already defined for icc */

#  undef  KMP_HAVE_QUAD

#  undef  KMP_HAVE_QUAD

#  undef  KMP_HAVE_QUAD

#  undef  KMP_HAVE_QUAD

#  define KMP_HAVE_QUAD 1

#  define KMP_HAVE_QUAD 1

# endif

# endif

#if KMP_OS_WINDOWS

#if KMP_OS_WINDOWS

  typedef char              kmp_int8;

  typedef char              kmp_int8;

# define KMP_UINT64_SPEC     "llu"

# define KMP_UINT64_SPEC     "llu"

#endif /* KMP_OS_UNIX */

#endif /* KMP_OS_UNIX */

# define KMP_SIZE_T_SPEC KMP_UINT32_SPEC

# define KMP_SIZE_T_SPEC KMP_UINT32_SPEC

#elif KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64

#elif KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64

# define KMP_SIZE_T_SPEC KMP_UINT64_SPEC

# define KMP_SIZE_T_SPEC KMP_UINT64_SPEC

# error "Can't determine size_t printf format specifier."

# error "Can't determine size_t printf format specifier."

#endif

#endif

# define KMP_SIZE_T_MAX (0xFFFFFFFF)

# define KMP_SIZE_T_MAX (0xFFFFFFFF)

#else

#else

# define KMP_SIZE_T_MAX (0xFFFFFFFFFFFFFFFF)

# define KMP_SIZE_T_MAX (0xFFFFFFFFFFFFFFFF)

# define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )

# define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )

# define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )

# define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )

#  define KMP_COMPARE_AND_STORE_PTR(p, cv, sv)  __kmp_compare_and_store32( (volatile kmp_int32*)(p), (kmp_int32)(cv), (kmp_int32)(sv) )

#  define KMP_COMPARE_AND_STORE_PTR(p, cv, sv)  __kmp_compare_and_store32( (volatile kmp_int32*)(p), (kmp_int32)(cv), (kmp_int32)(sv) )

# else /* 64 bit pointers */

# else /* 64 bit pointers */

#  define KMP_COMPARE_AND_STORE_PTR(p, cv, sv)  __kmp_compare_and_store64( (volatile kmp_int64*)(p), (kmp_int64)(cv), (kmp_int64)(sv) )

#  define KMP_COMPARE_AND_STORE_PTR(p, cv, sv)  __kmp_compare_and_store64( (volatile kmp_int64*)(p), (kmp_int64)(cv), (kmp_int64)(sv) )

# define KMP_COMPARE_AND_STORE_RET8(p, cv, sv)  __kmp_compare_and_store_ret8( (p), (cv), (sv) )

# define KMP_COMPARE_AND_STORE_RET8(p, cv, sv)  __kmp_compare_and_store_ret8( (p), (cv), (sv) )

# define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) __kmp_compare_and_store_ret16( (p), (cv), (sv) )

# define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) __kmp_compare_and_store_ret16( (p), (cv), (sv) )

# define KMP_XCHG_REAL64(p, v)                  __kmp_xchg_real64( (p), (v) );

# define KMP_XCHG_REAL64(p, v)                  __kmp_xchg_real64( (p), (v) );

# define KMP_TEST_THEN_ADD8(p, v)               __sync_fetch_and_add( (kmp_int8 *)(p), (v) )

# define KMP_TEST_THEN_ADD8(p, v)               __sync_fetch_and_add( (kmp_int8 *)(p), (v) )

/* cast p to correct type so that proper intrinsic will be used */

/* cast p to correct type so that proper intrinsic will be used */

# define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )

# define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )

# define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )

# define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )

#  define KMP_COMPARE_AND_STORE_PTR(p, cv, sv)  __kmp_compare_and_store32( (volatile kmp_int32*)(p), (kmp_int32)(cv), (kmp_int32)(sv) )

#  define KMP_COMPARE_AND_STORE_PTR(p, cv, sv)  __kmp_compare_and_store32( (volatile kmp_int32*)(p), (kmp_int32)(cv), (kmp_int32)(sv) )

# else /* 64 bit pointers */

# else /* 64 bit pointers */

#  define KMP_COMPARE_AND_STORE_PTR(p, cv, sv)  __kmp_compare_and_store64( (volatile kmp_int64*)(p), (kmp_int64)(cv), (kmp_int64)(sv) )

#  define KMP_COMPARE_AND_STORE_PTR(p, cv, sv)  __kmp_compare_and_store64( (volatile kmp_int64*)(p), (kmp_int64)(cv), (kmp_int64)(sv) )

# define KMP_COMPARE_AND_STORE_RET8(p, cv, sv)  __kmp_compare_and_store_ret8( (p), (cv), (sv) )

# define KMP_COMPARE_AND_STORE_RET8(p, cv, sv)  __kmp_compare_and_store_ret8( (p), (cv), (sv) )

# define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) __kmp_compare_and_store_ret16( (p), (cv), (sv) )

# define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) __kmp_compare_and_store_ret16( (p), (cv), (sv) )

#define TCW_SYNC_8(a,b)     (a) = (b)

#define TCW_SYNC_8(a,b)     (a) = (b)

#define TCX_SYNC_8(a,b,c)   KMP_COMPARE_AND_STORE_REL64((volatile kmp_int64 *)(volatile void *)&(a), (kmp_int64)(b), (kmp_int64)(c))

#define TCX_SYNC_8(a,b,c)   KMP_COMPARE_AND_STORE_REL64((volatile kmp_int64 *)(volatile void *)&(a), (kmp_int64)(b), (kmp_int64)(c))

// What about ARM?

// What about ARM?

    #define TCR_PTR(a)          ((void *)TCR_4(a))

    #define TCR_PTR(a)          ((void *)TCR_4(a))

    #define TCW_PTR(a,b)        TCW_4((a),(b))

    #define TCW_PTR(a,b)        TCW_4((a),(b))

    #define TCW_SYNC_PTR(a,b)   TCW_SYNC_8((a),(b))

    #define TCW_SYNC_PTR(a,b)   TCW_SYNC_8((a),(b))

    #define TCX_SYNC_PTR(a,b,c) ((void *)TCX_SYNC_8((a),(b),(c)))

    #define TCX_SYNC_PTR(a,b,c) ((void *)TCX_SYNC_8((a),(b),(c)))

/*

/*

 * If these FTN_{TRUE,FALSE} values change, may need to

 * If these FTN_{TRUE,FALSE} values change, may need to

#define KMP_LE                   __kmp_le_4

#define KMP_LE                   __kmp_le_4

/* Workaround for Intel(R) 64 code gen bug when taking address of static array (Intel(R) 64 Tracker #138) */

/* Workaround for Intel(R) 64 code gen bug when taking address of static array (Intel(R) 64 Tracker #138) */

# define STATIC_EFI2_WORKAROUND

# define STATIC_EFI2_WORKAROUND

#else

#else

# define STATIC_EFI2_WORKAROUND static

# define STATIC_EFI2_WORKAROUND static

// Enable TSX if dynamic user lock is turned on

// Enable TSX if dynamic user lock is turned on

#if KMP_USE_DYNAMIC_LOCK

#if KMP_USE_DYNAMIC_LOCK

// Visual studio can't handle the asm sections in this code

// Visual studio can't handle the asm sections in this code

# ifdef KMP_USE_ADAPTIVE_LOCKS

# ifdef KMP_USE_ADAPTIVE_LOCKS

#  undef KMP_USE_ADAPTIVE_LOCKS

#  undef KMP_USE_ADAPTIVE_LOCKS

# endif

# endif

// Enable tick time conversion of ticks to seconds

// Enable tick time conversion of ticks to seconds

#if KMP_STATS_ENABLED

#if KMP_STATS_ENABLED

#endif

#endif

// Warning levels

// Warning levels

#define KMP_ARCH_X86        0

#define KMP_ARCH_X86        0

#define KMP_ARCH_X86_64     0

#define KMP_ARCH_X86_64     0

#define KMP_ARCH_AARCH64    0

#define KMP_ARCH_AARCH64    0

#define KMP_ARCH_PPC64_BE   0

#define KMP_ARCH_PPC64_BE   0

#define KMP_ARCH_PPC64_LE   0

#define KMP_ARCH_PPC64_LE   0

#if KMP_OS_UNIX

#if KMP_OS_UNIX

# if defined __x86_64

# if defined __x86_64

# elif defined __i386

# elif defined __i386

#  undef KMP_ARCH_X86

#  undef KMP_ARCH_X86

#  define KMP_ARCH_X86 1

#  define KMP_ARCH_X86 1

#define KMP_32_BIT_ARCH (KMP_ARCH_X86 || KMP_ARCH_ARM)

#define KMP_32_BIT_ARCH (KMP_ARCH_X86 || KMP_ARCH_ARM)

// TODO: Fixme - This is clever, but really fugly

// TODO: Fixme - This is clever, but really fugly

# error Unknown or unsupported architecture

# error Unknown or unsupported architecture

#endif

#endif

    KMP_MB();

    KMP_MB();

}

}

//

//

// Propagate any changes to the floating point control registers out to the team

// Propagate any changes to the floating point control registers out to the team

// We try to avoid unnecessary writes to the relevant cache line in the team structure,

// We try to avoid unnecessary writes to the relevant cache line in the team structure,

#else

#else

# define propagateFPControl(x) ((void)0)

# define propagateFPControl(x) ((void)0)

# define updateHWFPControl(x)  ((void)0)

# define updateHWFPControl(x)  ((void)0)

static void

static void

__kmp_alloc_argv_entries( int argc, kmp_team_t *team, int realloc ); // forward declaration

__kmp_alloc_argv_entries( int argc, kmp_team_t *team, int realloc ); // forward declaration

    microtask_t microtask,

    microtask_t microtask,

    launch_t    invoker,

    launch_t    invoker,

/* TODO: revert workaround for Intel(R) 64 tracker #96 */

/* TODO: revert workaround for Intel(R) 64 tracker #96 */

    va_list   * ap

    va_list   * ap

#else

#else

    va_list     ap

    va_list     ap

        argv = (void**)parent_team->t.t_argv;

        argv = (void**)parent_team->t.t_argv;

        for( i=argc-1; i >= 0; --i )

        for( i=argc-1; i >= 0; --i )

/* TODO: revert workaround for Intel(R) 64 tracker #96 */

/* TODO: revert workaround for Intel(R) 64 tracker #96 */

            *argv++ = va_arg( *ap, void * );

            *argv++ = va_arg( *ap, void * );

#else

#else

            *argv++ = va_arg( ap, void * );

            *argv++ = va_arg( ap, void * );

    /* create a serialized parallel region? */

    /* create a serialized parallel region? */

    if ( nthreads == 1 ) {

    if ( nthreads == 1 ) {

        /* josh todo: hypothetical question: what do we do for OS X*? */

        /* josh todo: hypothetical question: what do we do for OS X*? */

        void *   args[ argc ];

        void *   args[ argc ];

#else

#else

        void * * args = (void**) KMP_ALLOCA( argc * sizeof( void * ) );

        void * * args = (void**) KMP_ALLOCA( argc * sizeof( void * ) );

        KA_TRACE( 20, ("__kmp_fork_call: T#%d serializing parallel region\n", gtid ));

        KA_TRACE( 20, ("__kmp_fork_call: T#%d serializing parallel region\n", gtid ));

                if ( ap ) {

                if ( ap ) {

                    for( i=argc-1; i >= 0; --i )

                    for( i=argc-1; i >= 0; --i )

// TODO: revert workaround for Intel(R) 64 tracker #96

// TODO: revert workaround for Intel(R) 64 tracker #96

                        *argv++ = va_arg( *ap, void * );

                        *argv++ = va_arg( *ap, void * );

# else

# else

                        *argv++ = va_arg( ap, void * );

                        *argv++ = va_arg( ap, void * );

                argv = args;

                argv = args;

                for( i=argc-1; i >= 0; --i )

                for( i=argc-1; i >= 0; --i )

// TODO: revert workaround for Intel(R) 64 tracker #96

// TODO: revert workaround for Intel(R) 64 tracker #96

                    *argv++ = va_arg( *ap, void * );

                    *argv++ = va_arg( *ap, void * );

#else

#else

                    *argv++ = va_arg( ap, void * );

                    *argv++ = va_arg( ap, void * );

#endif /* OMP_40_ENABLED */

#endif /* OMP_40_ENABLED */

        for ( i=argc-1; i >= 0; --i ) {

        for ( i=argc-1; i >= 0; --i ) {

// TODO: revert workaround for Intel(R) 64 tracker #96

// TODO: revert workaround for Intel(R) 64 tracker #96

            void *new_argv = va_arg(*ap, void *);

            void *new_argv = va_arg(*ap, void *);

#else

#else

            void *new_argv = va_arg(ap, void *);

            void *new_argv = va_arg(ap, void *);

    // TODO???: team->t.t_max_active_levels       = new_max_active_levels;

    // TODO???: team->t.t_max_active_levels       = new_max_active_levels;

    team->t.t_sched       = new_icvs->sched;

    team->t.t_sched       = new_icvs->sched;

    team->t.t_fp_control_saved = FALSE; /* not needed */

    team->t.t_fp_control_saved = FALSE; /* not needed */

    team->t.t_x87_fpu_control_word = 0; /* not needed */

    team->t.t_x87_fpu_control_word = 0; /* not needed */

    team->t.t_mxcsr = 0;                /* not needed */

    team->t.t_mxcsr = 0;                /* not needed */

    team->t.t_construct   = 0;

    team->t.t_construct   = 0;

    __kmp_init_lock( & team->t.t_single_lock );

    __kmp_init_lock( & team->t.t_single_lock );

// End of Library registration stuff.

// End of Library registration stuff.

// -------------------------------------------------------------------------------------------------

// -------------------------------------------------------------------------------------------------

static void __kmp_check_mic_type()

static void __kmp_check_mic_type()

{

{

    }

    }

}

}

static void

static void

__kmp_do_serial_initialize( void )

__kmp_do_serial_initialize( void )

    __kmp_runtime_initialize();

    __kmp_runtime_initialize();

    __kmp_check_mic_type();

    __kmp_check_mic_type();

#endif

#endif

        #undef kmp_reduction_barrier_release_bb

        #undef kmp_reduction_barrier_release_bb

        #undef kmp_reduction_barrier_gather_bb

        #undef kmp_reduction_barrier_gather_bb

    #endif // KMP_FAST_REDUCTION_BARRIER

    #endif // KMP_FAST_REDUCTION_BARRIER

    if (__kmp_mic_type == mic2) { // KNC

    if (__kmp_mic_type == mic2) { // KNC

        // AC: plane=3,2, forkjoin=2,1 are optimal for 240 threads on KNC

        // AC: plane=3,2, forkjoin=2,1 are optimal for 240 threads on KNC

        __kmp_barrier_gather_branch_bits [ bs_plain_barrier ] = 3;  // plain gather

        __kmp_barrier_gather_branch_bits [ bs_plain_barrier ] = 3;  // plain gather

    KA_TRACE( 10, ("__kmp_parallel_initialize: enter\n" ) );

    KA_TRACE( 10, ("__kmp_parallel_initialize: enter\n" ) );

    KMP_ASSERT( KMP_UBER_GTID( gtid ) );

    KMP_ASSERT( KMP_UBER_GTID( gtid ) );

    //

    //

    // Save the FP control regs.

    // Save the FP control regs.

    // Worker threads will set theirs to these values at thread startup.

    // Worker threads will set theirs to these values at thread startup.

    __kmp_store_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );

    __kmp_store_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );

    __kmp_store_mxcsr( &__kmp_init_mxcsr );

    __kmp_store_mxcsr( &__kmp_init_mxcsr );

    __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK;

    __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK;

#if KMP_OS_UNIX

#if KMP_OS_UNIX

# if KMP_HANDLE_SIGNALS

# if KMP_HANDLE_SIGNALS

        int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;

        int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;

        int tree_available   = FAST_REDUCTION_TREE_METHOD_GENERATED;

        int tree_available   = FAST_REDUCTION_TREE_METHOD_GENERATED;

            #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN

            #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN

	    int teamsize_cutoff = 4;

	    int teamsize_cutoff = 4;

                if( __kmp_mic_type != non_mic ) {

                if( __kmp_mic_type != non_mic ) {

                    teamsize_cutoff = 8;

                    teamsize_cutoff = 8;

                }

                }

// KMP_INHERIT_FP_CONTROL

// KMP_INHERIT_FP_CONTROL

// -------------------------------------------------------------------------------------------------

// -------------------------------------------------------------------------------------------------

static void

static void

__kmp_stg_parse_inherit_fp_control( char const * name, char const * value, void * data ) {

__kmp_stg_parse_inherit_fp_control( char const * name, char const * value, void * data ) {

#endif /* KMP_DEBUG */

#endif /* KMP_DEBUG */

} // __kmp_stg_print_inherit_fp_control

} // __kmp_stg_print_inherit_fp_control

// -------------------------------------------------------------------------------------------------

// -------------------------------------------------------------------------------------------------

// KMP_LIBRARY, OMP_WAIT_POLICY

// KMP_LIBRARY, OMP_WAIT_POLICY

            }; // if

            }; // if

            if ( __kmp_affinity_gran == affinity_gran_default ) {

            if ( __kmp_affinity_gran == affinity_gran_default ) {

                if( __kmp_mic_type != non_mic ) {

                if( __kmp_mic_type != non_mic ) {

                    if( __kmp_affinity_verbose || __kmp_affinity_warnings ) {

                    if( __kmp_affinity_verbose || __kmp_affinity_warnings ) {

                        KMP_WARNING( AffGranUsing, "KMP_AFFINITY", "fine" );

                        KMP_WARNING( AffGranUsing, "KMP_AFFINITY", "fine" );

            // OMP_PROC_BIND => granularity=core,scatter elsewhere

            // OMP_PROC_BIND => granularity=core,scatter elsewhere

            //

            //

            __kmp_affinity_type = affinity_scatter;

            __kmp_affinity_type = affinity_scatter;

            if( __kmp_mic_type != non_mic )

            if( __kmp_mic_type != non_mic )

                __kmp_affinity_gran = affinity_gran_fine;

                __kmp_affinity_gran = affinity_gran_fine;

            else

            else

    if ( __kmp_str_match( "all", 1, value ) ) {

    if ( __kmp_str_match( "all", 1, value ) ) {

       __kmp_affinity_top_method = affinity_top_method_all;

       __kmp_affinity_top_method = affinity_top_method_all;

    }

    }

    else if ( __kmp_str_match( "x2apic id", 9, value )

    else if ( __kmp_str_match( "x2apic id", 9, value )

      || __kmp_str_match( "x2apic_id", 9, value )

      || __kmp_str_match( "x2apic_id", 9, value )

      || __kmp_str_match( "x2apic-id", 9, value )

      || __kmp_str_match( "x2apic-id", 9, value )

      || __kmp_str_match( "leaf4", 5, value ) ) {

      || __kmp_str_match( "leaf4", 5, value ) ) {

        __kmp_affinity_top_method = affinity_top_method_apicid;

        __kmp_affinity_top_method = affinity_top_method_apicid;

    }

    }

    else if ( __kmp_str_match( "/proc/cpuinfo", 2, value )

    else if ( __kmp_str_match( "/proc/cpuinfo", 2, value )

      || __kmp_str_match( "cpuinfo", 5, value )) {

      || __kmp_str_match( "cpuinfo", 5, value )) {

        __kmp_affinity_top_method = affinity_top_method_cpuinfo;

        __kmp_affinity_top_method = affinity_top_method_cpuinfo;

        value = "all";

        value = "all";

        break;

        break;

        case affinity_top_method_x2apicid:

        case affinity_top_method_x2apicid:

        value = "x2APIC id";

        value = "x2APIC id";

        break;

        break;

        case affinity_top_method_apicid:

        case affinity_top_method_apicid:

        value = "APIC id";

        value = "APIC id";

        break;

        break;

        case affinity_top_method_cpuinfo:

        case affinity_top_method_cpuinfo:

        value = "cpuinfo";

        value = "cpuinfo";

    { "KMP_HANDLE_SIGNALS",                __kmp_stg_parse_handle_signals,     __kmp_stg_print_handle_signals,     NULL, 0, 0 },

    { "KMP_HANDLE_SIGNALS",                __kmp_stg_parse_handle_signals,     __kmp_stg_print_handle_signals,     NULL, 0, 0 },

#endif

#endif

    { "KMP_INHERIT_FP_CONTROL",            __kmp_stg_parse_inherit_fp_control, __kmp_stg_print_inherit_fp_control, NULL, 0, 0 },

    { "KMP_INHERIT_FP_CONTROL",            __kmp_stg_parse_inherit_fp_control, __kmp_stg_print_inherit_fp_control, NULL, 0, 0 },

#ifdef KMP_GOMP_COMPAT

#ifdef KMP_GOMP_COMPAT

    { "GOMP_STACKSIZE",                    __kmp_stg_parse_stacksize,          NULL,                               NULL, 0, 0 },

    { "GOMP_STACKSIZE",                    __kmp_stg_parse_stacksize,          NULL,                               NULL, 0, 0 },

# endif /* OMP_40_ENABLED */

# endif /* OMP_40_ENABLED */

                if ( __kmp_affinity_type == affinity_default ) {

                if ( __kmp_affinity_type == affinity_default ) {

#if OMP_40_ENABLED

#if OMP_40_ENABLED

                    if( __kmp_mic_type != non_mic ) {

                    if( __kmp_mic_type != non_mic ) {

                        __kmp_nested_proc_bind.bind_types[0] = proc_bind_intel;

                        __kmp_nested_proc_bind.bind_types[0] = proc_bind_intel;

                    } else

                    } else

                        __kmp_nested_proc_bind.bind_types[0] = proc_bind_false;

                        __kmp_nested_proc_bind.bind_types[0] = proc_bind_false;

                    }

                    }

#endif /* OMP_40_ENABLED */

#endif /* OMP_40_ENABLED */

                    if( __kmp_mic_type != non_mic ) {

                    if( __kmp_mic_type != non_mic ) {

                        __kmp_affinity_type = affinity_scatter;

                        __kmp_affinity_type = affinity_scatter;

                    } else

                    } else

                }

                }

                if ( ( __kmp_affinity_gran == affinity_gran_default )

                if ( ( __kmp_affinity_gran == affinity_gran_default )

                  &&  ( __kmp_affinity_gran_levels < 0 ) ) {

                  &&  ( __kmp_affinity_gran_levels < 0 ) ) {

                    if( __kmp_mic_type != non_mic ) {

                    if( __kmp_mic_type != non_mic ) {

                        __kmp_affinity_gran = affinity_gran_fine;

                        __kmp_affinity_gran = affinity_gran_fine;

                    } else

                    } else

    fprintf (statsOut, "# Time of run: %s\n", &buffer[0]);

    fprintf (statsOut, "# Time of run: %s\n", &buffer[0]);

    if (gethostname(&hostName[0], sizeof(hostName)) == 0)

    if (gethostname(&hostName[0], sizeof(hostName)) == 0)

        fprintf (statsOut,"# Hostname: %s\n", &hostName[0]);

        fprintf (statsOut,"# Hostname: %s\n", &hostName[0]);

    fprintf (statsOut, "# CPU:  %s\n", &__kmp_cpuinfo.name[0]);

    fprintf (statsOut, "# CPU:  %s\n", &__kmp_cpuinfo.name[0]);

    fprintf (statsOut, "# Family: %d, Model: %d, Stepping: %d\n", __kmp_cpuinfo.family, __kmp_cpuinfo.model, __kmp_cpuinfo.stepping);

    fprintf (statsOut, "# Family: %d, Model: %d, Stepping: %d\n", __kmp_cpuinfo.family, __kmp_cpuinfo.model, __kmp_cpuinfo.stepping);

    if (__kmp_cpuinfo.frequency == 0)

    if (__kmp_cpuinfo.frequency == 0)

    // pretty bad assumption of 1GHz clock for MIC

    // pretty bad assumption of 1GHz clock for MIC

    return 1/((double)1000*1.e6);

    return 1/((double)1000*1.e6);

}

}

#  include <string.h>

#  include <string.h>

// Extract the value from the CPUID information

// Extract the value from the CPUID information

double tsc_tick_count::tick_time()

double tsc_tick_count::tick_time()

    if (in_parallel) {

    if (in_parallel) {

#if KMP_ARCH_X86 || \

#if KMP_ARCH_X86 || \

        KMP_TEST_THEN_OR32( &queue->tq_flags, (kmp_int32) TQF_ALL_TASKS_QUEUED );

        KMP_TEST_THEN_OR32( &queue->tq_flags, (kmp_int32) TQF_ALL_TASKS_QUEUED );

#else

#else

        }

        }

        else {

        else {

#if KMP_ARCH_X86 || \

#if KMP_ARCH_X86 || \

            KMP_TEST_THEN_OR32( &queue->tq_flags, (kmp_int32) TQF_IS_LAST_TASK );

            KMP_TEST_THEN_OR32( &queue->tq_flags, (kmp_int32) TQF_IS_LAST_TASK );

#else

#else

static const char *unknown = "unknown";

static const char *unknown = "unknown";

/* NOTE: If called before serial_initialize (i.e. from runtime_initialize), then */

/* NOTE: If called before serial_initialize (i.e. from runtime_initialize), then */

/* the debugging package has not been initialized yet, and only "0" will print   */

/* the debugging package has not been initialized yet, and only "0" will print   */

    }

    }

}

}

/* ------------------------------------------------------------------------------------ */

/* ------------------------------------------------------------------------------------ */

/* ------------------------------------------------------------------------------------ */

/* ------------------------------------------------------------------------------------ */

#include "kmp_config.h"

#include "kmp_config.h"

# if KMP_MIC

# if KMP_MIC

//

//

#  endif /* KMP_OS_DARWIN */

#  endif /* KMP_OS_DARWIN */

# endif /* KMP_ARCH_X86 */

# endif /* KMP_ARCH_X86 */

#  if KMP_OS_DARWIN

#  if KMP_OS_DARWIN

        .data

        .data

        .comm .gomp_critical_user_,32

        .comm .gomp_critical_user_,32

        .type __kmp_unnamed_critical_addr,@object

        .type __kmp_unnamed_critical_addr,@object

        .size __kmp_unnamed_critical_addr,8

        .size __kmp_unnamed_critical_addr,8

#  endif /* KMP_OS_DARWIN */

#  endif /* KMP_OS_DARWIN */

#endif /* KMP_GOMP_COMPAT */

#endif /* KMP_GOMP_COMPAT */

#endif /* KMP_ARCH_X86 */

#endif /* KMP_ARCH_X86 */

// -----------------------------------------------------------------------

// -----------------------------------------------------------------------

// microtasking routines specifically written for IA-32 architecture and

// microtasking routines specifically written for IA-32 architecture and

// -----------------------------------------------------------------------

// -----------------------------------------------------------------------

// '

// '

#if KMP_OS_LINUX && KMP_ARCH_AARCH64

#if KMP_OS_LINUX && KMP_ARCH_AARCH64

#    error Wrong code for getaffinity system call.

#    error Wrong code for getaffinity system call.

#   endif /* __NR_sched_getaffinity */

#   endif /* __NR_sched_getaffinity */

#  elif KMP_ARCH_X86_64

#  elif KMP_ARCH_X86_64

#   ifndef __NR_sched_setaffinity

#   ifndef __NR_sched_setaffinity

#    define __NR_sched_setaffinity  203

#    define __NR_sched_setaffinity  203

/* ------------------------------------------------------------------------ */

/* ------------------------------------------------------------------------ */

/* ------------------------------------------------------------------------ */

/* ------------------------------------------------------------------------ */

/*

/*

 * Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to

 * Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to

 * use compare_and_store for these routines

 * use compare_and_store for these routines

    return old_value;

    return old_value;

}

}

void

void

__kmp_terminate_thread( int gtid )

__kmp_terminate_thread( int gtid )

    KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );

    KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );

#endif

#endif

    //

    //

    // Set the FP control regs to be a copy of

    // Set the FP control regs to be a copy of

    // the parallel initialization thread's.

    // the parallel initialization thread's.

    __kmp_clear_x87_fpu_status_word();

    __kmp_clear_x87_fpu_status_word();

    __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );

    __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );

    __kmp_load_mxcsr( &__kmp_init_mxcsr );

    __kmp_load_mxcsr( &__kmp_init_mxcsr );

#ifdef KMP_BLOCK_SIGNALS

#ifdef KMP_BLOCK_SIGNALS

    status = sigfillset( & new_set );

    status = sigfillset( & new_set );

        return;

        return;

    }; // if

    }; // if

        if ( ! __kmp_cpuinfo.initialized ) {

        if ( ! __kmp_cpuinfo.initialized ) {

            __kmp_query_cpuid( &__kmp_cpuinfo );

            __kmp_query_cpuid( &__kmp_cpuinfo );

        }; // if

        }; // if

    __kmp_xproc = __kmp_get_xproc();

    __kmp_xproc = __kmp_get_xproc();

#endif // USE_LOAD_BALANCE

#endif // USE_LOAD_BALANCE

// we really only need the case with 1 argument, because CLANG always build

// we really only need the case with 1 argument, because CLANG always build

// a struct of pointers to shared variables referenced in the outlined function

// a struct of pointers to shared variables referenced in the outlined function

#include "kmp.h"

#include "kmp.h"

/* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to

/* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to

 * use compare_and_store for these routines

 * use compare_and_store for these routines

 */

 */

    return old_value;

    return old_value;

}

}

/* ------------------------------------------------------------------------ */

/* ------------------------------------------------------------------------ */

/* ------------------------------------------------------------------------ */

/* ------------------------------------------------------------------------ */

#endif /* USE_ITT_BUILD */